diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 951a045c..00000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -.git/ -node_modules -/web -docker/ -db/ -bin/ \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 0638f159..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,65 +0,0 @@ -pipeline { - agent none - - environment { - CI = 'true' - } - - stages { - - - stage('Prepare Web Packages') { - - agent { - label 'linux' - } - - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE'){ - sh 'cd /home/jenkins/go/src/infini.sh/console && git stash && git pull origin master && make clean' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true|| rm -rif web' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || git clone ssh://git@git.infini.ltd:64221/infini/console-ui.git web' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git stash' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm install' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm run build' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-linux' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-arm' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-darwin' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-win' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && GOROOT="/infini/go-pkgs/go-loongarch" GOPATH="/home/jenkins/go" make build-linux-loong64' - sh "cd /home/jenkins/go/src/infini.sh/console/docker && chmod a+x *.sh && perl -pi -e 's/\r\n/\n/g' *.sh && \ - cd /home/jenkins/go/src/infini.sh/console/web/docker && chmod a+x *.sh && perl -pi -e 's/\r\n/\n/g' *.sh" - - sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' - - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.tpl bin/config' - - sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config' - sh label: 'package-linux-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-386.tar.gz console-linux-386 console.yml LICENSE NOTICE config' - sh label: 'package-linux-mips', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-mips.tar.gz console-linux-mips console.yml LICENSE NOTICE config' - sh label: 'package-linux-mipsle', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-mipsle.tar.gz console-linux-mipsle console.yml LICENSE NOTICE config' - sh label: 'package-linux-mips64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-mips64.tar.gz console-linux-mips64 console.yml LICENSE NOTICE config' - sh label: 'package-linux-mips64le', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-mips64le.tar.gz console-linux-mips64le console.yml LICENSE NOTICE config' - sh label: 'package-linux-loong64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-loong64.tar.gz console-linux-loong64 console.yml LICENSE NOTICE config' - sh label: 'package-linux-riscv64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-riscv64.tar.gz console-linux-riscv64 console.yml LICENSE NOTICE config' - sh label: 'package-linux-arm5', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-arm5.tar.gz console-linux-armv5 console.yml LICENSE NOTICE config' - sh label: 'package-linux-arm6', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-arm6.tar.gz console-linux-armv6 console.yml LICENSE NOTICE config' - sh label: 'package-linux-arm7', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-arm7.tar.gz console-linux-armv7 console.yml LICENSE NOTICE config' - sh label: 'package-linux-arm64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-arm64.tar.gz console-linux-arm64 console.yml LICENSE NOTICE config' - - sh label: 'package-mac-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && zip -r ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-mac-amd64.zip console-mac-amd64 console.yml LICENSE NOTICE config' - sh label: 'package-mac-arm64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && zip -r ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-mac-arm64.zip console-mac-arm64 console.yml LICENSE NOTICE config' - sh label: 'package-win-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && zip -r ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-windows-amd64.zip console-windows-amd64.exe console.yml LICENSE NOTICE config' - sh label: 'package-win-386', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && zip -r ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-windows-386.zip console-windows-386.exe console.yml LICENSE NOTICE config' - archiveArtifacts artifacts: 'console-$VERSION-$BUILD_NUMBER-*.*', fingerprint: true, followSymlinks: true, onlyIfSuccessful: false - } - } - } - - - - -} - -} diff --git a/Jenkinsfile-docker b/Jenkinsfile-docker deleted file mode 100644 index 660c0411..00000000 --- a/Jenkinsfile-docker +++ /dev/null @@ -1,41 +0,0 @@ -pipeline { - - agent none - - environment { - CI = 'true' - } - stages { - - stage('build') { - - parallel { - - stage('Build Docker Images') { - - agent { - label 'linux' - } - - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE'){ - sh 'cd /home/jenkins/go/src/infini.sh/console && git stash && git pull origin master && make clean' - - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true|| rm -rif web' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || git clone ssh://git@git.infini.ltd:64221/infini/console-ui.git web' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git stash' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm install' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm run build' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build && chmod a+x bin/console' - - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.tpl bin/config' - sh label: 'docker-build', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && docker build -t infini-console -f ../docker/Dockerfile .' - sh label: 'docker-tagging', script: 'docker tag infini-console infinilabs/console:latest && docker tag infini-console infinilabs/console:$VERSION-$BUILD_NUMBER' - sh label: 'docker-push', script: 'docker push infinilabs/console:$VERSION-$BUILD_NUMBER && docker push infinilabs/console:latest' - } - } - } - } } - } -} diff --git a/Jenkinsfile-linux b/Jenkinsfile-linux deleted file mode 100644 index 118dd4e8..00000000 --- a/Jenkinsfile-linux +++ /dev/null @@ -1,49 +0,0 @@ -pipeline { - agent none - - environment { - CI = 'true' - } - - stages { - - - stage('Prepare Web Packages') { - - agent { - label 'linux' - } - - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE'){ - sh 'cd /home/jenkins/go/src/infini.sh/console && git stash && git pull origin master && make clean' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || rm -rif web' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || git clone ssh://git@git.infini.ltd:64221/infini/console-ui.git web' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web/src && true || git clone ssh://git@git.infini.ltd:64221/infini/common-ui.git common' - sh 'cd /home/jenkins/go/src/infini.sh/console/web/src/common && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git stash' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm install' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm run build' - - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-linux-amd64' - sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.tpl bin/config' - sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config' - - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-arm' - sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp config/*.json bin/config && cp config/*.tpl bin/config' - sh label: 'package-linux-arm64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-arm64.tar.gz console-linux-arm64 console.yml LICENSE NOTICE config' - - archiveArtifacts artifacts: 'console-$VERSION-$BUILD_NUMBER-*.*', fingerprint: true, followSymlinks: true, onlyIfSuccessful: false - } - } - } - - - - -} - -} diff --git a/Jenkinsfile-linux-amd64 b/Jenkinsfile-linux-amd64 deleted file mode 100644 index a5945a23..00000000 --- a/Jenkinsfile-linux-amd64 +++ /dev/null @@ -1,43 +0,0 @@ -pipeline { - agent none - - environment { - CI = 'true' - } - - stages { - - - stage('Prepare Web Packages') { - - agent { - label 'linux' - } - - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE'){ - sh 'cd /home/jenkins/go/src/infini.sh/console && git stash && git pull origin master && make clean' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || rm -rif web' - sh 'cd /home/jenkins/go/src/infini.sh/console/ && true || git clone ssh://git@git.infini.ltd:64221/infini/console-ui.git web' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web/src && true || git clone ssh://git@git.infini.ltd:64221/infini/common-ui.git common' - sh 'cd /home/jenkins/go/src/infini.sh/console/web/src/common && git pull origin master' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && git stash' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm install' - sh 'cd /home/jenkins/go/src/infini.sh/console/web && cnpm run build' - sh 'cd /home/jenkins/go/src/infini.sh/console && git pull origin master && make config build-linux-amd64' - sh label: 'copy-license', script: 'cd /home/jenkins/go/src/infini.sh/console && cp ../framework/LICENSE bin && cat ../framework/NOTICE NOTICE > bin/NOTICE' - sh label: 'copy-configs', script: 'cd /home/jenkins/go/src/infini.sh/console && mkdir -p bin/config && cp -rf config/*.json bin/config && cp -rf config/*.tpl bin/config && cp -rf config/setup bin/config' - - sh label: 'package-linux-amd64', script: 'cd /home/jenkins/go/src/infini.sh/console/bin && tar cfz ${WORKSPACE}/console-$VERSION-$BUILD_NUMBER-linux-amd64.tar.gz console-linux-amd64 console.yml LICENSE NOTICE config' - archiveArtifacts artifacts: 'console-$VERSION-$BUILD_NUMBER-*.*', fingerprint: true, followSymlinks: true, onlyIfSuccessful: false - } - } - } - - - - -} - -} diff --git a/Makefile b/Makefile index 2e2098fb..d7ebbd24 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,6 @@ APP_STATIC_FOLDER := .public APP_STATIC_PACKAGE := public APP_UI_FOLDER := ui APP_PLUGIN_FOLDER := plugin -FRAMEWORK_BRANCH := console # GO15VENDOREXPERIMENT="1" GO111MODULE=off easyjson -all domain.go include ../framework/Makefile diff --git a/README.md b/README.md index e90e0f51..363cbbaf 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,11 @@ -#INFINI Cloud +#INFINI Console -INFINI Cloud for Elasticsearch +INFINI Console for Elasticsearch/OpenSearch/Easysearch ## 前端开发说明 前端采用 React 开发，最终输出为 `.public` 目录的纯静态资源，可以独立部署无需依赖 Node 环境。 -### Docker 开发环境准备 - -#### 安装 Docker - -#### 设置 Docker 国内镜像 - -修改 Docker engine 的设置，Windows 在 Docker Desktop 的 setting 里面，Linux 在 /etc/docker/daemon.json - -``` -{ - "registry-mirrors": [ - "https://registry.docker-cn.com", - "https://docker.mirrors.ustc.edu.cn/" - ], - "insecure-registries": [], - "debug": true, - "experimental": false -} -``` - -#### 启动开发环境 - -``` -cnpm run docker:dev -``` - -启动完成，稍等片刻，打开 http://localhost:8000/，手动刷新即可看到最新的更改。 - -#### 手动更新开发镜像 - -``` -docker login -u infini -p ltd docker.infini.ltd:64443 -docker pull docker.infini.ltd:64443/nodejs-dev:latest -``` - ### 本地开发环境准备 确保已经安装好`nodejs`（版本大于等于 8.5.0）环境： @@ -51,7 +16,7 @@ npm -v 在国内，你可以安装 `cnpm` 获得更快速、更安全的包管理体验。使用如下命令安装： ```sh -npm install -g cnpm --registry=https://registry.npm.taobao.org +npm install -g cnpm@9.2.0 --registry=https://registry.npm.taobao.org ``` ### 下载项目依赖包 @@ -75,10 +40,6 @@ cnpm run build 执行该命令后会生成最终的 HTML、CSS 和 JS 到 `/.public` 目录下。它们是浏览器可以直接识别并运行的代码，这样你就可以将它们部署到你想要的服务器上了。 -或者使用 Docker 来打包生成。 -``` -cnpm run docker:build -``` ### 新增项目依赖包 ``` diff --git a/build-web.sh b/build-web.sh deleted file mode 100644 index c70c1f39..00000000 --- a/build-web.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -WORKBASE=/home/jenkins/go/src/infini.sh/console - -if [ -d $WORKBASE/.public ]; then - echo "clean exists .pulbic folder." - rm -rf $WORKBASE/.public -fi - -if [ ! -d $WORKBASE/web ]; then - git clone ssh://git@git.infini.ltd:64221/infini/console-ui.git web -fi - -if [ ! -d $WORKBASE/web/src/common ]; then - cd $WORKBASE/web/src - git clone ssh://git@git.infini.ltd:64221/infini/common-ui.git common -fi - -cd $WORKBASE/web -git pull origin master - -cd $WORKBASE/web/src/common -git pull origin master - -git log --pretty=oneline -5 - -cd $WORKBASE/web - -#--quiet -cnpm install --quiet --no-progress -cnpm run clean -cnpm run build --quiet &>/dev/null diff --git a/build.sh b/build.sh deleted file mode 100644 index e3417a7c..00000000 --- a/build.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -#set -eo pipefail - -#init -WORKBASE=/home/jenkins/go/src/infini.sh -WORKDIR=$WORKBASE/$PNAME -DEST=/infini/Sync/Release/$PNAME/stable - -if [[ $VERSION =~ NIGHTLY ]]; then - BUILD_NUMBER=$BUILD_DAY - DEST=/infini/Sync/Release/$PNAME/snapshot -fi - -export DOCKER_CLI_EXPERIMENTAL=enabled - -#clean all -cd $WORKSPACE && git clean -fxd - -#pull code -cd $WORKDIR && git clean -fxd -e ".public" -git stash && git pull origin master - - #build -make clean config build-linux -make config build-arm -make config build-darwin -make config build-win -GOROOT="/infini/go-pkgs/go-loongarch" PATH=$GOROOT/bin:$PATH make build-linux-loong64 -#GOROOT="/infini/go-pkgs/go-swarch" PATH=$GOROOT/bin:$PATH make build-linux-sw64 - -#copy-configs -cp -rf $WORKBASE/framework/LICENSE $WORKDIR/bin && cat $WORKBASE/framework/NOTICE $WORKDIR/NOTICE > $WORKDIR/bin/NOTICE -mkdir -p $WORKDIR/bin/config -cp $WORKDIR/config/*.json $WORKDIR/bin/config -cp -rf $WORKDIR/config/*.tpl $WORKDIR/bin/config -[ -d $WORKDIR/config/setup ] && cp -rf $WORKDIR/config/setup $WORKDIR/bin/config - -cd $WORKDIR/bin -#编译出错后，根据文件是否存在判断是否进行下一步骤 -[ -f "$WORKDIR/bin/${PNAME}-linux-amd64" ] || exit - -for t in 386 amd64 arm64 armv5 armv6 armv7 loong64 mips mips64 mips64le mipsle riscv64 ; do - tar zcf ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-linux-$t.tar.gz "${PNAME}-linux-$t" $PNAME.yml LICENSE NOTICE config -done - -for t in mac-amd64 mac-arm64 windows-amd64 windows-386 ; do - zip -qr ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-$t.zip $PNAME-$t $PNAME.yml LICENSE NOTICE config -done - -for t in windows-amd64 windows-386 ; do - zip -qr ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-$t.zip $PNAME-$t.exe $PNAME.yml LICENSE NOTICE config -done - -#build image & push -for t in amd64 arm64 ; do - cat <Dockerfile -FROM --platform=linux/$t alpine:3.16.5 -MAINTAINER "hardy " -ARG APP_NAME=$PNAME -ARG APP_HOME=/ -ENV APP=\${APP_NAME} -WORKDIR / - -COPY ["$PNAME-linux-$t", "$PNAME.yml", "\${APP_HOME}/"] -COPY ["config", "\${APP_HOME}/config"] - -CMD ["/${PNAME}-linux-$t"] -EOF - - docker buildx build -t infinilabs/$PNAME-$t:latest --platform=linux/$t -o type=docker . - docker push infinilabs/$PNAME-$t:latest - docker tag infinilabs/$PNAME-$t:latest infinilabs/$PNAME-$t:$VERSION-$BUILD_NUMBER - docker push infinilabs/$PNAME-$t:$VERSION-$BUILD_NUMBER -done - -#composite tag -docker buildx imagetools create -t infinilabs/$PNAME:latest \ - infinilabs/$PNAME-arm64:latest \ - infinilabs/$PNAME-amd64:latest - -docker buildx imagetools create -t infinilabs/$PNAME:$VERSION-$BUILD_NUMBER \ - infinilabs/$PNAME-arm64:$VERSION-$BUILD_NUMBER \ - infinilabs/$PNAME-amd64:$VERSION-$BUILD_NUMBER - -#publish -for t in 386 amd64 arm64 armv5 armv6 armv7 loong64 mips mips64 mips64le mipsle riscv64 ; do - [ -f ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-linux-$t.tar.gz ] && ossuploader upload -p $PNAME -f ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-linux-$t.tar.gz - #cp -rf ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-linux-$t.tar.gz $DEST -done - -for t in mac-amd64 mac-arm64 windows-amd64 windows-386 ; do - [ -f ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-$t.zip ] && ossuploader upload -p $PNAME -f ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-$t.zip - #cp -rf ${WORKSPACE}/$PNAME-$VERSION-$BUILD_NUMBER-$t.zip $DEST -done - -#git reset -cd $WORKSPACE && git reset --hard -cd $WORKDIR && git reset --hard - -#clean weeks ago image -NEEDCLEN=$(docker images |grep "$PNAME" |grep "weeks ago") -if [ ! -z "$NEEDCLEN" ]; then - docker images |grep "$PNAME" |grep "weeks ago" |awk '{print $3}' |xargs docker rmi -f >/dev/null 2>&1 -fi diff --git a/config/setup/common/data/agent_relay_gateway_config.dat b/config/setup/common/data/agent_relay_gateway_config.dat index ec471064..b1410bdb 100644 --- a/config/setup/common/data/agent_relay_gateway_config.dat +++ b/config/setup/common/data/agent_relay_gateway_config.dat @@ -78,7 +78,7 @@ pipeline: processor: - bulk_indexing: max_connection_per_node: 100 - num_of_slices: 3 + num_of_slices: 1 max_worker_size: 30 idle_timeout_in_seconds: 10 bulk: diff --git a/config/system_config.tpl b/config/system_config.tpl index 8d625826..6b69cb08 100644 --- a/config/system_config.tpl +++ b/config/system_config.tpl @@ -63,18 +63,6 @@ pipeline: processor: - activity: elasticsearch: "$[[CLUSTER_ID]]" - - name: migration_task_dispatcher - auto_start: true - keep_running: true - retry_delay_in_ms: 1000 - processor: - - migration_dispatcher: - elasticsearch: "$[[CLUSTER_ID]]" - check_instance_available: true - max_tasks_per_instance: 10 - task_batch_size: 50 - when: - cluster_available: ["$[[CLUSTER_ID]]"] - name: merge_logging auto_start: true diff --git a/core/auth.go b/core/auth.go new file mode 100644 index 00000000..83ba29e1 --- /dev/null +++ b/core/auth.go @@ -0,0 +1,116 @@ +package core + +import ( + "infini.sh/console/core/security" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "net/http" +) + +// Handler is the object of http handler +type Handler struct { + api.Handler +} + +var authEnabled = false + +// BasicAuth register api with basic auth +func BasicAuth(h httprouter.Handle, requiredUser, requiredPassword string) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + // Get the Basic Authentication credentials + user, password, hasAuth := r.BasicAuth() + + if hasAuth && user == requiredUser && password == requiredPassword { + // Delegate request to the given handle + h(w, r, ps) + } else { + // Request Basic Authentication otherwise + w.Header().Set("WWW-Authenticate", "Basic realm=Restricted") + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + } + } +} + +func EnableAuth(enable bool) { + authEnabled = enable +} + +func IsAuthEnable() bool { + return authEnabled +} + +func (handler Handler) RequireLogin(h httprouter.Handle) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + if authEnabled { + claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + r = r.WithContext(security.NewUserContext(r.Context(), claims)) + } + + h(w, r, ps) + } +} + +func (handler Handler) RequirePermission(h httprouter.Handle, permissions ...string) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + if global.Env().SetupRequired() { + return + } + + if authEnabled { + claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + err = security.ValidatePermission(claims, permissions) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusForbidden) + return + } + r = r.WithContext(security.NewUserContext(r.Context(), claims)) + } + + h(w, r, ps) + } +} + +func (handler Handler) RequireClusterPermission(h httprouter.Handle, permissions ...string) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + if authEnabled { + id := ps.ByName("id") + claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + r = r.WithContext(security.NewUserContext(r.Context(), claims)) + hasAllPrivilege, clusterIDs := security.GetCurrentUserCluster(r) + if !hasAllPrivilege && (len(clusterIDs) == 0 || !util.StringInArray(clusterIDs, id)) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(http.StatusText(http.StatusForbidden))) + return + } + } + + h(w, r, ps) + } +} + +func (handler Handler) GetCurrentUser(req *http.Request) string { + if authEnabled { + claims, ok := req.Context().Value("user").(*security.UserClaims) + if ok { + return claims.Username + } + } + return "" +} diff --git a/core/elastic.go b/core/elastic.go new file mode 100644 index 00000000..cf98ce05 --- /dev/null +++ b/core/elastic.go @@ -0,0 +1,184 @@ +package core + +import ( + rbac "infini.sh/console/core/security" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "net/http" +) + +func (handler Handler) IndexRequired(h httprouter.Handle, route ...string) httprouter.Handle { + + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if authEnabled { + claims, err := rbac.ValidateLogin(r.Header.Get("Authorization")) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + newRole := rbac.CombineUserRoles(claims.Roles) + + indexReq := rbac.NewIndexRequest(ps, route) + + err = rbac.ValidateIndex(indexReq, newRole) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusForbidden) + return + } + } + + h(w, r, ps) + } +} + +func (handler Handler) ClusterRequired(h httprouter.Handle, route ...string) httprouter.Handle { + + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + if authEnabled { + claims, err := rbac.ValidateLogin(r.Header.Get("Authorization")) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + //newRole := biz.CombineUserRoles(claims.Roles) + clusterReq := rbac.NewClusterRequest(ps, route) + newRole := rbac.CombineUserRoles(claims.Roles) + err = rbac.ValidateCluster(clusterReq, newRole) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusForbidden) + return + } + } + + h(w, r, ps) + } +} + +func (handler Handler) GetClusterFilter(r *http.Request, field string) (util.MapStr, bool) { + if !IsAuthEnable() { + return nil, true + } + hasAllPrivilege, clusterIds := rbac.GetCurrentUserCluster(r) + if hasAllPrivilege { + return nil, true + } + if len(clusterIds) == 0 { + return nil, false + } + return util.MapStr{ + "terms": util.MapStr{ + field: clusterIds, + }, + }, false +} +func (handler Handler) GetAllowedClusters(r *http.Request) ([]string, bool) { + if !IsAuthEnable() { + return nil, true + } + hasAllPrivilege, clusterIds := rbac.GetCurrentUserCluster(r) + return clusterIds, hasAllPrivilege +} + +func (handler Handler) GetAllowedIndices(r *http.Request, clusterID string) ([]string, bool) { + if !IsAuthEnable() { + return nil, true + } + hasAllPrivilege, indices := handler.GetCurrentUserClusterIndex(r, clusterID) + if hasAllPrivilege { + return nil, true + } + return indices, false +} + +func (handler Handler) IsIndexAllowed(r *http.Request, clusterID string, indexName string) bool { + if !IsAuthEnable() { + return true + } + hasAllPrivilege, indices := handler.GetCurrentUserClusterIndex(r, clusterID) + if hasAllPrivilege { + return true + } + if len(indices) == 0 { + return false + } + return radix.Compile(indices...).Match(indexName) +} + +func (handler Handler) ValidateProxyRequest(req *http.Request, clusterID string) (bool, string, error) { + if !IsAuthEnable() { + return false, "", nil + } + claims, err := rbac.ValidateLogin(req.Header.Get("Authorization")) + if err != nil { + return false, "", err + } + if util.StringInArray(claims.Roles, rbac.RoleAdminName) { + return true, "", nil + } + + permission, params, matched := rbac.SearchAPIPermission("elasticsearch", req.Method, req.URL.Path) + if matched && permission != "" { + + newRole := rbac.CombineUserRoles(claims.Roles) + if indexName, ok := params["index_name"]; ok { + + indexReq := rbac.IndexRequest{ + Cluster: clusterID, + Index: indexName, + Privilege: []string{permission}, + } + + err = rbac.ValidateIndex(indexReq, newRole) + if err != nil { + return false, permission, err + } + } else { + clusterReq := rbac.ClusterRequest{ + Cluster: clusterID, + Privilege: []string{permission}, + } + err = rbac.ValidateCluster(clusterReq, newRole) + if err != nil { + return false, permission, err + } + } + } + return false, permission, nil +} + +func (handler Handler) GetCurrentUserIndex(req *http.Request) (bool, map[string][]string) { + if !IsAuthEnable() { + return true, nil + } + ctxVal := req.Context().Value("user") + if userClaims, ok := ctxVal.(*rbac.UserClaims); ok { + roles := userClaims.Roles + var realIndex = map[string][]string{} + for _, roleName := range roles { + role, ok := rbac.RoleMap[roleName] + if ok { + for _, ic := range role.Privilege.Elasticsearch.Cluster.Resources { + for _, ip := range role.Privilege.Elasticsearch.Index { + if ic.ID == "*" && util.StringInArray(ip.Name, "*") { + return true, nil + } + realIndex[ic.ID] = append(realIndex[ic.ID], ip.Name...) + } + } + } + } + return false, realIndex + } + return false, nil +} + +func (handler Handler) GetCurrentUserClusterIndex(req *http.Request, clusterID string) (bool, []string) { + ctxVal := req.Context().Value("user") + if userClaims, ok := ctxVal.(*rbac.UserClaims); ok { + return rbac.GetRoleIndex(userClaims.Roles, clusterID) + } else { + panic("user context value not found") + } +} diff --git a/core/security/access_token.go b/core/security/access_token.go new file mode 100644 index 00000000..d54c98e5 --- /dev/null +++ b/core/security/access_token.go @@ -0,0 +1,52 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package security + +import ( + "github.com/golang-jwt/jwt" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/util" + "time" +) + +func GenerateAccessToken(user *User) (map[string]interface{}, error) { + + var data map[string]interface{} + roles, privilege := user.GetPermissions() + + token1 := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ + ShortUser: &ShortUser{ + Provider: user.AuthProvider, + Username: user.Username, + UserId: user.ID, + Roles: roles, + }, + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)), + }, + }) + + tokenString, err := token1.SignedString([]byte(Secret)) + if tokenString == "" || err != nil { + return nil, errors.Errorf("failed to generate access_token for user: %v", user.Username) + } + + token := Token{ExpireIn: time.Now().Unix() + 86400} + SetUserToken(user.ID, token) + + data = util.MapStr{ + "access_token": tokenString, + "username": user.Username, + "id": user.ID, + "expire_in": 86400, + "roles": roles, + "privilege": privilege, + } + + data["status"] = "ok" + + return data, err + +} diff --git a/core/security/adapter.go b/core/security/adapter.go new file mode 100644 index 00000000..d2fedb8f --- /dev/null +++ b/core/security/adapter.go @@ -0,0 +1,53 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "fmt" + "infini.sh/framework/core/orm" +) + +type IRole interface { + Get(id string) (Role, error) + GetBy(field string, value interface{}) (Role, error) + Update(role *Role) error + Create(role *Role) (string, error) + Delete(id string) error + Search(keyword string, from, size int) (orm.Result, error) +} + +type IUser interface { + Get(id string) (User, error) + GetBy(field string, value interface{}) (*User, error) + Update(user *User) error + Create(user *User) (string, error) + Delete(id string) error + Search(keyword string, from, size int) (orm.Result, error) +} + +type SecurityRealm interface { + GetType() string + Authenticate(username, password string) (bool, *User, error) // Return true if authentication is successful, otherwise false + Authorize(user *User) (bool, error) // Return true if authorization is granted, otherwise false +} + +type Adapter struct { + Role IRole + User IUser +} + +var adapterHandlers = map[string]Adapter{} + +func RegisterAdapter(typ string, handler Adapter) { + adapterHandlers[typ] = handler +} + +func GetAdapter(typ string) Adapter { + handler, ok := adapterHandlers[typ] + if !ok { + panic(fmt.Errorf("dal handler %s not found", typ)) + } + return handler +} diff --git a/core/security/context.go b/core/security/context.go new file mode 100644 index 00000000..9e7fc751 --- /dev/null +++ b/core/security/context.go @@ -0,0 +1,43 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "context" + "fmt" + "github.com/golang-jwt/jwt" +) + +const ctxUserKey = "user" + +type UserClaims struct { + *jwt.RegisteredClaims + *ShortUser +} + +type ShortUser struct { + Provider string `json:"provider"` + Username string `json:"username"` + UserId string `json:"user_id"` + Roles []string `json:"roles"` +} + +const Secret = "console" + +func NewUserContext(ctx context.Context, clam *UserClaims) context.Context { + return context.WithValue(ctx, ctxUserKey, clam) +} + +func FromUserContext(ctx context.Context) (*ShortUser, error) { + ctxUser := ctx.Value(ctxUserKey) + if ctxUser == nil { + return nil, fmt.Errorf("user not found") + } + reqUser, ok := ctxUser.(*UserClaims) + if !ok { + return nil, fmt.Errorf("invalid context user") + } + return reqUser.ShortUser, nil +} diff --git a/core/security/enum/const.go b/core/security/enum/const.go new file mode 100644 index 00000000..4de6d514 --- /dev/null +++ b/core/security/enum/const.go @@ -0,0 +1,280 @@ +package enum + +var PermissionMap = make(map[string][]string) + +const ( + UserRead = "system.user:read" + UserAll = "system.user:all" + RoleRead = "system.role:read" + RoleAll = "system.role:all" + SecurityRead = "system.security:read" + SecurityAll = "system.security:all" + ClusterAll = "system.cluster:all" + ClusterRead = "system.cluster:read" + CommandAll = "system.command:all" + CommandRead = "system.command:read" + CredentialAll = "system.credential:all" + CredentialRead = "system.credential:read" + + InstanceRead = "gateway.instance:read" + InstanceAll = "gateway.instance:all" + EntryAll = "gateway.entry:all" + EntryRead = "gateway.entry:read" + RouterRead = "gateway.router:read" + RouterAll = "gateway.router:all" + FlowRead = "gateway.flow:read" + FlowAll = "gateway.flow:all" + + AgentInstanceRead = "agent.instance:read" + AgentInstanceAll = "agent.instance:all" + + IndexAll = "data.index:all" + IndexRead = "data.index:read" + AliasAll = "data.alias:all" + AliasRead = "data.alias:read" + ViewsAll = "data.view:all" + ViewsRead = "data.view:read" + DiscoverAll = "data.discover:all" + DiscoverRead = "data.discover:read" + + RuleRead = "alerting.rule:read" + RuleAll = "alerting.rule:all" + AlertRead = "alerting.alert:read" + AlertAll = "alerting.alert:all" + AlertMessageRead = "alerting.message:read" + AlertMessageAll = "alerting.message:all" + ChannelRead = "alerting.channel:read" + ChannelAll = "alerting.channel:all" + + ClusterOverviewRead = "cluster.overview:read" + ClusterOverviewAll = "cluster.overview:all" + MonitoringRead = "cluster.monitoring:read" + MonitoringAll = "cluster.monitoring:all" + ActivitiesRead = "cluster.activities:read" + ActivitiesAll = "cluster.activities:all" + AuditLogsRead = "system.audit_logs:read" + AuditLogsAll = "system.audit_logs:all" + DataMigrationRead = "data_tools.migration:read" + DataMigrationAll = "data_tools.migration:all" + DataComparisonRead = "data_tools.comparison:read" + DataComparisonAll = "data_tools.comparison:all" + DashboardRead = "insight.dashboard:read" + DashboardAll = "insight.dashboard:all" + DevtoolConsoleAll = "devtool.console:all" + DevtoolConsoleRead = "devtool.console:read" + WorkbenchAll = "workbench:all" + WorkbenchRead = "workbench:read" + + TenantCustomerRead = "tenant.customer:read" + TenantCustomerAll = "tenant.customer:all" + + SubscriptionRead = "tenant.subscription:read" + SubscriptionAll = "tenant.subscription:all" +) + +const ( + PermissionUserRead string = "user:read" + PermissionUserWrite = "user:write" + PermissionDisableBuiltinAdmin = "user:disable_builtin_admin" + PermissionRoleRead = "role:read" + PermissionRoleWrite = "role:write" + PermissionCommandRead = "command:read" + PermissionCommandWrite = "command:write" + PermissionElasticsearchClusterRead = "es.cluster:read" + PermissionElasticsearchClusterWrite = "es.cluster:write" // es cluster + PermissionElasticsearchIndexRead = "es.index:read" + PermissionElasticsearchIndexWrite = "es.index:write" // es index metadata + PermissionElasticsearchNodeRead = "es.node:read" //es node metadata + PermissionActivityRead = "activity:read" + PermissionActivityWrite = "activity:write" + PermissionAuditLogRead = "audit_log:read" + PermissionAuditLogWrite = "audit_log:write" + PermissionAlertRuleRead = "alert.rule:read" + PermissionAlertRuleWrite = "alert.rule:write" + PermissionAlertHistoryRead = "alert.history:read" + PermissionAlertHistoryWrite = "alert.history:write" + PermissionAlertMessageRead = "alert.message:read" + PermissionAlertMessageWrite = "alert.message:write" + PermissionAlertChannelRead = "alert.channel:read" + PermissionAlertChannelWrite = "alert.channel:write" + PermissionViewRead = "view:read" + PermissionViewWrite = "view:write" + PermissionLayoutRead = "layout:read" + PermissionLayoutWrite = "layout:write" + PermissionGatewayInstanceRead = "gateway.instance:read" + PermissionGatewayInstanceWrite = "gateway.instance:write" + PermissionGatewayEntryRead = "gateway.entry:read" + PermissionGatewayEntryWrite = "gateway.entry:write" + PermissionGatewayRouterRead = "gateway.router:read" + PermissionGatewayRouterWrite = "gateway.router:write" + PermissionGatewayFlowRead = "gateway.flow:read" + PermissionGatewayFlowWrite = "gateway.flow:write" + PermissionElasticsearchMetricRead = "es.metric:read" + + PermissionAgentInstanceRead = "agent.instance:read" + PermissionAgentInstanceWrite = "agent.instance:write" + PermissionCredentialRead = "credential:read" + PermissionCredentialWrite = "credential:write" + PermissionMigrationTaskRead = "task:read" + PermissionMigrationTaskWrite = "task:write" + PermissionComparisonTaskRead = "comparison_task:read" + PermissionComparisonTaskWrite = "comparison_task:write" +) + +var ( + UserReadPermission = []string{PermissionUserRead} + UserAllPermission = []string{PermissionUserRead, PermissionUserWrite, PermissionRoleRead} + + RoleReadPermission = []string{PermissionRoleRead} + RoleAllPermission = []string{PermissionRoleRead, PermissionRoleWrite} + SecurityReadPermission = []string{PermissionUserRead, PermissionRoleRead} + SecurityAllPermission = []string{PermissionUserRead, PermissionUserWrite, PermissionRoleRead, PermissionRoleWrite, PermissionDisableBuiltinAdmin} + + ClusterReadPermission = []string{PermissionElasticsearchClusterRead} + ClusterAllPermission = []string{PermissionElasticsearchClusterRead, PermissionElasticsearchClusterWrite} + + CommandReadPermission = []string{PermissionCommandRead} + CommandAllPermission = []string{PermissionCommandRead, PermissionCommandWrite} + + InstanceReadPermission = []string{PermissionGatewayInstanceRead} + InstanceAllPermission = []string{PermissionGatewayInstanceRead, PermissionGatewayInstanceWrite} + + EntryReadPermission = []string{PermissionGatewayEntryRead} + EntryAllPermission = []string{PermissionGatewayEntryRead, PermissionGatewayEntryWrite} + + RouterReadPermission = []string{PermissionGatewayRouterRead} + RouterAllPermission = []string{PermissionGatewayRouterRead, PermissionGatewayRouterWrite} + + FlowReadPermission = []string{PermissionGatewayFlowRead} + FlowAllPermission = []string{PermissionGatewayFlowRead, PermissionGatewayFlowWrite} + + IndexAllPermission = []string{"index:read"} + IndexReadPermission = []string{"index:read", "alias:write"} + AliasAllPermission = []string{"alias:read"} + AliasReadPermission = []string{"alias:read", "alias:write"} + ViewsAllPermission = []string{PermissionViewRead, PermissionViewWrite, PermissionLayoutRead, PermissionLayoutWrite} + ViewsReadPermission = []string{PermissionViewRead, PermissionLayoutRead} + DiscoverReadPermission = []string{PermissionViewRead} + DiscoverAllPermission = []string{PermissionViewRead} + + RuleReadPermission = []string{PermissionAlertRuleRead, PermissionAlertHistoryRead} + RuleAllPermission = []string{PermissionAlertRuleRead, PermissionAlertRuleWrite, PermissionAlertHistoryRead, PermissionElasticsearchClusterRead} + AlertReadPermission = []string{PermissionAlertHistoryRead} + AlertAllPermission = []string{PermissionAlertHistoryRead, PermissionAlertHistoryWrite} + AlertMessageReadPermission = []string{PermissionAlertMessageRead, PermissionAlertHistoryRead} + AlertMessageAllPermission = []string{PermissionAlertMessageRead, PermissionAlertMessageWrite, PermissionAlertHistoryRead} + ChannelReadPermission = []string{PermissionAlertChannelRead} + ChannelAllPermission = []string{PermissionAlertChannelRead, PermissionAlertChannelWrite} + + ClusterOverviewReadPermission = []string{PermissionElasticsearchClusterRead, PermissionElasticsearchIndexRead, PermissionElasticsearchNodeRead, PermissionElasticsearchMetricRead} + ClusterOverviewAllPermission = ClusterOverviewReadPermission + MonitoringReadPermission = ClusterOverviewAllPermission + + ActivitiesReadPermission = []string{PermissionActivityRead} + ActivitiesAllPermission = []string{PermissionActivityRead, PermissionActivityWrite} + + AuditLogsReadPermission = []string{PermissionAuditLogRead} + AuditLogsAllPermission = []string{PermissionAuditLogRead, PermissionAuditLogWrite} + + TenantCustomerReadPermission = []string{TenantCustomerRead} + TenantCustomerAllPermission = []string{TenantCustomerRead, TenantCustomerAll} + + SubscriptionReadPermission = []string{SubscriptionRead} + SubscriptionAllPermission = []string{SubscriptionRead, SubscriptionAll} + + AgentInstanceReadPermission = []string{PermissionAgentInstanceRead} + AgentInstanceAllPermission = []string{PermissionAgentInstanceRead, PermissionAgentInstanceWrite} + CredentialReadPermission = []string{PermissionCredentialRead} + CredentialAllPermission = []string{PermissionCredentialRead, PermissionCredentialWrite} + DataMigrationReadPermission = []string{PermissionMigrationTaskRead} + DataMigrationAllPermission = []string{PermissionMigrationTaskRead, PermissionMigrationTaskWrite} + DataComparisonReadPermission = []string{PermissionComparisonTaskRead} + DataComparisonAllPermission = []string{PermissionComparisonTaskRead, PermissionComparisonTaskWrite} + DashboardReadPermission = []string{PermissionLayoutRead} + DashboardAllPermission = []string{PermissionLayoutRead, PermissionLayoutWrite} + WorkbenchReadPermission = []string{PermissionElasticsearchClusterRead, PermissionActivityRead, PermissionAlertMessageRead, PermissionElasticsearchMetricRead} + WorkbenchAllPermission = WorkbenchReadPermission +) + +var AdminPrivilege = []string{ + SecurityAll, ClusterAll, CommandAll, + InstanceAll, EntryAll, RouterAll, FlowAll, + IndexAll, ViewsAll, DiscoverAll, + RuleAll, AlertAll, ChannelAll, + AlertMessageAll, + ClusterOverviewAll, MonitoringAll, ActivitiesAll, + AliasAll, AgentInstanceAll, CredentialAll, + DataMigrationAll, DataComparisonAll, DashboardAll, DevtoolConsoleAll, + WorkbenchAll, TenantCustomerAll, SubscriptionAll, AuditLogsAll, +} + +func init() { + + PermissionMap = map[string][]string{ + UserRead: UserReadPermission, + UserAll: UserAllPermission, + RoleRead: RoleReadPermission, + RoleAll: RoleAllPermission, + SecurityAll: SecurityAllPermission, + SecurityRead: SecurityReadPermission, + + ClusterRead: ClusterReadPermission, + ClusterAll: ClusterAllPermission, + CommandRead: CommandReadPermission, + CommandAll: CommandAllPermission, + + InstanceRead: InstanceReadPermission, + InstanceAll: InstanceAllPermission, + EntryRead: EntryReadPermission, + EntryAll: EntryAllPermission, + RouterRead: RouterReadPermission, + RouterAll: RouterAllPermission, + FlowRead: FlowReadPermission, + FlowAll: FlowAllPermission, + + IndexAll: IndexAllPermission, + IndexRead: IndexReadPermission, + AliasAll: AliasAllPermission, + AliasRead: AliasReadPermission, + ViewsAll: ViewsAllPermission, + ViewsRead: ViewsReadPermission, + DiscoverRead: DiscoverReadPermission, + DiscoverAll: DiscoverAllPermission, + + RuleRead: RuleReadPermission, + RuleAll: RuleAllPermission, + AlertRead: AlertReadPermission, + AlertAll: AlertAllPermission, + ChannelRead: ChannelReadPermission, + ChannelAll: ChannelAllPermission, + AlertMessageRead: AlertMessageReadPermission, + AlertMessageAll: AlertMessageAllPermission, + + ClusterOverviewRead: ClusterOverviewReadPermission, + ClusterOverviewAll: ClusterOverviewAllPermission, + MonitoringAll: MonitoringReadPermission, + MonitoringRead: MonitoringReadPermission, + ActivitiesAll: ActivitiesAllPermission, + ActivitiesRead: ActivitiesReadPermission, + AuditLogsAll: AuditLogsAllPermission, + AuditLogsRead: AuditLogsReadPermission, + AgentInstanceAll: AgentInstanceAllPermission, + AgentInstanceRead: AgentInstanceReadPermission, + CredentialAll: CredentialAllPermission, + CredentialRead: CredentialReadPermission, + DataMigrationRead: DataMigrationReadPermission, + DataMigrationAll: DataMigrationAllPermission, + DataComparisonRead: DataComparisonReadPermission, + DataComparisonAll: DataComparisonAllPermission, + DashboardRead: DashboardReadPermission, + DashboardAll: DashboardAllPermission, + WorkbenchAll: WorkbenchAllPermission, + WorkbenchRead: WorkbenchReadPermission, + TenantCustomerRead: TenantCustomerReadPermission, + TenantCustomerAll: TenantCustomerAllPermission, + + SubscriptionRead: SubscriptionReadPermission, + SubscriptionAll: SubscriptionAllPermission, + } + +} diff --git a/core/security/permission.go b/core/security/permission.go new file mode 100644 index 00000000..f6b8ecb9 --- /dev/null +++ b/core/security/permission.go @@ -0,0 +1,88 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "infini.sh/framework/core/api/routetree" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/util" + log "src/github.com/cihub/seelog" + "sync" +) + +var permissionsMap = map[string]interface{}{} +var permissionsLocker = sync.Mutex{} + +func RegisterPermission(typ string, permissions interface{}) { + permissionsLocker.Lock() + defer permissionsLocker.Unlock() + permissionsMap[typ] = permissions +} + +func GetPermissions(typ string) interface{} { + permissionsLocker.Lock() + defer permissionsLocker.Unlock() + return permissionsMap[typ] +} + +var RoleMap = make(map[string]Role) + +type Token struct { + JwtStr string `json:"jwt_str"` + Value string `json:"value"` + ExpireIn int64 `json:"expire_in"` +} + +var userTokenLocker = sync.RWMutex{} +var tokenMap = make(map[string]Token) + +const KVUserToken = "user_token" + +func SetUserToken(key string, token Token) { + userTokenLocker.Lock() + tokenMap[key] = token + userTokenLocker.Unlock() + _ = kv.AddValue(KVUserToken, []byte(key), util.MustToJSONBytes(token)) +} +func GetUserToken(key string) *Token { + userTokenLocker.RLock() + defer userTokenLocker.RUnlock() + if token, ok := tokenMap[key]; ok { + return &token + } + tokenBytes, err := kv.GetValue(KVUserToken, []byte(key)) + if err != nil { + log.Errorf("get user token from kv error: %v", err) + return nil + } + if tokenBytes == nil { + return nil + } + token := Token{} + util.MustFromJSONBytes(tokenBytes, &token) + return &token +} + +func DeleteUserToken(key string) { + userTokenLocker.Lock() + delete(tokenMap, key) + userTokenLocker.Unlock() + _ = kv.DeleteKey(KVUserToken, []byte(key)) +} + +var apiPermissionRouter = map[string]*routetree.Router{} +var apiPermissionLocker = sync.Mutex{} + +func RegisterAPIPermissionRouter(typ string, router *routetree.Router) { + apiPermissionLocker.Lock() + defer apiPermissionLocker.Unlock() + apiPermissionRouter[typ] = router +} + +func GetAPIPermissionRouter(typ string) *routetree.Router { + apiPermissionLocker.Lock() + defer apiPermissionLocker.Unlock() + return apiPermissionRouter[typ] +} diff --git a/core/security/role.go b/core/security/role.go new file mode 100644 index 00000000..8a8d5582 --- /dev/null +++ b/core/security/role.go @@ -0,0 +1,94 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "fmt" + "infini.sh/console/core/security/enum" + "infini.sh/framework/core/orm" + "time" +) + +type Role struct { + orm.ORMObjectBase + + Name string `json:"name" elastic_mapping:"name: { type: keyword }"` + Type string `json:"type" elastic_mapping:"type: { type: keyword }"` + Description string `json:"description" elastic_mapping:"description: { type: text }"` + Builtin bool `json:"builtin" elastic_mapping:"builtin: { type: boolean }"` + Privilege RolePrivilege `json:"privilege" elastic_mapping:"privilege: { type: object }"` +} + +type RolePrivilege struct { + Platform []string `json:"platform,omitempty" elastic_mapping:"platform: { type: keyword }"` + Elasticsearch ElasticsearchPrivilege `json:"elasticsearch,omitempty" elastic_mapping:"elasticsearch: { type: object }"` +} + +type ElasticsearchPrivilege struct { + Cluster ClusterPrivilege `json:"cluster,omitempty" elastic_mapping:"cluster: { type: object }"` + Index []IndexPrivilege `json:"index,omitempty" elastic_mapping:"index: { type: object }"` +} + +type InnerCluster struct { + ID string `json:"id" elastic_mapping:"id: { type: keyword }"` + Name string `json:"name" elastic_mapping:"name: { type: keyword }"` +} + +type ClusterPrivilege struct { + Resources []InnerCluster `json:"resources,omitempty" elastic_mapping:"resources: { type: object }"` + Permissions []string `json:"permissions,omitempty" elastic_mapping:"permissions: { type: keyword }"` +} + +type IndexPrivilege struct { + Name []string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` + Permissions []string `json:"permissions,omitempty" elastic_mapping:"permissions: { type: keyword }"` +} + +type RoleType = string + +const ( + Platform RoleType = "platform" + Elasticsearch RoleType = "elasticsearch" +) + +func IsAllowRoleType(roleType string) (err error) { + if roleType != Platform && roleType != Elasticsearch { + err = fmt.Errorf("invalid role type %s ", roleType) + return + } + return +} + +var BuiltinRoles = make(map[string]Role, 0) + +const RoleAdminName = "Administrator" + +func init() { + now := time.Now() + BuiltinRoles[RoleAdminName] = Role{ + ORMObjectBase: orm.ORMObjectBase{ + ID: RoleAdminName, + Created: &now, + }, + Name: RoleAdminName, + Type: "platform", + Privilege: RolePrivilege{ + Platform: enum.AdminPrivilege, + Elasticsearch: ElasticsearchPrivilege{ + Cluster: ClusterPrivilege{ + Resources: []InnerCluster{{"*", "*"}}, + Permissions: []string{"*"}, + }, + Index: []IndexPrivilege{ + {Name: []string{"*"}, + Permissions: []string{"*"}, + }, + }, + }, + }, + Builtin: true, + Description: "Administrator is a super role.", + } +} diff --git a/core/security/user.go b/core/security/user.go new file mode 100644 index 00000000..c8abce40 --- /dev/null +++ b/core/security/user.go @@ -0,0 +1,41 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "infini.sh/framework/core/orm" +) + +type User struct { + orm.ORMObjectBase + + AuthProvider string `json:"auth_provider" elastic_mapping:"auth_provider: { type: keyword }"` + Username string `json:"name" elastic_mapping:"name: { type: keyword }"` + Nickname string `json:"nick_name" elastic_mapping:"nick_name: { type: keyword }"` + Password string `json:"password" elastic_mapping:"password: { type: keyword }"` + Email string `json:"email" elastic_mapping:"email: { type: keyword }"` + Phone string `json:"phone" elastic_mapping:"phone: { type: keyword }"` + Tags []string `json:"tags" elastic_mapping:"mobile: { type: keyword }"` + + AvatarUrl string `json:"avatar_url" elastic_mapping:"avatar_url: { type: keyword }"` + Roles []UserRole `json:"roles" elastic_mapping:"roles: { type: object }"` + Payload interface{} `json:"-"` //used for storing additional data derived from auth provider +} + +func (user *User) GetPermissions() (roles []string, privileges []string) { + for _, v := range user.Roles { + role, ok := RoleMap[v.Name] + if ok { + roles = append(roles, v.Name) + privileges = append(privileges, role.Privilege.Platform...) + } + } + return roles, privileges +} + +type UserRole struct { + ID string `json:"id" elastic_mapping:"id: { type: keyword }"` + Name string `json:"name" elastic_mapping:"name: { type: keyword }"` +} diff --git a/core/security/validate.go b/core/security/validate.go new file mode 100644 index 00000000..c7c4cf7a --- /dev/null +++ b/core/security/validate.go @@ -0,0 +1,383 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package security + +import ( + "errors" + "fmt" + "github.com/golang-jwt/jwt" + "infini.sh/console/core/security/enum" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "net/http" + "strings" + "time" +) + +type EsRequest struct { + Doc string `json:"doc"` + Privilege string `json:"privilege"` + ClusterRequest + IndexRequest +} + +type ClusterRequest struct { + Cluster string `json:"cluster"` + Privilege []string `json:"privilege"` +} + +type IndexRequest struct { + Cluster string `json:"cluster"` + Index string `json:"index"` + Privilege []string `json:"privilege"` +} + +type ElasticsearchAPIPrivilege map[string]map[string]struct{} + +func (ep ElasticsearchAPIPrivilege) Merge(epa ElasticsearchAPIPrivilege) { + for k, permissions := range epa { + if _, ok := ep[k]; ok { + for permission := range permissions { + ep[k][permission] = struct{}{} + } + } else { + ep[k] = permissions + } + } +} + +type RolePermission struct { + Platform []string `json:"platform,omitempty"` + ElasticPrivilege struct { + Cluster ElasticsearchAPIPrivilege + Index map[string]ElasticsearchAPIPrivilege + } +} + +func NewIndexRequest(ps httprouter.Params, privilege []string) IndexRequest { + index := ps.ByName("index") + clusterId := ps.ByName("id") + return IndexRequest{ + Cluster: clusterId, + Index: index, + Privilege: privilege, + } +} + +func NewClusterRequest(ps httprouter.Params, privilege []string) ClusterRequest { + clusterId := ps.ByName("id") + return ClusterRequest{ + Cluster: clusterId, + Privilege: privilege, + } +} + +func validateApiPermission(apiPrivileges map[string]struct{}, permissions map[string]struct{}) { + if _, ok := permissions["*"]; ok { + for privilege := range apiPrivileges { + delete(apiPrivileges, privilege) + } + return + } + for permission := range permissions { + if _, ok := apiPrivileges[permission]; ok { + delete(apiPrivileges, permission) + } + } + for privilege := range apiPrivileges { + position := strings.Index(privilege, ".") + if position == -1 { + continue + } + prefix := privilege[:position] + + if _, ok := permissions[prefix+".*"]; ok { + delete(apiPrivileges, privilege) + } + } +} +func validateIndexPermission(indexName string, apiPrivileges map[string]struct{}, privilege ElasticsearchAPIPrivilege) bool { + permissions, hasAll := privilege["*"] + if hasAll { + validateApiPermission(apiPrivileges, permissions) + } + for indexPattern, v := range privilege { + if radix.Match(indexPattern, indexName) { + validateApiPermission(apiPrivileges, v) + } + } + + return len(apiPrivileges) == 0 +} + +func ValidateIndex(req IndexRequest, userRole RolePermission) (err error) { + var ( + apiPrivileges = map[string]struct{}{} + allowed bool + ) + + for _, privilege := range req.Privilege { + apiPrivileges[privilege] = struct{}{} + } + indexPermissions, hasAllCluster := userRole.ElasticPrivilege.Index["*"] + if hasAllCluster { + allowed = validateIndexPermission(req.Index, apiPrivileges, indexPermissions) + if allowed { + return nil + } + } + if _, ok := userRole.ElasticPrivilege.Index[req.Cluster]; !ok { + return fmt.Errorf("no permission of cluster [%s]", req.Cluster) + } + allowed = validateIndexPermission(req.Index, apiPrivileges, userRole.ElasticPrivilege.Index[req.Cluster]) + if allowed { + return nil + } + var apiPermission string + for k := range apiPrivileges { + apiPermission = k + } + + return fmt.Errorf("no index api permission: %s", apiPermission) + +} + +func ValidateCluster(req ClusterRequest, userRole RolePermission) (err error) { + var ( + apiPrivileges = map[string]struct{}{} + ) + + for _, privilege := range req.Privilege { + apiPrivileges[privilege] = struct{}{} + } + + clusterPermissions, hasAllCluster := userRole.ElasticPrivilege.Cluster["*"] + if hasAllCluster { + validateApiPermission(apiPrivileges, clusterPermissions) + if len(apiPrivileges) == 0 { + return nil + } + } + if _, ok := userRole.ElasticPrivilege.Cluster[req.Cluster]; !ok && !hasAllCluster { + return fmt.Errorf("no permission of cluster [%s]", req.Cluster) + } + validateApiPermission(apiPrivileges, userRole.ElasticPrivilege.Cluster[req.Cluster]) + if len(apiPrivileges) == 0 { + return nil + } + var apiPermission string + for k := range apiPrivileges { + apiPermission = k + } + + return fmt.Errorf("no cluster api permission: %s", apiPermission) +} + +func CombineUserRoles(roleNames []string) RolePermission { + newRole := RolePermission{} + clusterPrivilege := ElasticsearchAPIPrivilege{} + indexPrivilege := map[string]ElasticsearchAPIPrivilege{} + platformM := map[string]struct{}{} + for _, val := range roleNames { + role := RoleMap[val] + for _, pm := range role.Privilege.Platform { + if _, ok := platformM[pm]; !ok { + newRole.Platform = append(newRole.Platform, pm) + platformM[pm] = struct{}{} + } + + } + + singleIndexPrivileges := ElasticsearchAPIPrivilege{} + for _, ip := range role.Privilege.Elasticsearch.Index { + for _, indexName := range ip.Name { + if _, ok := singleIndexPrivileges[indexName]; !ok { + singleIndexPrivileges[indexName] = map[string]struct{}{} + } + for _, permission := range ip.Permissions { + singleIndexPrivileges[indexName][permission] = struct{}{} + } + } + } + + for _, cp := range role.Privilege.Elasticsearch.Cluster.Resources { + if _, ok := indexPrivilege[cp.ID]; ok { + indexPrivilege[cp.ID].Merge(singleIndexPrivileges) + } else { + indexPrivilege[cp.ID] = singleIndexPrivileges + } + var ( + privileges map[string]struct{} + ok bool + ) + if privileges, ok = clusterPrivilege[cp.ID]; !ok { + privileges = map[string]struct{}{} + } + for _, permission := range role.Privilege.Elasticsearch.Cluster.Permissions { + privileges[permission] = struct{}{} + } + clusterPrivilege[cp.ID] = privileges + } + + } + newRole.ElasticPrivilege.Cluster = clusterPrivilege + newRole.ElasticPrivilege.Index = indexPrivilege + return newRole +} + +func GetRoleClusterMap(roles []string) map[string][]string { + userClusterMap := make(map[string][]string, 0) + for _, roleName := range roles { + role, ok := RoleMap[roleName] + if ok { + for _, ic := range role.Privilege.Elasticsearch.Cluster.Resources { + userClusterMap[ic.ID] = append(userClusterMap[ic.ID], role.Privilege.Elasticsearch.Cluster.Permissions...) + } + } + } + return userClusterMap +} + +// GetRoleCluster get cluster id by given role names +// return true when has all cluster privilege, otherwise return cluster id list +func GetRoleCluster(roles []string) (bool, []string) { + userClusterMap := GetRoleClusterMap(roles) + if _, ok := userClusterMap["*"]; ok { + return true, nil + } + realCluster := make([]string, 0, len(userClusterMap)) + for k, _ := range userClusterMap { + realCluster = append(realCluster, k) + } + return false, realCluster +} + +// GetCurrentUserCluster get cluster id by current login user +// return true when has all cluster privilege, otherwise return cluster id list +func GetCurrentUserCluster(req *http.Request) (bool, []string) { + ctxVal := req.Context().Value("user") + if userClaims, ok := ctxVal.(*UserClaims); ok { + return GetRoleCluster(userClaims.Roles) + } else { + panic("user context value not found") + } +} + +func GetRoleIndex(roles []string, clusterID string) (bool, []string) { + var realIndex []string + for _, roleName := range roles { + role, ok := RoleMap[roleName] + if ok { + for _, ic := range role.Privilege.Elasticsearch.Cluster.Resources { + if ic.ID != "*" && ic.ID != clusterID { + continue + } + for _, ip := range role.Privilege.Elasticsearch.Index { + if util.StringInArray(ip.Name, "*") { + return true, nil + } + realIndex = append(realIndex, ip.Name...) + } + } + } + } + return false, realIndex +} + +func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { + + if authorizationHeader == "" { + err = errors.New("authorization header is empty") + return + } + fields := strings.Fields(authorizationHeader) + if fields[0] != "Bearer" || len(fields) != 2 { + err = errors.New("authorization header is invalid") + return + } + tokenString := fields[1] + + token, err := jwt.ParseWithClaims(tokenString, &UserClaims{}, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(Secret), nil + }) + if err != nil { + return + } + clams, ok := token.Claims.(*UserClaims) + + if clams.UserId == "" { + err = errors.New("user id is empty") + return + } + //fmt.Println("user token", clams.UserId, TokenMap[clams.UserId]) + tokenVal := GetUserToken(clams.UserId) + if tokenVal == nil { + err = errors.New("token is invalid") + return + } + if tokenVal.ExpireIn < time.Now().Unix() { + err = errors.New("token is expire in") + DeleteUserToken(clams.UserId) + return + } + if ok && token.Valid { + return clams, nil + } + return + +} + +func ValidatePermission(claims *UserClaims, permissions []string) (err error) { + + user := claims.ShortUser + + if user.UserId == "" { + err = errors.New("user id is empty") + return + } + if user.Roles == nil { + err = errors.New("api permission is empty") + return + } + + // 权限校验 + userPermissions := make([]string, 0) + for _, role := range user.Roles { + if _, ok := RoleMap[role]; ok { + for _, v := range RoleMap[role].Privilege.Platform { + userPermissions = append(userPermissions, v) + } + } + } + userPermissionMap := make(map[string]struct{}) + for _, val := range userPermissions { + for _, v := range enum.PermissionMap[val] { + userPermissionMap[v] = struct{}{} + } + + } + + for _, v := range permissions { + if _, ok := userPermissionMap[v]; !ok { + err = errors.New("permission denied") + return + } + } + return nil + +} + +func SearchAPIPermission(typ string, method, path string) (permission string, params map[string]string, matched bool) { + method = strings.ToLower(method) + router := GetAPIPermissionRouter(typ) + if router == nil { + panic(fmt.Errorf("can not found api permission router of %s", typ)) + } + return router.Search(method, path) +} diff --git a/core/user.go b/core/user.go new file mode 100644 index 00000000..3a38751d --- /dev/null +++ b/core/user.go @@ -0,0 +1,34 @@ +package core + +import ( + "github.com/emirpasic/gods/sets/hashset" +) + +const ( + ROLE_GUEST string = "guest" + ROLE_ADMIN string = "admin" +) + +const ( + //GUEST + PERMISSION_SNAPSHOT_VIEW string = "view_snapshot" + + //ADMIN + PERMISSION_ADMIN_MINIMAL string = "admin_minimal" +) + +func GetPermissionsByRole(role string) (*hashset.Set, error) { + initRolesMap() + return rolesMap[role], nil +} + +var rolesMap = map[string]*hashset.Set{} + +func initRolesMap() { + if rolesMap != nil { + return + } + set := hashset.New() + set.Add(PERMISSION_SNAPSHOT_VIEW) + rolesMap[ROLE_GUEST] = set +} diff --git a/db/README.md b/db/README.md deleted file mode 100644 index 3bb285cb..00000000 --- a/db/README.md +++ /dev/null @@ -1,2 +0,0 @@ -## 数据库初始化脚本 - diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 9ad9a2d4..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM scratch - -COPY ./console /console -COPY ./config/ /config/ -COPY ./console.yml /console.yml - -CMD ["/console"] diff --git a/docker/centos-Dockerfile b/docker/centos-Dockerfile deleted file mode 100644 index eb1e7b67..00000000 --- a/docker/centos-Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM amd64/centos:7.9.2009 - -COPY ./console /console -COPY ./config/ /config/ -COPY ./console.yml /console.yml - -CMD ["/console"] diff --git a/docker/deploy.key b/docker/deploy.key deleted file mode 100644 index e684174d..00000000 --- a/docker/deploy.key +++ /dev/null @@ -1,38 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn -NhAAAAAwEAAQAAAYEAzQ4wJFpd+kt5hVntfyVvhUnWhUPvfzpQf0NDyn7TnYxnG0C6uEVs -DdKEoMabNwz/zgK0Hlh+9qTBZ/HdddCKH18dDjIrjob+3YKi107yb4nbAJRKueJ9sK+ZWt -zv2ZjaYav9S9vGx+NWbC0ODqsTywg3VgRoYfai/Tz6iH5FIrSYp4ds15m+bEdtpHs3G2x3 -Of8Q937lJb7W14rg4RZuTMg7FjirCEK8dk3pzLt7K0I1fiDpC3VxluX6p27apjcpx6IGo9 -OMJzTI2SgO+RHrx29gNMKiq0oz1eE4OBDEe9dhNkRV0Hh6BjJ39K8VbwjhvdFCHTkAm12o -bbFxB/ZYc3IKEK7OMeRqdBlTx5yq2H4W5xhLy2qmX7itWPvBGBzhuNclwXvOjYwm/HQjlu -OIgUKQjUBWm7I0wJdM1n+kh/+kLMmFJC9fncz2nJAU6J5WMD1nx7CQh1AZfdEKs/AFWG70 -FUKUlvsZHmuNrBuggiXKq4m17rKcxLGphYHL6mSJAAAFiO+1v/rvtb/6AAAAB3NzaC1yc2 -EAAAGBAM0OMCRaXfpLeYVZ7X8lb4VJ1oVD7386UH9DQ8p+052MZxtAurhFbA3ShKDGmzcM -/84CtB5YfvakwWfx3XXQih9fHQ4yK46G/t2CotdO8m+J2wCUSrnifbCvmVrc79mY2mGr/U -vbxsfjVmwtDg6rE8sIN1YEaGH2ov08+oh+RSK0mKeHbNeZvmxHbaR7Nxtsdzn/EPd+5SW+ -1teK4OEWbkzIOxY4qwhCvHZN6cy7eytCNX4g6Qt1cZbl+qdu2qY3KceiBqPTjCc0yNkoDv -kR68dvYDTCoqtKM9XhODgQxHvXYTZEVdB4egYyd/SvFW8I4b3RQh05AJtdqG2xcQf2WHNy -ChCuzjHkanQZU8ecqth+FucYS8tqpl+4rVj7wRgc4bjXJcF7zo2MJvx0I5bjiIFCkI1AVp -uyNMCXTNZ/pIf/pCzJhSQvX53M9pyQFOieVjA9Z8ewkIdQGX3RCrPwBVhu9BVClJb7GR5r -jawboIIlyquJte6ynMSxqYWBy+pkiQAAAAMBAAEAAAGABfKsaNGKOlFoI/sYzYBbfMVIiL -MQxmL9pMNhuwT0xHQnJX46LFAvMzNxD2zTYcRpwyMG8H5mqGbdCVPVta4n44MRx7Ci3M6D -pA8/A/nRRHT+OkUS6dNtC+v8Ccuw1WH+q6ief03PtUqd3iNsbfZ+a3xAhqk4EedikO/s4H -qxLLGKYAmomZRnFqL3xjagwZXi23bPmi4/HVosxzHLFhxddLK2LA3WwDWXW+MkrgCeMQIJ -pS/1MpTkh5kCLUsk4n9lFI4P3gB+IFGNtGnBmIhwz/2rjXc5OKD5WlXBdGGQ2mWK49NMlJ -LGBSDrAeFErY3Ja8NnOZcXG9o76V6qKQIib8wVDJ0klstDPxBZSLxs2OkCZpKya8ARA1Ci -T48Lbsc/DCdsmajC3zpNuI3Li7ofbzvgCSf7A5rxOghztPY9fD9vdSdPRWoBqIsUfizgO1 -mdXzzsF/iBqwlbSCVrzeKleZAAsCUU/0XLUnaBuSKT2LhYvDu3aIC8vf5tN9uAAIWBAAAA -wHsJpPmlt4gKNmWEm3NyARj8oxJdpcaV4M95ofmb2bWzbvW6Vqce+eswe2ctieISN+7w+2 -JQB5saNgHhV3D4V3Be3OWkB1KwOjDJmwN5IcmTT+Kmtn7AJ+0yukNMbnNXEbT1oO3KUmfv -wI294u0jlxgSgsRRqYz/dP0UxPUY/z0g40E8pmV4FK7ogxzHKVcLTFTMXqNqaJQCRp4cg+ -JgigXgoFzRJLx3CrcROxLex4uocbnSXNYhCCURDVT5S2krWAAAAMEA7CaEAEX+zt6kIINs -kIH6F8HJJ7FcaVKdzmvt/fK5njm/b5NXgo5Iaar9u0OFEpmkI0v5BZEK6IWZN96jLL8iBx -kqkjbE2LuqliXcB+61zVCBi/RcqYDTYOmhyJcG/NcE1e+IAIhdMtNtpr+Dcd3FsGW6GltN -Ul5U6AGcvacT/lJw0kYqRsJ8La0es9Oxsks9DsKTigCVL+rCv+ZJ63mTqPCtsYCagfUzJA -AkgaSCiHNwgvsM2H7x3T3s9KEH8EGRAAAAwQDeSpHY94RZDY4rUQFxlHno+xWeZvbTHFoy -IfXF5drt/eEnfaGJY7eeBNVJI5PAbcNAuN050ZyxLov221nz9Fu8IlqeoNAfrUFfJnWVKg -ppDz3hHq7WKlxwHEJY3Pwd3/G0ppsMlaTMWyNWCkJ7QNuL3zmxgxx2/Dq/tvxDI2DvXCve -HPOdBIM2Y05he0n/zjhko3Qal+zb52Ie6qAEmQE2GEyQf27KLUZ/ww2kKa/HTjvqR9/dwd -eDxswDpr5Rd3kAAAARcm9vdEA2YTliOThkZTA0YzABAg== ------END OPENSSH PRIVATE KEY----- diff --git a/docker/docker-compose-mysql.dev.yml b/docker/docker-compose-mysql.dev.yml deleted file mode 100644 index 0dd6be98..00000000 --- a/docker/docker-compose-mysql.dev.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '3' - -services: - infini-search-center-db: - image: mariadb:10.1.19 - # volumes: - # - ../data/db_data:/var/lib/mysql - restart: always - container_name: "infini-search-center-db" - environment: - MYSQL_ROOT_PASSWORD: admin - ports: - - "3306:3306" diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml deleted file mode 100644 index 075f1d11..00000000 --- a/docker/docker-compose.dev.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: "3.5" - -services: - infini-search-center-api-dev: - image: docker.infini.ltd:64443/golang-dev:latest - ports: - - 9010:9000 - container_name: "infini-search-center-dev" - volumes: - - ../:/go/src/infini.sh/console - - ./entrypoint.sh:/entrypoint.sh - -volumes: - dist: \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index 9f494bb1..00000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cd /go/src/infini.sh/ - -echo "INFINI GOLANG ENV READY TO ROCK!" - -cd search-center -make build - -cd /go/src/infini.sh/console && ./bin/search-center \ No newline at end of file diff --git a/docker/ssh_config b/docker/ssh_config deleted file mode 100644 index 2d47a8b7..00000000 --- a/docker/ssh_config +++ /dev/null @@ -1,2 +0,0 @@ -Host * - StrictHostKeyChecking no \ No newline at end of file diff --git a/main.go b/main.go index 1508ee64..162eb804 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "infini.sh/console/plugin/audit_log" "infini.sh/framework/core/api" model2 "infini.sh/framework/core/model" + elastic2 "infini.sh/framework/modules/elastic" _ "time/tzdata" log "github.com/cihub/seelog" @@ -16,29 +17,27 @@ import ( "infini.sh/console/model" "infini.sh/console/model/alerting" "infini.sh/console/model/insight" + "infini.sh/console/modules/security" _ "infini.sh/console/plugin" + _ "infini.sh/console/plugin/managed" setup1 "infini.sh/console/plugin/setup" alerting2 "infini.sh/console/service/alerting" "infini.sh/framework" "infini.sh/framework/core/elastic" "infini.sh/framework/core/env" "infini.sh/framework/core/global" - _ "infini.sh/framework/core/log" "infini.sh/framework/core/module" "infini.sh/framework/core/orm" task1 "infini.sh/framework/core/task" _ "infini.sh/framework/modules/api" - elastic2 "infini.sh/framework/modules/elastic" "infini.sh/framework/modules/metrics" "infini.sh/framework/modules/pipeline" queue2 "infini.sh/framework/modules/queue/disk_queue" "infini.sh/framework/modules/redis" - "infini.sh/framework/modules/security" "infini.sh/framework/modules/stats" "infini.sh/framework/modules/task" "infini.sh/framework/modules/web" _ "infini.sh/framework/plugins" - _ "infini.sh/framework/plugins/managed" ) var appConfig *config.AppConfig @@ -120,7 +119,6 @@ func main() { module.Start() var initFunc = func() { - elastic2.InitTemplate(false) //orm.RegisterSchema(model.Dict{}, "dict") diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index 40786123..5e1bb3bf 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" log "github.com/cihub/seelog" + "infini.sh/console/plugin/managed/server" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" @@ -18,7 +19,6 @@ import ( "infini.sh/framework/modules/elastic/adapter" "infini.sh/framework/modules/elastic/common" "infini.sh/framework/modules/elastic/metadata" - "infini.sh/framework/plugins/managed/server" "net/http" "runtime" "sync/atomic" diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index b3b19ce0..ad670545 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -5,13 +5,14 @@ package api import ( + "infini.sh/console/core" + "infini.sh/console/core/security/enum" + "infini.sh/console/plugin/managed/server" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" - "infini.sh/framework/plugins/managed/server" ) type APIHandler struct { - api.Handler + core.Handler } func Init() { diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index 3e2cdab5..b097e397 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" log "github.com/cihub/seelog" + "infini.sh/console/plugin/managed/common" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" "infini.sh/framework/core/kv" @@ -16,7 +17,6 @@ import ( "infini.sh/framework/core/util" common2 "infini.sh/framework/modules/elastic/common" metadata2 "infini.sh/framework/modules/elastic/metadata" - "infini.sh/framework/plugins/managed/common" "time" ) diff --git a/modules/agent/common/config.go b/modules/agent/common/config.go index bc605f8f..a1da1b87 100644 --- a/modules/agent/common/config.go +++ b/modules/agent/common/config.go @@ -7,11 +7,10 @@ package common import ( log "github.com/cihub/seelog" "infini.sh/console/modules/agent/model" + "infini.sh/console/plugin/managed/common" "infini.sh/framework/core/env" - "infini.sh/framework/plugins/managed/common" ) - func GetAgentConfig() *model.AgentConfig { agentCfg := &model.AgentConfig{ Enabled: true, @@ -19,7 +18,7 @@ func GetAgentConfig() *model.AgentConfig { DownloadURL: "https://release.infinilabs.com/agent/stable", }, } - _, err := env.ParseConfig("agent", agentCfg ) + _, err := env.ParseConfig("agent", agentCfg) if err != nil { log.Errorf("agent config not found: %v", err) } diff --git a/modules/elastic/api/activity.go b/modules/elastic/api/activity.go new file mode 100644 index 00000000..29400b4e --- /dev/null +++ b/modules/elastic/api/activity.go @@ -0,0 +1,190 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "strings" +) + +func (h *APIHandler) HandleSearchActivityAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody:=util.MapStr{} + reqBody := struct{ + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + StartTime interface{} `json:"start_time"` + EndTime interface{} `json:"end_time"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) + aggs["term_cluster_id"] = util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.cluster_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "term_cluster_name": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.cluster_name", + "size": 1, + }, + }, + }, + } + filter := elastic.BuildSearchTermFilter(reqBody.Filter) + if reqBody.StartTime != "" { + filter = append(filter, util.MapStr{ + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": reqBody.StartTime, + "lte": reqBody.EndTime, + }, + }, + }) + } + + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "metadata.labels.cluster_id") + if !hasAllPrivilege && clusterFilter == nil { + h.WriteJSON(w, elastic.SearchResponse{ + + }, http.StatusOK) + return + } + if !hasAllPrivilege && clusterFilter != nil { + filter = append(filter, clusterFilter) + } + + hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req) + if !hasAllPrivilege && len(indexPrivilege) == 0 { + h.WriteJSON(w, elastic.SearchResponse{ + + }, http.StatusOK) + return + } + if !hasAllPrivilege { + indexShould := make([]interface{}, 0, len(indexPrivilege)) + for clusterID, indices := range indexPrivilege { + var ( + wildcardIndices []string + normalIndices []string + ) + for _, index := range indices { + if strings.Contains(index,"*") { + wildcardIndices = append(wildcardIndices, index) + continue + } + normalIndices = append(normalIndices, index) + } + subShould := []util.MapStr{} + if len(wildcardIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "query_string": util.MapStr{ + "query": strings.Join(wildcardIndices, " "), + "fields": []string{"metadata.labels.index_name"}, + "default_operator": "OR", + }, + }) + } + if len(normalIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "terms": util.MapStr{ + "metadata.labels.index_name": normalIndices, + }, + }) + } + indexShould = append(indexShould, util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "wildcard": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": subShould, + }, + }, + }, + }, + }) + } + indexFilter := util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": indexShould, + }, + } + filter = append(filter, indexFilter) + } + + var should = []util.MapStr{} + if reqBody.Keyword != "" { + should = []util.MapStr{ + { + "query_string": util.MapStr{ + "default_field": "*", + "query": reqBody.Keyword, + }, + }, + } + } + var boolQuery = util.MapStr{ + "filter": filter, + } + if len(should) >0 { + boolQuery["should"] = should + boolQuery["minimum_should_match"] = 1 + } + query := util.MapStr{ + "aggs": aggs, + "size": reqBody.Size, + "from": reqBody.From, + "_source": []string{"changelog", "id", "metadata", "timestamp"}, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": boolQuery, + }, + } + if len(reqBody.Sort) == 0 { + reqBody.Sort = []string{"timestamp", "desc"} + } + + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetWildcardIndexName(event.Activity{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + w.Write(response.RawResult.Body) +} \ No newline at end of file diff --git a/modules/elastic/api/alias.go b/modules/elastic/api/alias.go new file mode 100644 index 00000000..d605cbe4 --- /dev/null +++ b/modules/elastic/api/alias.go @@ -0,0 +1,85 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "github.com/segmentio/encoding/json" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" + "net/http" +) + +func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists{ + errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(errStr) + h.WriteError(w, errStr, http.StatusInternalServerError) + return + } + + var aliasReq = &elastic.AliasRequest{} + + err = h.DecodeJSON(req, aliasReq) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + esVersion := elastic.GetMetadata(targetClusterID).Config.Version + if r, _ := util.VersionCompare(esVersion, "6.4"); r == -1 { + for i := range aliasReq.Actions { + for k, v := range aliasReq.Actions[i] { + if v != nil && v["is_write_index"] != nil { + delete(aliasReq.Actions[i][k], "is_write_index") + log.Warnf("elasticsearch aliases api of version [%s] not supports parameter is_write_index", esVersion) + } + } + } + } + + bodyBytes, _ := json.Marshal(aliasReq) + + err = client.Alias(bodyBytes) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteAckOKJSON(w) +} + +func (h *APIHandler) HandleGetAliasAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + targetClusterID := ps.ByName("id") + exists, client, err := h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + h.WriteJSON(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists { + errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID) + log.Error(errStr) + h.WriteError(w, errStr, http.StatusInternalServerError) + return + } + res, err := client.GetAliasesDetail() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, res, http.StatusOK) +} \ No newline at end of file diff --git a/modules/elastic/api/cluster_overview.go b/modules/elastic/api/cluster_overview.go new file mode 100644 index 00000000..0eff0309 --- /dev/null +++ b/modules/elastic/api/cluster_overview.go @@ -0,0 +1,1294 @@ +package api + +import ( + "fmt" + "infini.sh/framework/modules/elastic/adapter" + "net/http" + "strings" + "time" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" +) + +func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var clusterIDs = []string{} + h.DecodeJSON(req, &clusterIDs) + + if len(clusterIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + cids := make([]interface{}, 0, len(clusterIDs)) + for _, clusterID := range clusterIDs { + cids = append(cids, clusterID) + } + healthMap := map[string]interface{}{} + + //fetch extra cluster status + q1 := orm.Query{WildcardIndex: true} + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "cluster_stats"), + orm.In("metadata.labels.cluster_id", cids), + ) + q1.Collapse("metadata.labels.cluster_id") + q1.AddSort("timestamp", orm.DESC) + q1.Size = len(clusterIDs) + 1 + + err, results := orm.Search(&event.Event{}, &q1) + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + clusterID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "cluster_id"}, result) + if ok { + source := map[string]interface{}{} + indicesCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "count"}, result) + if ok { + source["number_of_indices"] = indicesCount + } + + docsCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "docs", "count"}, result) + if ok { + source["number_of_documents"] = docsCount + } + + docsDeletedCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "docs", "deleted"}, result) + if ok { + source["number_of_deleted_documents"] = docsDeletedCount + } + fs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "fs"}, result) + if ok { + source["fs"] = fs + } + jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "jvm", "mem"}, result) + if ok { + source["jvm"] = jvm + } + nodeCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "count", "total"}, result) + if ok { + source["number_of_nodes"] = nodeCount + } + shardCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "shards", "total"}, result) + if ok { + source["number_of_shards"] = shardCount + } + source["timestamp"] = result["timestamp"] + + healthMap[util.ToString(clusterID)] = source + } + } + } + + var ( + // cluster_id => cluster_uuid + clustersM = map[string]string{} + clusterUUIDs []string + ) + for _, cid := range clusterIDs { + clusterUUID, err := adapter.GetClusterUUID(cid) + if err != nil { + log.Error(err) + continue + } + clusterUUIDs = append(clusterUUIDs, clusterUUID) + clustersM[cid] = clusterUUID + } + query := util.MapStr{ + "size": 0, + } + + var top = len(clusterIDs) + 1 + + bucketSize := GetMinBucketSize() + if bucketSize < 20 { + bucketSize = 20 + } + var metricLen = 15 + if bucketSize <= 60 { + metricLen += 2 + } + var bucketSizeStr = fmt.Sprintf("%vs", bucketSize) + indexMetricItems := []GroupMetricItem{} + metricItem := newMetricItem("cluster_indexing", 2, "cluster") + metricItem.OnlyPrimary = true + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "cluster_indexing", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: metricItem, + FormatType: "num", + Units: "doc/s", + }) + + metricItem = newMetricItem("cluster_search", 2, "cluster") + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "cluster_search", + Field: "payload.elasticsearch.node_stats.indices.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: metricItem, + FormatType: "num", + Units: "query/s", + }) + + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + panic(err) + } + + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.cluster_uuid": clusterUUIDs, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": fmt.Sprintf("now-%ds", metricLen * bucketSize), + }, + }, + }, + }, + }, + } + aggs:=map[string]interface{}{} + sumAggs := util.MapStr{} + + for _,metricItem:=range indexMetricItems { + leafAgg := util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + var sumBucketPath = "term_node>"+ metricItem.ID + aggs[metricItem.ID] = leafAgg + + sumAggs[metricItem.ID] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath, + }, + } + if metricItem.IsDerivative{ + sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + } + } + sumAggs["term_node"]= util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 1000, + }, + "aggs": aggs, + } + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.cluster_uuid", + "size": top, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":sumAggs, + }, + }, + }, + } + indexMetrics := h.getMetrics(query, indexMetricItems, bucketSize) + indexingMetricData := util.MapStr{} + for _, line := range indexMetrics["cluster_indexing"].Lines { + // remove first metric dot + data := line.Data + if v, ok := data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { + // remove first metric dot + temp := v[1:] + // // remove first last dot + if len(temp) > 0 { + temp = temp[0: len(temp)-1] + } + data = temp + } + indexingMetricData[line.Metric.Label] = data + } + searchMetricData := util.MapStr{} + for _, line := range indexMetrics["cluster_search"].Lines { + data := line.Data + if v, ok := data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { + // remove first metric dot + temp := v[1:] + // // remove first last dot + if len(temp) > 0 { + temp = temp[0: len(temp)-1] + } + data = temp + } + searchMetricData[line.Metric.Label] = data + } + + searchR1, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + panic(err) + } + + + //fetch recent cluster health status + bucketItem := common.NewBucketItem( + common.TermsBucket, util.MapStr{ + "field": "metadata.labels.cluster_id", + "size": top, + }) + + bucketSizeStr = "1d" + histgram := common.NewBucketItem( + common.DateRangeBucket, util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + }, { + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }) + + termBucket := common.NewBucketItem(common.TermsBucket, util.MapStr{ + "field": "payload.elasticsearch.cluster_health.status", + "size": top, + "missing": "", + }) + histgram.AddNestBucket(termBucket) + + bucketItem.AddNestBucket(histgram) + + aggs = ConvertBucketItemsToAggQuery([]*common.BucketItem{bucketItem}, nil) + query = util.MapStr{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.cluster_id": clusterIDs, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-15d", + "lte": "now", + }, + }, + }, + }, + }, + } + query["size"] = 0 + + util.MergeFields(query, aggs, true) + + searchR1, err = elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + panic(err) + } + searchResponse := SearchResponse{} + util.FromJSONBytes(searchR1.RawResult.Body, &searchResponse) + m3 := ParseAggregationBucketResult(bucketSize, searchResponse.Aggregations, bucketItem.Key, histgram.Key, termBucket.Key, nil) + + infos := util.MapStr{} + for _, clusterID := range clusterIDs { + result := util.MapStr{} + + //TODO update last active timestamp + //source := hit.Source + //source["project"]=util.MapStr{ + // "id":"12312312", + // "name":"统一日志云平台v1.0", + //} + //source["location"]=util.MapStr{ + // "provider" : "阿里云", + // "region" : "北京", + // "dc" : "昌平机房", + // "rack" : "rack-1", + //} + //source["owner"]=[]util.MapStr{util.MapStr{ + // "department" : "运维部", + // "name":"廖石阳", + // "id":"123123123", + //}} + + //result["metadata"] = source + result["summary"] = healthMap[clusterID] + result["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Cluster Status", + "units": "day", + }, + "data": getClusterMetrics(clusterID, m3), + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": indexingMetricData[clustersM[clusterID]], + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": searchMetricData[clustersM[clusterID]], + }, + } + infos[clusterID] = result + } + + h.WriteJSON(w, infos, 200) +} + +func getClusterMetrics(id string, data MetricData) [][]interface{} { + return data[id] +} + +func (h *APIHandler) GetClusterInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + q := &orm.Query{WildcardIndex: true, Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "cluster_health"), + orm.Eq("metadata.labels.cluster_id", id), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + var healthInfo interface{} = util.MapStr{} + if len(result.Result) > 0 { + if rowM, ok := result.Result[0].(map[string]interface{}); ok { + healthInfo, _ = util.MapStr(rowM).GetValue("payload.elasticsearch.cluster_health") + } + } + + h.WriteJSON(w, healthInfo, 200) +} + +func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + resBody := map[string]interface{}{} + id := ps.ByName("id") + q := &orm.Query{Size: 1000} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + ) + + err, result := orm.Search(elastic.NodeConfig{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + clusterUUID, err := adapter.GetClusterUUID(id) + query := util.MapStr{ + "size": 1000, + "collapse": util.MapStr{ + "field": "metadata.labels.node_id", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + }, + }, + } + q = &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + err, searchResult := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + nodeInfos := map[string]util.MapStr{} + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, hitM) + var totalShards float64 + if v, ok := shardInfo.(map[string]interface{}); ok { + shardCount := v["shard_count"] + replicasCount := v["replicas_count"] + if v1, ok := shardCount.(float64); ok { + totalShards += v1 + } + if v1, ok := replicasCount.(float64); ok { + totalShards += v1 + } + } + uptime, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, hitM) + cpu, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "percent"}, hitM) + load, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "load_average", "1m"}, hitM) + heapUsage, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem", "heap_used_percent"}, hitM) + availDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "available_in_bytes"}, hitM) + totalDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "total_in_bytes"}, hitM) + nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) + var usedDisk string + if v, ok := availDisk.(float64); ok { + availDisk = util.ByteSize(uint64(v)) + if v1, ok := totalDisk.(float64); ok { + usedDisk = util.ByteSize(uint64(v1 - v)) + } + } + + + if v, ok := nodeID.(string); ok { + nodeInfos[v] = util.MapStr{ + "timestamp": hitM["timestamp"], + "shards": totalShards, + "cpu": cpu, + "load_1m": load, + "heap.percent": heapUsage, + "disk.avail": availDisk, + "disk.used": usedDisk, + "uptime": uptime, + } + + } + } + } + nodes := []interface{}{} + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nodeId, _ := util.GetMapValueByKeys([]string{"metadata", "node_id"}, hitM) + nodeName, _ := util.GetMapValueByKeys([]string{"metadata", "node_name"}, hitM) + status, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "status"}, hitM) + ip, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "ip"}, hitM) + transportAddress, _ := util.GetMapValueByKeys([]string{"payload", "node_state", "transport_address"}, hitM) + var port string + if v, ok := transportAddress.(string); ok { + parts := strings.Split(v, ":") + if len(parts) > 1 { + port = parts[1] + } + } + + if v, ok := nodeId.(string); ok { + ninfo := util.MapStr{ + "id": v, + "name": nodeName, + "ip": ip, + "port": port, + "status": status, + } + if nodeInfos[v] != nil { + if nodeInfos[v]["timestamp"] != nil { + if ts, ok := nodeInfos[v]["timestamp"].(string); ok { + tt, _ := time.Parse(time.RFC3339, ts) + if time.Now().Sub(tt).Seconds() > 30 { + ninfo["status"] = "unavailable" + } + } + } + util.MergeFields(ninfo, nodeInfos[v], true) + } else { + ninfo["timestamp"] = hitM["timestamp"] + } + nodes = append(nodes, ninfo) + } + } + } + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h *APIHandler) GetRealtimeClusterNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + meta := elastic.GetMetadata(id) + if meta == nil || !meta.IsAvailable() { + log.Debugf("cluster [%s] is not available", id) + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + esClient := elastic.GetClient(id) + if esClient == nil { + h.WriteJSON(w, util.MapStr{ + "error": "cluster not found", + }, http.StatusNotFound) + return + } + catNodesInfo, err := esClient.CatNodes("id,name,ip,port,master,heap.percent,disk.avail,disk.used,cpu,load_1m,uptime") + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + return + } + catShardsInfo, err := esClient.CatShards() + if err != nil { + log.Error(err) + } + shardCounts := map[string]int{} + nodeM := map[string]string{} + for _, shardInfo := range catShardsInfo { + nodeM[shardInfo.NodeName] = shardInfo.NodeID + if c, ok := shardCounts[shardInfo.NodeName]; ok { + shardCounts[shardInfo.NodeName] = c + 1 + } else { + shardCounts[shardInfo.NodeName] = 1 + } + } + qps, err := h.getNodeQPS(id, 20) + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + return + } + + nodeInfos := []RealtimeNodeInfo{} + for _, nodeInfo := range catNodesInfo { + if len(nodeInfo.Id) == 4 { //node short id, use nodeId from shard info isnstead + nodeInfo.Id = nodeM[nodeInfo.Name] + } + if c, ok := shardCounts[nodeInfo.Name]; ok { + nodeInfo.Shards = c + } + info := RealtimeNodeInfo{ + CatNodeResponse: CatNodeResponse(nodeInfo), + } + if _, ok := qps[nodeInfo.Id]; ok { + info.IndexQPS = qps[nodeInfo.Id]["index"] + info.QueryQPS = qps[nodeInfo.Id]["query"] + info.IndexBytesQPS = qps[nodeInfo.Id]["index_bytes"] + } + nodeInfos = append(nodeInfos, info) + } + h.WriteJSON(w, nodeInfos, http.StatusOK) +} + +func (h *APIHandler) GetClusterIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + if GetMonitorState(id) == Console { + h.APIHandler.GetClusterIndices(w, req, ps) + return + } + var ( + //size = h.GetIntOrDefault(req, "size", 20) + //from = h.GetIntOrDefault(req, "from", 0) + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + resBody := map[string]interface{}{} + q := &orm.Query{Size: 2000} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + //orm.NotEq("metadata.labels.state", "delete"), + ) + + err, result := orm.Search(elastic.IndexConfig{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + indices, err := h.getLatestIndices(req, min, max, id, &result) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + + h.WriteJSON(w, indices, http.StatusOK) +} + +func (h *APIHandler) GetRealtimeClusterIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + if GetMonitorState(id) == Console { + h.APIHandler.GetRealtimeClusterIndices(w, req, ps) + return + } + meta := elastic.GetMetadata(id) + if meta == nil || !meta.IsAvailable() { + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + //filter indices + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, id) + if !hasAllPrivilege && len(allowedIndices) == 0 { + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + + esClient := elastic.GetClient(id) + indexInfos, err := esClient.GetIndices("") + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if !hasAllPrivilege { + filterIndices := map[string]elastic.IndexInfo{} + pattern := radix.Compile(allowedIndices...) + for indexName, indexInfo := range *indexInfos { + if pattern.Match(indexName) { + filterIndices[indexName] = indexInfo + } + } + indexInfos = &filterIndices + } + + qps, err := h.getIndexQPS(id, 20) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + var indices []RealtimeIndexInfo + for _, item := range *indexInfos { + info := RealtimeIndexInfo{ + IndexInfo: IndexInfo(item), + } + if _, ok := qps[item.Index]; ok { + info.IndexQPS = qps[item.Index]["index"] + info.QueryQPS = qps[item.Index]["query"] + info.IndexBytesQPS = qps[item.Index]["index_bytes"] + } + indices = append(indices, info) + } + h.WriteJSON(w, indices, http.StatusOK) +} + +type IndexInfo elastic.IndexInfo +type RealtimeIndexInfo struct { + IndexQPS interface{} `json:"index_qps"` + QueryQPS interface{} `json:"query_qps"` + IndexBytesQPS interface{} `json:"index_bytes_qps"` + IndexInfo +} +type CatNodeResponse elastic.CatNodeResponse +type RealtimeNodeInfo struct { + IndexQPS interface{} `json:"index_qps"` + QueryQPS interface{} `json:"query_qps"` + IndexBytesQPS interface{} `json:"index_bytes_qps"` + CatNodeResponse +} + +func (h *APIHandler) getIndexQPS(clusterID string, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + ver := h.Client().GetVersion() + bucketSizeStr := fmt.Sprintf("%ds", bucketSizeInSeconds) + intervalField, err := elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSizeStr) + if err != nil { + return nil, err + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + query := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "term_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 1000, + }, + "aggs": util.MapStr{ + "date": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "term_shard": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "filter_pri": util.MapStr{ + "filter": util.MapStr{ "term": util.MapStr{ "payload.elasticsearch.shard_stats.routing.primary": true } }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.indexing.index_total", + }, + }, + "index_bytes_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.store.size_in_bytes", + }, + }, + }, + }, + "query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.search.query_total", + }, + }, + }, + }, + "index_total": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>filter_pri>index_total", + }, + }, + "query_total": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>query_total", + }, + }, + "index_bytes_total": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>filter_pri>index_bytes_total", + }, + }, + "index_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + "query_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "query_total", + }, + }, + "index_bytes_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_bytes_total", + }, + }, + }, + }, + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-1m", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + }, + }, + }, + } + return h.QueryQPS(query, bucketSizeInSeconds) +} + +func (h *APIHandler) getShardQPS(clusterID string, nodeUUID string, indexName string, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + ver := h.Client().GetVersion() + bucketSizeStr := fmt.Sprintf("%ds", bucketSizeInSeconds) + intervalField, err := elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSizeStr) + if err != nil { + return nil, err + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + must := []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + } + if nodeUUID != "" { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeUUID, + }, + }, + }) + } + if indexName != "" { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }) + } + query := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "term_shard_id": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 5000, + }, + "aggs": util.MapStr{ + "date": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.search.query_total", + }, + }, + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.indexing.index_total", + }, + }, + "index_bytes_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.store.size_in_bytes", + }, + }, + "index_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + "query_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "query_total", + }, + }, + "index_bytes_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_bytes_total", + }, + }, + }, + }, + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-1m", + "lte": "now", + }, + }, + }, + }, + "must": must, + }, + }, + } + return h.QueryQPS(query, bucketSizeInSeconds) +} + +func (h *APIHandler) getNodeQPS(clusterID string, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + ver := h.Client().GetVersion() + bucketSizeStr := fmt.Sprintf("%ds", bucketSizeInSeconds) + intervalField, err := elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSizeStr) + if err != nil { + return nil, err + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + query := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "term_node": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "date": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + }, + }, + "index_bytes_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + }, + }, + "query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + }, + }, + "index_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + "index_bytes_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_bytes_total", + }, + }, + "query_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "query_total", + }, + }, + }, + }, + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-1m", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + }, + }, + } + return h.QueryQPS(query, bucketSizeInSeconds) +} + +func (h *APIHandler) SearchClusterMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := util.MapStr{} + reqBody := struct { + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + SearchField string `json:"search_field"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var should []util.MapStr + if reqBody.SearchField != "" { + should = []util.MapStr{ + { + "prefix": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "boost": 2, + }, + }, + }, + } + } else { + should = []util.MapStr{ + { + "prefix": util.MapStr{ + "name": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "prefix": util.MapStr{ + "host": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "prefix": util.MapStr{ + "version": util.MapStr{ + "value": reqBody.Keyword, + "boost": 15, + }, + }, + }, + { + "match_phrase_prefix": util.MapStr{ + "name.text": util.MapStr{ + "query": reqBody.Keyword, + "boost": 6, + }, + }, + }, + { + "match": util.MapStr{ + "search_text": util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "boost": 2, + }, + }, + }, + { + "query_string": util.MapStr{ + "fields": []string{"*"}, + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "fuzzy_prefix_length": 2, + "fuzzy_max_expansions": 10, + "allow_leading_wildcard": false, + }, + }, + } + } + + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + if !hasAllPrivilege && clusterFilter == nil { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + must := []interface{}{} + if !hasAllPrivilege && clusterFilter != nil { + must = append(must, clusterFilter) + } + query := util.MapStr{ + "aggs": elastic.BuildSearchTermAggregations(reqBody.Aggregations), + "size": reqBody.Size, + "from": reqBody.From, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": elastic.BuildSearchTermFilter(reqBody.Filter), + "should": should, + "must": must, + }, + }, + "sort": []util.MapStr{ + { + "updated": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(reqBody.Sort) > 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.ElasticsearchConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + w.Write(util.MustToJSONBytes(response)) +} diff --git a/modules/elastic/api/discover.go b/modules/elastic/api/discover.go new file mode 100644 index 00000000..ec30be7d --- /dev/null +++ b/modules/elastic/api/discover.go @@ -0,0 +1,299 @@ +package api + +import ( + "context" + "fmt" + "github.com/buger/jsonparser" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "time" +) + +func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists{ + errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(errStr) + h.WriteError(w, errStr, http.StatusNotFound) + return + } + + var reqParams = struct{ + Index string `json:"index"` + Body map[string]interface{} `json:"body"` + DistinctByField map[string]interface{} `json:"distinct_by_field"` + }{} + + err = h.DecodeJSON(req, &reqParams) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + ver := client.GetVersion() + if _, ok := reqParams.Body["track_total_hits"]; ok { + if ver.Distribution == "" || ver.Distribution == "elasticsearch" { + vr, _ := util.VersionCompare(ver.Number, "7.0") + if vr < 0 { + delete(reqParams.Body, "track_total_hits") + } + } + } + if reqParams.DistinctByField != nil { + if query, ok := reqParams.Body["query"]; ok { + if qm, ok := query.(map[string]interface{}); ok { + + filter, _ := util.MapStr(qm).GetValue("bool.filter") + if fv, ok := filter.([]interface{}); ok{ + fv = append(fv, util.MapStr{ + "script": util.MapStr{ + "script": util.MapStr{ + "source": "distinct_by_field", + "lang": "infini", + "params": reqParams.DistinctByField, + }, + }, + }) + util.MapStr(qm).Put("bool.filter", fv) + } + + } + } + } + if ver.Distribution == "" || ver.Distribution == "elasticsearch" { + vr, err := util.VersionCompare(ver.Number, "7.2") + if err != nil { + errStr := fmt.Sprintf("version compare error: %v", err) + log.Error(errStr) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if vr < 0 { + if aggs, ok := reqParams.Body["aggs"]; ok { + if maggs, ok := aggs.(map[string]interface{}); ok { + if aggsCounts, ok := maggs["counts"].(map[string]interface{}); ok { + if aggVals, ok := aggsCounts["date_histogram"].(map[string]interface{}); ok { + var interval interface{} + if calendarInterval, ok := aggVals["calendar_interval"]; ok { + interval = calendarInterval + delete(aggVals, "calendar_interval") + } + if fixedInterval, ok := aggVals["fixed_interval"]; ok { + interval = fixedInterval + delete(aggVals, "fixed_interval") + } + aggVals["interval"] = interval + } + } + } + } + } + } + indices, hasAll := h.GetAllowedIndices(req, targetClusterID) + if !hasAll { + if len(indices) == 0 { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + reqParams.Body["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []interface{}{ + util.MapStr{ + "terms": util.MapStr{ + "_index": indices, + }, + }, + reqParams.Body["query"], + }, + }, + } + } + + reqDSL := util.MustToJSONBytes(reqParams.Body) + timeout := h.GetParameterOrDefault(req, "timeout", "") + var queryArgs *[]util.KV + var ctx context.Context + if timeout != "" { + queryArgs = &[]util.KV{ + { + Key: "timeout", + Value: timeout, + }, + } + du, err := util.ParseDuration(timeout) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + var cancel context.CancelFunc + // here add one second for network delay + ctx, cancel = context.WithTimeout(context.Background(), du + time.Second) + defer cancel() + } + + searchRes, err := client.QueryDSL(ctx, reqParams.Index, queryArgs, reqDSL) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if searchRes.StatusCode != http.StatusOK { + h.WriteError(w, string(searchRes.RawResult.Body), http.StatusInternalServerError) + return + } + failures, _, _, _ := jsonparser.Get(searchRes.RawResult.Body, "_shards", "failures") + if len(failures) > 0 { + h.WriteError(w, string(failures), http.StatusInternalServerError) + return + } + h.WriteJSONHeader(w) + h.WriteHeader(w, http.StatusOK) + h.Write(w, searchRes.RawResult.Body) +} + + +func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists{ + errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) + h.WriteError(w, errStr, http.StatusNotFound) + return + } + + var reqParams = struct{ + BoolFilter interface{} `json:"boolFilter"` + FieldName string `json:"field"` + Query string `json:"query"` + }{} + err = h.DecodeJSON(req, &reqParams) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + indexName := ps.ByName("index") + boolQ := util.MapStr{ + "filter": reqParams.BoolFilter, + } + var values = []interface{}{} + indices, hasAll := h.GetAllowedIndices(req, targetClusterID) + if !hasAll { + if len(indices) == 0 { + h.WriteJSON(w, values,http.StatusOK) + return + } + boolQ["must"] = []util.MapStr{ + { + "terms": util.MapStr{ + "_index": indices, + }, + }, + } + } + queryBody := util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": boolQ, + }, + "aggs": util.MapStr{ + "suggestions": util.MapStr{ + "terms": util.MapStr{ + "field": reqParams.FieldName, + "include": reqParams.Query + ".*", + "execution_hint": "map", + "shard_size": 10, + }, + }, + }, + } + var queryBodyBytes = util.MustToJSONBytes(queryBody) + + searchRes, err := client.SearchWithRawQueryDSL(indexName, queryBodyBytes) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + for _, bucket := range searchRes.Aggregations["suggestions"].Buckets { + values = append(values, bucket["key"]) + } + h.WriteJSON(w, values,http.StatusOK) +} + +func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + traceID := h.GetParameterOrDefault(req, "traceID", "") + traceIndex := h.GetParameterOrDefault(req, "traceIndex", orm.GetIndexName(elastic.TraceMeta{})) + traceField := h.GetParameterOrDefault(req, "traceField", "trace_id") + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !exists{ + errStr := fmt.Sprintf("cluster [%s] not found",targetClusterID) + h.WriteError(w, errStr, http.StatusNotFound) + return + } + var queryDSL = util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + traceField: traceID, + }, + }, + { + "term": util.MapStr{ + "cluster_id": targetClusterID, + }, + }, + }, + }, + }, + } + searchRes, err := client.SearchWithRawQueryDSL(traceIndex, util.MustToJSONBytes(queryDSL)) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if searchRes.GetTotal() == 0 { + h.WriteJSON(w, []string{}, http.StatusOK) + return + } + var indexNames []string + for _, hit := range searchRes.Hits.Hits { + indexNames = append(indexNames, hit.Source["index"].(string)) + } + h.WriteJSON(w, indexNames, http.StatusOK) +} + diff --git a/modules/elastic/api/host.go b/modules/elastic/api/host.go new file mode 100644 index 00000000..436ae338 --- /dev/null +++ b/modules/elastic/api/host.go @@ -0,0 +1,1306 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/host" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net" + "net/http" + "strings" + "time" +) + +func (h *APIHandler) SearchHostMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := util.MapStr{} + reqBody := struct { + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + SearchField string `json:"search_field"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if reqBody.Size <= 0 { + reqBody.Size = 20 + } + aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) + var should = []util.MapStr{} + if reqBody.SearchField != "" { + should = []util.MapStr{ + { + "prefix": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + } + } else { + if reqBody.Keyword != "" { + should = []util.MapStr{ + { + "match": util.MapStr{ + "search_text": util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + } + } + } + + query := util.MapStr{ + "aggs": aggs, + "size": reqBody.Size, + "from": reqBody.From, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": elastic.BuildSearchTermFilter(reqBody.Filter), + "should": should, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(reqBody.Sort) > 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + q := &orm.Query{ + RawQuery: dsl, + } + err, result := orm.Search(host.HostInfo{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + w.Write(result.Raw) +} + +func (h *APIHandler) updateHost(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("host_id") + obj := host.HostInfo{} + + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + toUpObj := host.HostInfo{} + err = h.DecodeJSON(req, &toUpObj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + //protect + if toUpObj.Name != "" { + obj.Name = toUpObj.Name + } + obj.Tags = toUpObj.Tags + if toUpObj.IP != "" { + obj.IP = toUpObj.IP + } + err = orm.Update(nil, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "updated", + }, 200) +} + +func (h *APIHandler) getDiscoverHosts(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hosts, err := discoverHost() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + var hostlist = []interface{}{} + for _, host := range hosts { + hostlist = append(hostlist, host) + } + + h.WriteJSON(w, hostlist, http.StatusOK) +} + +func getHostSummary(agentIDs []string, metricName string, summary map[string]util.MapStr) error { + if summary == nil { + summary = map[string]util.MapStr{ + } + } + + if len(agentIDs) == 0 { + return fmt.Errorf("empty agent ids") + } + + q1 := orm.Query{WildcardIndex: true} + query := util.MapStr{ + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "collapse": util.MapStr{ + "field": "agent.id", + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "host", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": metricName, + }, + }, + }, + { + "terms": util.MapStr{ + "agent.id": agentIDs, + }, + }, + }, + }, + }, + } + q1.RawQuery = util.MustToJSONBytes(query) + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + return err + } + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + agentID, ok := util.GetMapValueByKeys([]string{"agent", "id"}, result) + if ok { + metric, ok := util.GetMapValueByKeys([]string{"payload", "host", metricName}, result) + if ok { + strAgentID := util.ToString(agentID) + if _, ok = summary[strAgentID]; ok { + summary[strAgentID][metricName] = metric + } else { + summary[strAgentID] = util.MapStr{ + metricName: metric, + } + } + } + + } + } + } + return nil +} + +func getHostSummaryFromNode(nodeIDs []string) (map[string]util.MapStr, error) { + q1 := orm.Query{WildcardIndex: true} + query := util.MapStr{ + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "collapse": util.MapStr{ + "field": "metadata.labels.node_id", + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + }, + }, + } + q1.RawQuery = util.MustToJSONBytes(query) + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + return nil, err + } + summary := map[string]util.MapStr{} + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) + if ok { + strNodeID := util.ToString(nodeID) + summary[strNodeID] = util.MapStr{} + osCPUPercent, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "percent"}, result) + if ok { + summary[strNodeID]["cpu"] = util.MapStr{ + "used_percent": osCPUPercent, + } + } + osMem, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "mem"}, result) + if osMemM, ok := osMem.(map[string]interface{}); ok { + summary[strNodeID]["memory"] = util.MapStr{ + "used.percent": osMemM["used_percent"], + "available.bytes": osMemM["free_in_bytes"], + "total.bytes": osMemM["total_in_bytes"], + "used.bytes": osMemM["used_in_bytes"], + } + } + fsTotal, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total"}, result) + if fsM, ok := fsTotal.(map[string]interface{}); ok { + total, ok1 := fsM["total_in_bytes"].(float64) + free, ok2 := fsM["free_in_bytes"].(float64) + if ok1 && ok2 { + summary[strNodeID]["filesystem_summary"] = util.MapStr{ + "used.percent": (total - free) * 100 / total, + "total.bytes": total, + "free.bytes": free, + "used.bytes": total - free, + } + } + } + } + } + } + return summary, nil +} + +func getHostSummaryFromAgent(agentIDs []string) (map[string]util.MapStr, error) { + summary := map[string]util.MapStr{} + if len(agentIDs) == 0 { + return summary, nil + } + err := getHostSummary(agentIDs, "cpu", summary) + if err != nil { + return nil, err + } + err = getHostSummary(agentIDs, "memory", summary) + if err != nil { + return nil, err + } + err = getHostSummary(agentIDs, "filesystem_summary", summary) + return summary, err +} + +func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var hostIDs = []string{} + h.DecodeJSON(req, &hostIDs) + + if len(hostIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + queryDsl := util.MapStr{ + "query": util.MapStr{ + "terms": util.MapStr{ + "id": hostIDs, + }, + }, + } + q := &orm.Query{ + RawQuery: util.MustToJSONBytes(queryDsl), + } + err, result := orm.Search(host.HostInfo{}, q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(result.Result) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + var agentIDs []string + var nodeIDs []string + var hostIDToNodeID = map[string]string{} + var hostIDToAgentID = map[string]string{} + for _, row := range result.Result { + tempHost := host.HostInfo{} + buf := util.MustToJSONBytes(row) + err = util.FromJSONBytes(buf, &tempHost) + if err != nil { + log.Error(err) + continue + } + if tempHost.AgentID != "" { + agentIDs = append(agentIDs, tempHost.AgentID) + hostIDToAgentID[tempHost.ID] = tempHost.AgentID + continue + } + if tempHost.NodeID != "" { + nodeIDs = append(nodeIDs, tempHost.NodeID) + hostIDToNodeID[tempHost.ID] = tempHost.NodeID + } + } + + summaryFromAgent, err := getHostSummaryFromAgent(agentIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + var summaryFromNode = map[string]util.MapStr{} + if len(nodeIDs) > 0 { + summaryFromNode, err = getHostSummaryFromNode(nodeIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + + statusMetric, err := getAgentOnlineStatusOfRecentDay(hostIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, 15) + if err != nil { + panic(err) + return + } + networkInMetricItem := newMetricItem("network_in_rate", 1, SystemGroupKey) + networkInMetricItem.AddAxi("network_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + networkOutMetricItem := newMetricItem("network_out_rate", 1, SystemGroupKey) + networkOutMetricItem.AddAxi("network_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + hostMetricItems := []GroupMetricItem{ + { + Key: "network_in_rate", + Field: "payload.host.network_summary.in.bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: networkInMetricItem, + FormatType: "bytes", + Units: "/s", + }, + { + Key: "network_out_rate", + Field: "payload.host.network_summary.out.bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: networkOutMetricItem, + FormatType: "bytes", + Units: "/s", + }, + } + hostMetrics := h.getGroupHostMetric(agentIDs, min, max, bucketSize, hostMetricItems, "agent.id") + + networkMetrics := map[string]util.MapStr{} + for key, item := range hostMetrics { + for _, line := range item.Lines { + if _, ok := networkMetrics[line.Metric.Label]; !ok { + networkMetrics[line.Metric.Label] = util.MapStr{ + } + } + networkMetrics[line.Metric.Label][key] = line.Data + } + } + + infos := util.MapStr{} + for _, hostID := range hostIDs { + source := util.MapStr{} + metrics := util.MapStr{ + "agent_status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Agent Status", + "units": "day", + }, + "data": statusMetric[hostID], + }, + } + if agentID, ok := hostIDToAgentID[hostID]; ok { + source["summary"] = summaryFromAgent[agentID] + metrics["network_in_rate"] = util.MapStr{ + "metric": util.MapStr{ + "label": "Network In Rate", + "units": "", + }, + "data": networkMetrics[agentID]["network_in_rate"], + } + metrics["network_out_rate"] = util.MapStr{ + "metric": util.MapStr{ + "label": "Network Out Rate", + "units": "", + }, + "data": networkMetrics[agentID]["network_out_rate"], + } + } else { + if nid, ok := hostIDToNodeID[hostID]; ok { + source["summary"] = summaryFromNode[nid] + } + } + source["metrics"] = metrics + infos[hostID] = source + } + h.WriteJSON(w, infos, http.StatusOK) +} +func (h *APIHandler) GetHostInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteJSON(w, util.MapStr{ + "_id": hostID, + "found": false, + }, http.StatusNotFound) + return + } + + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": hostID, + "_source": hostInfo, + }, 200) + +} + +func (h *APIHandler) getSingleHostMetric(agentID string, min, max int64, bucketSize int, metricItems []*common.MetricItem) map[string]*common.MetricItem { + var must = []util.MapStr{ + { + "term": util.MapStr{ + "agent.id": util.MapStr{ + "value": agentID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "host", + }, + }, + }, + } + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + return h.getSingleMetrics(metricItems, query, bucketSize) +} + +func (h *APIHandler) getSingleHostMetricFromNode(nodeID string, min, max int64, bucketSize int) map[string]*common.MetricItem { + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeID, + }, + }, + }, + } + + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem := newMetricItem("cpu_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("cpu", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("CPU Used Percent", "CPU", "cpu used percent of host.", "group1", "payload.elasticsearch.node_stats.os.cpu.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("memory_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("Memory", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Memory Used Percent", "Memory Used Percent", "memory used percent of host.", "group1", "payload.elasticsearch.node_stats.os.mem.used_percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("disk_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("disk", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Disk Used Percent", "Disk Used Percent", "disk used percent of host.", "group1", "payload.elasticsearch.node_stats.fs.total.free_in_bytes", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.fs.total.total_in_bytes" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return 100 - value*100/value2 + } + metricItems = append(metricItems, metricItem) + return h.getSingleMetrics(metricItems, query, bucketSize) +} + +func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteError(w, fmt.Sprintf("host [%s] not found", hostID), http.StatusNotFound) + return + } + + resBody := map[string]interface{}{} + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 60) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if hostInfo.AgentID == "" { + resBody["metrics"] = h.getSingleHostMetricFromNode(hostInfo.NodeID, min, max, bucketSize) + h.WriteJSON(w, resBody, http.StatusOK) + return + } + isOverview := h.GetIntOrDefault(req, "overview", 0) + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem := newMetricItem("cpu_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("cpu", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("CPU Used Percent", "CPU", "cpu used percent of host.", "group1", "payload.host.cpu.used_percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + if isOverview == 0 { + metricItem = newMetricItem("system_load", 1, SystemGroupKey) + metricItem.AddAxi("system_load", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Load1", "Load1", "system load1.", "group1", "payload.host.cpu.load.load1", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Load5", "Load5", "system load5.", "group1", "payload.host.cpu.load.load5", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Load15", "Load15", "system load15.", "group1", "payload.host.cpu.load.load15", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("cpu_iowait", 1, SystemGroupKey) + metricItem.AddAxi("cpu_iowait", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("iowait", "iowait", "cpu iowait.", "group1", "payload.host.cpu.iowait", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + } + + metricItem = newMetricItem("memory_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("Memory", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Memory Used Percent", "Memory Used Percent", "memory used percent of host.", "group1", "payload.host.memory.used.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + if isOverview == 0 { + metricItem = newMetricItem("swap_memory_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("Swap Memory", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Swap Memory Used Percent", "Swap Memory Used Percent", "swap memory used percent of host.", "group1", "payload.host.memory_swap.used_percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + } + + metricItem = newMetricItem("network_summary", 1, SystemGroupKey) + metricItem.AddAxi("network_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Network In Rate", "Network In Rate", "network in rate of host.", "group1", "payload.host.network_summary.in.bytes", "max", bucketSizeStr, "/s", "bytes", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Network Out Rate", "Network Out Rate", "network out rate of host.", "group1", "payload.host.network_summary.out.bytes", "max", bucketSizeStr, "/s", "bytes", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + if isOverview == 0 { + metricItem = newMetricItem("network_packets_summary", 1, SystemGroupKey) + metricItem.AddAxi("network_packets_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Network Packets In Rate", "Network Packets In Rate", "network packets in rate of host.", "group1", "payload.host.network_summary.in.packets", "max", bucketSizeStr, "packets/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Network Packets Out Rate", "Network Packets Out Rate", "network packets out rate of host.", "group1", "payload.host.network_summary.out.packets", "max", bucketSizeStr, "packets/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + } + + metricItem = newMetricItem("disk_used_percent", 1, SystemGroupKey) + metricItem.AddAxi("disk", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Disk Used Percent", "Disk Used Percent", "disk used percent of host.", "group1", "payload.host.filesystem_summary.used.percent", "max", bucketSizeStr, "%", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("disk_read_rate", 1, SystemGroupKey) + metricItem.AddAxi("disk_read_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Disk Read Rate", "Disk Read Rate", "Disk read rate of host.", "group1", "payload.host.diskio_summary.read.bytes", "max", bucketSizeStr, "%", "bytes", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("disk_write_rate", 1, SystemGroupKey) + metricItem.AddAxi("disk_write_rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddLine("Disk Write Rate", "Disk Write Rate", "network write rate of host.", "group1", "payload.host.diskio_summary.write.bytes", "max", bucketSizeStr, "%", "bytes", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + hostMetrics := h.getSingleHostMetric(hostInfo.AgentID, min, max, bucketSize, metricItems) + if isOverview == 0 { + groupMetrics := h.getGroupHostMetrics(hostInfo.AgentID, min, max, bucketSize) + if hostMetrics == nil { + hostMetrics = map[string]*common.MetricItem{} + } + for k, v := range groupMetrics { + hostMetrics[k] = v + } + } + + resBody["metrics"] = hostMetrics + + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) getGroupHostMetrics(agentID string, min, max int64, bucketSize int) map[string]*common.MetricItem { + diskPartitionMetric := newMetricItem("disk_partition_usage", 2, SystemGroupKey) + diskPartitionMetric.AddAxi("Disk Partition Usage", "group1", common.PositionLeft, "ratio", "0.[0]", "0.[0]", 5, true) + hostMetricItems := []GroupMetricItem{ + { + Key: "disk_partition_usage", + Field: "payload.host.disk_partition_usage.used_percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: diskPartitionMetric, + FormatType: "ratio", + Units: "%", + }, + } + hostMetrics := h.getGroupHostMetric([]string{agentID}, min, max, bucketSize, hostMetricItems, "payload.host.disk_partition_usage.partition") + networkOutputMetric := newMetricItem("network_interface_output_rate", 2, SystemGroupKey) + networkOutputMetric.AddAxi("Network interface output rate", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + hostMetricItems = []GroupMetricItem{ + { + Key: "network_interface_output_rate", + Field: "payload.host.network_interface.output_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: networkOutputMetric, + FormatType: "bytes", + Units: "", + }, + } + networkOutMetrics := h.getGroupHostMetric([]string{agentID}, min, max, bucketSize, hostMetricItems, "payload.host.network_interface.name") + if networkOutMetrics != nil { + hostMetrics["network_interface_output_rate"] = networkOutMetrics["network_interface_output_rate"] + } + return hostMetrics +} + +func (h *APIHandler) getGroupHostMetric(agentIDs []string, min, max int64, bucketSize int, hostMetricItems []GroupMetricItem, groupField string) map[string]*common.MetricItem { + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "host", + }, + }, + }, + } + if len(agentIDs) > 0 { + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "agent.id": agentIDs, + }, + }) + } + query := map[string]interface{}{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + } + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + aggs := generateGroupAggs(hostMetricItems) + query["aggs"] = util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": groupField, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + "fixed_interval": bucketSizeStr, + }, + "aggs": aggs, + }, + }, + }, + } + return h.getMetrics(query, hostMetricItems, bucketSize) +} + +func getHost(hostID string) (*host.HostInfo, error) { + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + return nil, fmt.Errorf("get host info error: %w", err) + } + if !exists { + return nil, fmt.Errorf("host [%s] not found", hostID) + } + return hostInfo, nil +} + +func (h *APIHandler) GetHostMetricStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo, err := getHost(hostID) + if err != nil { + log.Error(err) + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + if hostInfo.AgentID == "" { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + queryDSL := util.MapStr{ + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "collapse": util.MapStr{ + "field": "metadata.name", + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "agent.id": util.MapStr{ + "value": hostInfo.AgentID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "host", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.name": []string{ + "filesystem_summary", + "cpu", + "memory", + "network_summary", + "network", + }, + }, + }, + }, + }, + }, + } + q := &orm.Query{ + WildcardIndex: true, + RawQuery: util.MustToJSONBytes(queryDSL), + } + err, result := orm.Search(event.Event{}, q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusNotFound) + return + } + var metricStats []util.MapStr + for _, row := range result.Result { + if rowM, ok := row.(map[string]interface{}); ok { + metricName, _ := util.GetMapValueByKeys([]string{"metadata", "name"}, rowM) + if mv, ok := metricName.(string); ok { + var status = "failure" + if ts, ok := rowM["timestamp"].(string); ok { + lastTime, _ := time.Parse(time.RFC3339, ts) + if time.Since(lastTime).Seconds() < 60 { + status = "success" + } + } + metricStats = append(metricStats, util.MapStr{ + "metric_name": mv, + "timestamp": rowM["timestamp"], + "status": status, + }) + } + } + } + h.WriteJSON(w, metricStats, http.StatusOK) +} + +func (h *APIHandler) GetHostOverviewInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + hostID := ps.MustGetParameter("host_id") + hostInfo := &host.HostInfo{} + hostInfo.ID = hostID + exists, err := orm.Get(hostInfo) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteJSON(w, util.MapStr{ + "_id": hostID, + "found": false, + }, http.StatusNotFound) + return + } + + var ( + summary util.MapStr + ) + if hostInfo.AgentID != "" { + summaries, err := getHostSummaryFromAgent([]string{hostID}) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if v, ok := summaries[hostID]; ok { + summary = v + } + + } else if hostInfo.NodeID != "" { + summaries, err := getHostSummaryFromNode([]string{hostInfo.NodeID}) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if v, ok := summaries[hostInfo.NodeID]; ok { + summary = v + } + } + h.WriteJSON(w, util.MapStr{ + "host_mame": hostInfo.Name, + "ip": hostInfo.IP, + "os_info": hostInfo.OSInfo, + "agent_status": hostInfo.AgentStatus, + "summary": summary, + "agent_id": hostInfo.AgentID, + }, http.StatusOK) + +} + +// discoverHost auto discover host ip from elasticsearch node metadata and agent ips +func discoverHost() (map[string]interface{}, error) { + queryDsl := util.MapStr{ + "size": 1000, + "_source": []string{"ip", "name"}, + } + q := &orm.Query{RawQuery: util.MustToJSONBytes(queryDsl)} + err, result := orm.Search(host.HostInfo{}, q) + if err != nil { + return nil, fmt.Errorf("search host error: %w", err) + } + hosts := map[string]interface{}{} + for _, row := range result.Result { + if rowM, ok := row.(map[string]interface{}); ok { + if ip, ok := rowM["ip"].(string); ok { + hosts[ip] = rowM["name"] + } + } + } + + queryDsl = util.MapStr{ + "_source": []string{"metadata.labels.ip", "metadata.node_id", "metadata.node_name", "payload.node_state.os"}, + "collapse": util.MapStr{ + "field": "metadata.labels.ip", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + } + q = &orm.Query{ + RawQuery: util.MustToJSONBytes(queryDsl), + } + err, result = orm.Search(elastic.NodeConfig{}, q) + if err != nil { + return nil, fmt.Errorf("search node metadata error: %w", err) + } + hostsFromES := map[string]interface{}{} + for _, row := range result.Result { + if rowM, ok := row.(map[string]interface{}); ok { + rowV := util.MapStr(rowM) + hostIP, _ := rowV.GetValue("metadata.labels.ip") + if v, ok := hostIP.(string); ok { + if _, ok = hosts[v]; ok { + continue + } + nodeUUID, _ := rowV.GetValue("metadata.node_id") + nodeName, _ := rowV.GetValue("metadata.node_name") + osName, _ := rowV.GetValue("payload.node_state.os.name") + osArch, _ := rowV.GetValue("payload.node_state.os.arch") + hostsFromES[v] = util.MapStr{ + "ip": v, + "node_uuid": nodeUUID, + "node_name": nodeName, + "source": "es_node", + "os_name": osName, + "host_name": "", + "os_arch": osArch, + } + } + + } + } + + queryDsl = util.MapStr{ + "size": 1000, + "_source": []string{"id", "ip", "remote_ip", "major_ip", "host"}, + //"query": util.MapStr{ + // "term": util.MapStr{ + // "enrolled": util.MapStr{ + // "value": true, + // }, + // }, + //}, + } + q = &orm.Query{RawQuery: util.MustToJSONBytes(queryDsl)} + err, result = orm.Search(model.Instance{}, q) + if err != nil { + return nil, fmt.Errorf("search agent error: %w", err) + } + + hostsFromAgent := map[string]interface{}{} + for _, row := range result.Result { + ag := model.Instance{} + bytes := util.MustToJSONBytes(row) + err = util.FromJSONBytes(bytes, &ag) + if err != nil { + log.Errorf("got unexpected agent: %s, error: %v", string(bytes), err) + continue + } + var ip = ag.Network.MajorIP + if ip = strings.TrimSpace(ip); ip == "" { + for _, ipr := range ag.Network.IP { + if net.ParseIP(ipr).IsPrivate() { + ip = ipr + break + } + } + } + if _, ok := hosts[ip]; ok { + continue + } + + data := util.MapStr{ + "ip": ip, + "agent_id": ag.ID, + "agent_host": ag.Endpoint, + "source": "agent", + } + if ag.Host != nil { + data["os_name"] = ag.Host.OS.Name + data["os_arch"] = ag.Host.OS.Architecture + data["host_name"] = ag.Host.Name + } + hostsFromAgent[ip] = data + } + err = util.MergeFields(hostsFromES, hostsFromAgent, true) + + return hostsFromES, err +} + +func getAgentOnlineStatusOfRecentDay(hostIDs []string) (map[string][]interface{}, error) { + if hostIDs == nil { + hostIDs = []string{} + } + + q := orm.Query{ + WildcardIndex: true, + } + query := util.MapStr{ + "aggs": util.MapStr{ + "group_by_host_id": util.MapStr{ + "terms": util.MapStr{ + "field": "agent.host_id", + "size": 100, + }, + "aggs": util.MapStr{ + "uptime_histogram": util.MapStr{ + "date_range": util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + }, { + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }, + "aggs": util.MapStr{ + "min_uptime": util.MapStr{ + "min": util.MapStr{ + "field": "payload.agent.agent_basic.uptime_in_ms", + }, + }, + }, + }, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-15d", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "agent_basic", + }, + }, + }, + { + "terms": util.MapStr{ + "agent.host_id": hostIDs, + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + return nil, err + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + recentStatus := map[string][]interface{}{} + for _, bk := range response.Aggregations["group_by_host_id"].Buckets { + agentKey := bk["key"].(string) + recentStatus[agentKey] = []interface{}{} + if histogramAgg, ok := bk["uptime_histogram"].(map[string]interface{}); ok { + if bks, ok := histogramAgg["buckets"].([]interface{}); ok { + for _, bkItem := range bks { + if bkVal, ok := bkItem.(map[string]interface{}); ok { + if minUptime, ok := util.GetMapValueByKeys([]string{"min_uptime", "value"}, bkVal); ok { + //mark agent status as offline when uptime less than 10m + if v, ok := minUptime.(float64); ok && v >= 600000 { + recentStatus[agentKey] = append(recentStatus[agentKey], []interface{}{bkVal["key"], "online"}) + } else { + recentStatus[agentKey] = append(recentStatus[agentKey], []interface{}{bkVal["key"], "offline"}) + } + } + } + } + } + } + } + emptyStatus := getAgentEmptyStatusOfRecentDay(14) + for _, hostID := range hostIDs { + if _, ok := recentStatus[hostID]; !ok { + recentStatus[hostID] = emptyStatus + } + } + return recentStatus, nil +} + +func getAgentEmptyStatusOfRecentDay(days int) []interface{} { + now := time.Now() + startTime := now.Add(-time.Duration(days-1) * time.Hour * 24) + year, month, day := startTime.Date() + startTime = time.Date(year, month, day, 0, 0, 0, 0, startTime.Location()) + var status []interface{} + for i := 1; i <= days; i++ { + nextTime := startTime.Add(time.Hour * 24) + if nextTime.After(now) { + nextTime = now + } + status = append(status, []interface{}{ + fmt.Sprintf("%s-%s", startTime.Format("2006-01-02"), nextTime.Format("2006-01-02")), + "offline", + }) + startTime = nextTime + } + return status +} diff --git a/modules/elastic/api/ilm.go b/modules/elastic/api/ilm.go new file mode 100644 index 00000000..08f62f6e --- /dev/null +++ b/modules/elastic/api/ilm.go @@ -0,0 +1,57 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "io" + "net/http" +) + +func (h *APIHandler) HandleGetILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + clusterID := ps.MustGetParameter("id") + esClient := elastic.GetClient(clusterID) + policies, err := esClient.GetILMPolicy("") + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, policies, http.StatusOK) +} + +func (h *APIHandler) HandleSaveILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + clusterID := ps.MustGetParameter("id") + policy := ps.MustGetParameter("policy") + esClient := elastic.GetClient(clusterID) + reqBody, err := io.ReadAll(req.Body) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + err = esClient.PutILMPolicy(policy, reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteAckOKJSON(w) +} + +func (h *APIHandler) HandleDeleteILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + clusterID := ps.MustGetParameter("id") + policy := ps.MustGetParameter("policy") + esClient := elastic.GetClient(clusterID) + err := esClient.DeleteILMPolicy(policy) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteAckOKJSON(w) +} \ No newline at end of file diff --git a/modules/elastic/api/index_metrics.go b/modules/elastic/api/index_metrics.go new file mode 100644 index 00000000..e1d6f8c5 --- /dev/null +++ b/modules/elastic/api/index_metrics.go @@ -0,0 +1,953 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "net/http" + "sort" + "strings" + "time" +) + +func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int, shardID string) (map[string]*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + } + if v := strings.TrimSpace(shardID); v != "" { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "metadata.labels.shard_id": util.MapStr{ + "value": shardID, + }, + }, + }) + } + var ( + indexNames []string + ) + if indexName != "" { + indexNames = strings.Split(indexName, ",") + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return nil, nil + } + if !hasAllPrivilege{ + namePattern := radix.Compile(allowedIndices...) + var filterNames []string + for _, name := range indexNames { + if namePattern.Match(name){ + filterNames = append(filterNames, name) + } + } + if len(filterNames) == 0 { + return nil, nil + } + indexNames = filterNames + } + top = len(indexNames) + + }else{ + indexNames, err = h.getTopIndexName(req, clusterID, top, 15) + if err != nil { + return nil, err + } + + } + if len(indexNames) > 0 { + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "metadata.labels.index_name": indexNames, + }, + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": "_all", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + //索引存储大小 + indexStorageMetric := newMetricItem("index_storage", 1, StorageGroupKey) + indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + + indexMetricItems := []GroupMetricItem{ + { + Key: "index_storage", + Field: "payload.elasticsearch.shard_stats.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexStorageMetric, + FormatType: "bytes", + Units: "", + }, + } + // segment 数量 + segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey) + segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_count", + Field: "payload.elasticsearch.shard_stats.segments.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentCountMetric, + FormatType: "num", + Units: "", + }) + //索引文档个数 + docCountMetric := newMetricItem("doc_count", 2, DocumentGroupKey) + docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "doc_count", + Field: "payload.elasticsearch.shard_stats.docs.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docCountMetric, + FormatType: "num", + Units: "", + }) + // docs 删除数量 + docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey) + docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "docs_deleted", + Field: "payload.elasticsearch.shard_stats.docs.deleted", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsDeletedMetric, + FormatType: "num", + Units: "", + }) + //查询次数 + queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey) + queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_times", + Field: "payload.elasticsearch.shard_stats.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + + //Fetch次数 + fetchTimesMetric := newMetricItem("fetch_times", 3, OperationGroupKey) + fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "fetch_times", + Field: "payload.elasticsearch.shard_stats.search.fetch_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //scroll 次数 + scrollTimesMetric := newMetricItem("scroll_times", 4, OperationGroupKey) + scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "scroll_times", + Field: "payload.elasticsearch.shard_stats.search.scroll_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //Merge次数 + mergeTimesMetric := newMetricItem("merge_times", 7, OperationGroupKey) + mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "merge_times", + Field: "payload.elasticsearch.shard_stats.merges.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //Refresh次数 + refreshTimesMetric := newMetricItem("refresh_times", 5, OperationGroupKey) + refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "refresh_times", + Field: "payload.elasticsearch.shard_stats.refresh.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //flush 次数 + flushTimesMetric := newMetricItem("flush_times", 6, OperationGroupKey) + flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "flush_times", + Field: "payload.elasticsearch.shard_stats.flush.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + + //写入速率 + indexingRateMetric := newMetricItem("indexing_rate", 1, OperationGroupKey) + if shardID == "" { + indexingRateMetric.OnlyPrimary = true + } + indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_rate", + Field: "payload.elasticsearch.shard_stats.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingRateMetric, + FormatType: "num", + Units: "doc/s", + }) + indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey) + if shardID == "" { + indexingBytesMetric.OnlyPrimary = true + } + indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_bytes", + Field: "payload.elasticsearch.shard_stats.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingBytesMetric, + FormatType: "bytes", + Units: "bytes/s", + }) + //写入时延 + indexingLatencyMetric := newMetricItem("indexing_latency", 1, LatencyGroupKey) + if shardID == "" { + indexingLatencyMetric.OnlyPrimary = true + } + indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_latency", + Field: "payload.elasticsearch.shard_stats.indexing.index_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.indexing.index_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //查询时延 + queryLatencyMetric := newMetricItem("query_latency", 2, LatencyGroupKey) + queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_latency", + Field: "payload.elasticsearch.shard_stats.search.query_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.search.query_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //fetch时延 + fetchLatencyMetric := newMetricItem("fetch_latency", 3, LatencyGroupKey) + fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "fetch_latency", + Field: "payload.elasticsearch.shard_stats.search.fetch_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.search.fetch_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //merge时延 + mergeLatencyMetric := newMetricItem("merge_latency", 7, LatencyGroupKey) + mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "merge_latency", + Field: "payload.elasticsearch.shard_stats.merges.total_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.merges.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //refresh时延 + refreshLatencyMetric := newMetricItem("refresh_latency", 5, LatencyGroupKey) + refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "refresh_latency", + Field: "payload.elasticsearch.shard_stats.refresh.total_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.refresh.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //scroll时延 + scrollLatencyMetric := newMetricItem("scroll_latency", 4, LatencyGroupKey) + scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "scroll_latency", + Field: "payload.elasticsearch.shard_stats.search.scroll_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.search.scroll_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //flush 时延 + flushLatencyMetric := newMetricItem("flush_latency", 6, LatencyGroupKey) + flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "flush_latency", + Field: "payload.elasticsearch.shard_stats.flush.total_time_in_millis", + Field2: "payload.elasticsearch.shard_stats.flush.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //queryCache + queryCacheMetric := newMetricItem("query_cache", 1, CacheGroupKey) + queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_cache", + Field: "payload.elasticsearch.shard_stats.query_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: queryCacheMetric, + FormatType: "bytes", + Units: "", + }) + //requestCache + requestCacheMetric := newMetricItem("request_cache", 2, CacheGroupKey) + requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "request_cache", + Field: "payload.elasticsearch.shard_stats.request_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: requestCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache Hit + requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey) + requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "request_cache_hit", + Field: "payload.elasticsearch.shard_stats.request_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + // Request Cache Miss + requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey) + requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "request_cache_miss", + Field: "payload.elasticsearch.shard_stats.request_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Query Cache Count + queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey) + queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_count", + Field: "payload.elasticsearch.shard_stats.query_cache.cache_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheCountMetric, + FormatType: "num", + Units: "", + }) + // Query Cache Miss + queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey) + queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_hit", + Field: "payload.elasticsearch.shard_stats.query_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + + //// Query Cache evictions + //queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 11, CacheGroupKey) + //queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + //indexMetricItems=append(indexMetricItems, GroupMetricItem{ + // Key: "query_cache_evictions", + // Field: "payload.elasticsearch.index_stats.total.query_cache.evictions", + // ID: util.GetUUID(), + // IsDerivative: true, + // MetricItem: queryCacheEvictionsMetric, + // FormatType: "num", + // Units: "evictions", + //}) + + // Query Cache Miss + queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey) + queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_miss", + Field: "payload.elasticsearch.shard_stats.query_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Fielddata内存占用大小 + fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey) + fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "fielddata_cache", + Field: "payload.elasticsearch.shard_stats.fielddata.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldDataCacheMetric, + FormatType: "bytes", + Units: "", + }) + //segment memory + segmentMemoryMetric := newMetricItem("segment_memory", 13, MemoryGroupKey) + segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_memory", + Field: "payload.elasticsearch.shard_stats.segments.memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment doc values memory + docValuesMemoryMetric := newMetricItem("segment_doc_values_memory", 13, MemoryGroupKey) + docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_doc_values_memory", + Field: "payload.elasticsearch.shard_stats.segments.doc_values_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docValuesMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment terms memory + termsMemoryMetric := newMetricItem("segment_terms_memory", 13, MemoryGroupKey) + termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_terms_memory", + Field: "payload.elasticsearch.shard_stats.segments.terms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: termsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment fields memory + fieldsMemoryMetric := newMetricItem("segment_fields_memory", 13, MemoryGroupKey) + fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_fields_memory", + Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment index writer memory + segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey) + segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_index_writer_memory", + Field: "payload.elasticsearch.shard_stats.segments.index_writer_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentIndexWriterMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment term vectors memory + segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey) + segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_term_vectors_memory", + Field: "payload.elasticsearch.shard_stats.segments.term_vectors_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermVectorsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + aggs:=map[string]interface{}{} + sumAggs := util.MapStr{} + var filterSubAggs = util.MapStr{} + + for _,metricItem:=range indexMetricItems { + leafAgg := util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + var sumBucketPath = "term_shard>"+ metricItem.ID + if metricItem.MetricItem.OnlyPrimary { + filterSubAggs[metricItem.ID] = leafAgg + aggs["filter_pri"]=util.MapStr{ + "filter": util.MapStr{ + "term": util.MapStr{ + "payload.elasticsearch.shard_stats.routing.primary": util.MapStr{ + "value": true, + }, + }, + }, + "aggs": filterSubAggs, + } + sumBucketPath = "term_shard>filter_pri>"+ metricItem.ID + }else{ + aggs[metricItem.ID]= leafAgg + } + + sumAggs[metricItem.ID] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath, + }, + } + + if metricItem.Field2 != ""{ + leafAgg2 := util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field2, + }, + } + if metricItem.MetricItem.OnlyPrimary { + filterSubAggs[metricItem.ID+"_field2"] = leafAgg2 + }else{ + aggs[metricItem.ID+"_field2"] = leafAgg2 + } + sumAggs[metricItem.ID + "_field2"] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath + "_field2", + }, + } + } + + if metricItem.IsDerivative{ + sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + sumAggs[metricItem.ID + "_deriv_field2"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + sumAggs["term_shard"]= util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 10000, + }, + "aggs": aggs, + } + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": top, + // max_store is a pipeline agg, sort not support + //"order": util.MapStr{ + // "max_store": "desc", + //}, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":sumAggs, + }, + "max_store_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_store": util.MapStr{"order": "desc"}}}, + "size": top, + }, + }, + "term_shard": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_store": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.store.size_in_bytes", + }, + }, + }, + }, + "max_store": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>max_store", + }, + }, + }, + }, + } + return h.getMetrics(query, indexMetricItems, bucketSize), nil + +} + +func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error){ + ver := h.Client().GetVersion() + cr, _ := util.VersionCompare(ver.Number, "6.1") + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { + return nil, nil + } + var ( + now = time.Now() + max = now.UnixNano()/1e6 + min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 + ) + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + } + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return nil, fmt.Errorf("no index permission") + } + if !hasAllPrivilege { + must = append(must, util.MapStr{ + "query_string": util.MapStr{ + "query": strings.Join(allowedIndices, " "), + "fields": []string{"metadata.labels.index_name"}, + "default_operator": "OR", + }, + }) + } + bucketSizeStr := "60s" + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + + query := util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": "_all", + }, + }, + }, + }, + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "group_by_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>search_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}}, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "term_shard": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 10000, + }, + "aggs": util.MapStr{ + "search_query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.search.query_total", + }, + }, + }, + }, + "sum_search_query_total": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>search_query_total", + }, + }, + "search_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "sum_search_query_total", + }, + }, + }, + }, + }, + }, + "group_by_index1": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>index_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}, + }, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "term_shard": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 10000, + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.indexing.index_total", + }, + }, + }, + }, + "sum_index_total": util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": "term_shard>index_total", + }, + }, + "index_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "sum_index_total", + }, + }, + }, + }, + }, + }, + }, + } + response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) + if err!=nil{ + log.Error(err) + return nil, err + } + var maxQpsKVS = map[string] float64{} + for _, agg := range response.Aggregations { + for _, bk := range agg.Buckets { + key := bk["key"].(string) + if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { + val := maxQps["value"].(float64) + if _, ok = maxQpsKVS[key] ; ok { + maxQpsKVS[key] = maxQpsKVS[key] + val + }else{ + maxQpsKVS[key] = val + } + } + } + } + var ( + qpsValues TopTermOrder + ) + for k, v := range maxQpsKVS { + qpsValues = append(qpsValues, TopTerm{ + Key: k, + Value: v, + }) + } + sort.Sort(qpsValues) + var length = top + if top > len(qpsValues) { + length = len(qpsValues) + } + indexNames := []string{} + for i := 0; i t[j].Value //desc +} +func (t TopTermOrder) Swap(i, j int){ + t[i], t[j] = t[j], t[i] +} diff --git a/modules/elastic/api/index_overview.go b/modules/elastic/api/index_overview.go new file mode 100644 index 00000000..e007c340 --- /dev/null +++ b/modules/elastic/api/index_overview.go @@ -0,0 +1,1432 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "net/http" + "strings" +) + +func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody:=util.MapStr{} + reqBody := struct{ + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + SearchField string `json:"search_field"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) + aggs["term_cluster_id"] = util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.cluster_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "term_cluster_name": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.cluster_name", + "size": 1, + }, + }, + }, + } + filter := elastic.BuildSearchTermFilter(reqBody.Filter) + var should []util.MapStr + if reqBody.SearchField != ""{ + should = []util.MapStr{ + { + "prefix": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + } + }else{ + if reqBody.Keyword != ""{ + should = []util.MapStr{ + { + "prefix": util.MapStr{ + "metadata.index_name": util.MapStr{ + "value": reqBody.Keyword, + "boost": 30, + }, + }, + }, + { + "prefix": util.MapStr{ + "metadata.aliases": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + "search_text": util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + { + "query_string": util.MapStr{ + "fields": []string{"*"}, + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "fuzzy_prefix_length": 2, + "fuzzy_max_expansions": 10, + "fuzzy_transpositions": true, + "allow_leading_wildcard": false, + }, + }, + } + } + } + + must := []interface{}{ + } + hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req) + if !hasAllPrivilege && len(indexPrivilege) == 0 { + h.WriteJSON(w, elastic.SearchResponse{ + + }, http.StatusOK) + return + } + if !hasAllPrivilege { + indexShould := make([]interface{}, 0, len(indexPrivilege)) + for clusterID, indices := range indexPrivilege { + var ( + wildcardIndices []string + normalIndices []string + ) + for _, index := range indices { + if strings.Contains(index,"*") { + wildcardIndices = append(wildcardIndices, index) + continue + } + normalIndices = append(normalIndices, index) + } + subShould := []util.MapStr{} + if len(wildcardIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "query_string": util.MapStr{ + "query": strings.Join(wildcardIndices, " "), + "fields": []string{"metadata.index_name"}, + "default_operator": "OR", + }, + }) + } + if len(normalIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "terms": util.MapStr{ + "metadata.index_name": normalIndices, + }, + }) + } + indexShould = append(indexShould, util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "wildcard": util.MapStr{ + "metadata.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": subShould, + }, + }, + }, + }, + }) + } + indexFilter := util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": indexShould, + }, + } + must = append(must, indexFilter) + } + boolQuery := util.MapStr{ + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_status": "deleted", + }, + }, + }, + "filter": filter, + "must": must, + } + if len(should) > 0 { + boolQuery["should"] = should + boolQuery["minimum_should_match"] = 1 + } + query := util.MapStr{ + "aggs": aggs, + "size": reqBody.Size, + "from": reqBody.From, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": boolQuery, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(reqBody.Sort) > 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.IndexConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + w.Write(util.MustToJSONBytes(response)) + +} +func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var indexIDs []interface{} + h.DecodeJSON(req, &indexIDs) + + if len(indexIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + // map indexIDs(cluster_id:index_name => cluster_uuid:indexName) + var ( + indexIDM = map[string]string{} + newIndexIDs []interface{} + clusterIndexNames = map[string][]string{} + ) + for _, indexID := range indexIDs { + if v, ok := indexID.(string); ok { + parts := strings.Split(v, ":") + if len(parts) != 2 { + log.Warnf("got wrong index_id: %s", v) + continue + } + clusterIndexNames[parts[0]] = append(clusterIndexNames[parts[0]], parts[1]) + } + } + for clusterID, indexNames := range clusterIndexNames { + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Warnf("get cluster uuid error: %v", err) + continue + } + for _, indexName := range indexNames { + newIndexID := fmt.Sprintf("%s:%s", clusterUUID, indexName) + newIndexIDs = append(newIndexIDs, newIndexID) + indexIDM[fmt.Sprintf("%s:%s", clusterID, indexName)] = newIndexID + } + } + q1 := orm.Query{WildcardIndex: true} + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "shard_stats"), + orm.In("metadata.labels.index_id", newIndexIDs), + ) + q1.Collapse("metadata.labels.shard_id") + q1.AddSort("timestamp", orm.DESC) + q1.Size = 20000 + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + } + + summaryMap := map[string]*ShardsSummary{} + for _, hit := range results.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardDocCount, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, hitM) + shardDocDeleted, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "deleted"}, hitM) + storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, hitM) + indexID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_id"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) + if v, ok := indexID.(string); ok { + if _, ok = summaryMap[v]; !ok { + summaryMap[v] = &ShardsSummary{} + } + indexInfo := summaryMap[v] + if iv, ok := indexName.(string); ok { + indexInfo.Index = iv + } + if count, ok := shardDocCount.(float64); ok && primary == true { + indexInfo.DocsCount += int64(count) + } + if deleted, ok := shardDocDeleted.(float64); ok && primary == true { + indexInfo.DocsDeleted += int64(deleted) + } + if storeSize, ok := storeInBytes.(float64); ok { + indexInfo.StoreInBytes += int64(storeSize) + if primary == true { + indexInfo.PriStoreInBytes += int64(storeSize) + } + } + if primary == true { + indexInfo.Shards++ + }else{ + indexInfo.Replicas++ + } + indexInfo.Timestamp = hitM["timestamp"] + } + } + } + + statusMetric, err := getIndexStatusOfRecentDay(indexIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, (15)) + if err != nil { + panic(err) + return + } + // 索引速率 + indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) + indexMetric.OnlyPrimary = true + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems := []GroupMetricItem{} + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing", + Field: "payload.elasticsearch.shard_stats.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "Indexing/s", + }) + queryMetric:=newMetricItem("search", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "search", + Field: "payload.elasticsearch.shard_stats.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "Search/s", + }) + + aggs:=map[string]interface{}{} + query :=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.index_id": newIndexIDs, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + sumAggs := util.MapStr{} + for _,metricItem:=range nodeMetricItems{ + leafAgg := util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + var sumBucketPath = "term_shard>"+ metricItem.ID + if metricItem.MetricItem.OnlyPrimary { + filterSubAggs := util.MapStr{ + metricItem.ID: leafAgg, + } + aggs["filter_pri"]=util.MapStr{ + "filter": util.MapStr{ + "term": util.MapStr{ + "payload.elasticsearch.shard_stats.routing.primary": util.MapStr{ + "value": true, + }, + }, + }, + "aggs": filterSubAggs, + } + sumBucketPath = "term_shard>filter_pri>"+ metricItem.ID + }else{ + aggs[metricItem.ID] = leafAgg + } + + sumAggs[metricItem.ID] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath, + }, + } + if metricItem.IsDerivative{ + sumAggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + } + } + sumAggs["term_shard"]= util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 10000, + }, + "aggs": aggs, + } + + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + panic(err) + } + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_id", + "size": 100, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":sumAggs, + }, + }, + }, + } + metrics := h.getMetrics(query, nodeMetricItems, bucketSize) + indexMetrics := map[string]util.MapStr{} + for key, item := range metrics { + for _, line := range item.Lines { + if _, ok := indexMetrics[line.Metric.Label]; !ok{ + indexMetrics[line.Metric.Label] = util.MapStr{ + } + } + indexMetrics[line.Metric.Label][key] = line.Data + } + } + infos := util.MapStr{} + for _, tempIndexID := range indexIDs { + result := util.MapStr{} + + indexID := tempIndexID.(string) + newIndexID := indexIDM[indexID] + + result["summary"] = summaryMap[newIndexID] + result["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Index Status", + "units": "day", + }, + "data": statusMetric[indexID], + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": indexMetrics[newIndexID]["indexing"], + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": indexMetrics[newIndexID]["search"], + }, + } + infos[indexID] = result + } + h.WriteJSON(w, infos, http.StatusOK) +} + +func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + if GetMonitorState(clusterID) == Console { + h.APIHandler.GetIndexInfo(w, req, ps) + return + } + indexID := ps.MustGetParameter("index") + parts := strings.Split(indexID, ":") + if len(parts) > 1 && !h.IsIndexAllowed(req, clusterID, parts[1]) { + h.WriteError(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return + } + if len(parts) < 2 { + h.WriteError(w, "invalid index id: "+ indexID, http.StatusInternalServerError) + return + } + + q := orm.Query{ + Size: 1, + } + q.Conds = orm.And(orm.Eq("metadata.index_name", parts[1]), orm.Eq("metadata.cluster_id", clusterID)) + q.AddSort("timestamp", orm.DESC) + + err, res := orm.Search(&elastic.IndexConfig{}, &q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + if len(response.Hits.Hits) == 0 { + log.Warnf("index [%v][%v] not found, may be you should wait several seconds", clusterID, indexID) + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + q1 := orm.Query{ + Size: 1000, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "shard_stats"), + orm.Eq("metadata.labels.index_name", parts[1]), + orm.Eq("metadata.labels.cluster_uuid", clusterUUID), + orm.Ge("timestamp", "now-15m"), + ) + q1.Collapse("metadata.labels.shard_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + summary := util.MapStr{} + hit := response.Hits.Hits[0].Source + var ( + shardsNum int + replicasNum int + indexInfo = util.MapStr{ + "index": parts[1], + } + ) + if aliases, ok := util.GetMapValueByKeys([]string{"metadata", "aliases"}, hit); ok { + health, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, hit) + indexUUID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_uuid"}, hit) + indexInfo["id"] = indexUUID + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, hit) + shards, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_shards"}, hit) + replicas, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_replicas"}, hit) + shardsNum, _ = util.ToInt(shards.(string)) + replicasNum, _ = util.ToInt(replicas.(string)) + summary["aliases"] = aliases + summary["timestamp"] = hit["timestamp"] + if state == "delete" { + health = "N/A" + } + indexInfo["health"] = health + indexInfo["status"] = state + } + if len(result.Result) > 0 { + shardSum := ShardsSummary{} + for _, row := range result.Result { + resultM, ok := row.(map[string]interface{}) + if ok { + primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, resultM) + storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, resultM) + if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, resultM); ok { + //summary["docs"] = docs + if v, ok := docs.(float64); ok && primary == true{ + shardSum.DocsCount += int64(v) + } + } + if storeSize, ok := storeInBytes.(float64); ok { + shardSum.StoreInBytes += int64(storeSize) + if primary == true { + shardSum.PriStoreInBytes += int64(storeSize) + } + } + if primary == true { + shardSum.Shards++ + }else{ + shardSum.Replicas++ + } + } + summary["timestamp"] = resultM["timestamp"] + } + indexInfo["docs_count"] = shardSum.DocsCount + indexInfo["pri_store_size"] = util.FormatBytes(float64(shardSum.PriStoreInBytes), 1) + indexInfo["store_size"] = util.FormatBytes(float64(shardSum.StoreInBytes), 1) + indexInfo["shards"] = shardSum.Shards + shardSum.Replicas + + summary["unassigned_shards"] = (replicasNum + 1) * shardsNum - shardSum.Shards - shardSum.Replicas + } + summary["index_info"] = indexInfo + + h.WriteJSON(w, summary, http.StatusOK) +} + +func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + if GetMonitorState(clusterID) == Console { + h.APIHandler.GetIndexShards(w, req, ps) + return + } + indexName := ps.MustGetParameter("index") + q1 := orm.Query{ + Size: 1000, + WildcardIndex: true, + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "shard_stats"), + orm.Eq("metadata.labels.index_name", indexName), + orm.Eq("metadata.labels.cluster_uuid", clusterUUID), + orm.Ge("timestamp", "now-15m"), + ) + q1.Collapse("metadata.labels.shard_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + if err != nil { + log.Error(err) + h.WriteError(w,err.Error(), http.StatusInternalServerError ) + return + } + var shards = []interface{}{} + if len(result.Result) > 0 { + q := &orm.Query{ + Size: 500, + } + q.Conds = orm.And( + orm.Eq("metadata.cluster_id", clusterID), + ) + err, nodesResult := orm.Search(elastic.NodeConfig{}, q) + if err != nil { + log.Error(err) + h.WriteError(w,err.Error(), http.StatusInternalServerError ) + return + } + nodeIDToName := util.MapStr{} + for _, row := range nodesResult.Result { + if rowM, ok := row.(map[string]interface{}); ok { + nodeName, _ := util.MapStr(rowM).GetValue("metadata.node_name") + nodeID, _ := util.MapStr(rowM).GetValue("metadata.node_id") + if v, ok := nodeID.(string); ok { + nodeIDToName[v] = nodeName + } + } + } + qps, err := h.getShardQPS(clusterID, "", indexName, 20) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + for _, item := range result.Result { + row, ok := item.(map[string]interface{}) + if ok { + shardInfo := util.MapStr{} + source := util.MapStr(row) + nodeID, _ := source.GetValue("metadata.labels.node_id") + if v, ok := nodeID.(string); ok { + if v, ok := nodeIDToName[v]; ok { + shardInfo["node"] = v + } + + } + shardV, err := source.GetValue("payload.elasticsearch.shard_stats") + if err != nil { + log.Error(err) + continue + } + shardInfo["id"], _ = source.GetValue("metadata.labels.node_id") + shardInfo["index"], _ = source.GetValue("metadata.labels.index_name") + shardInfo["ip"], _ = source.GetValue("metadata.labels.ip") + shardInfo["shard"], _ = source.GetValue("metadata.labels.shard") + shardInfo["shard_id"], _ = source.GetValue("metadata.labels.shard_id") + if v, ok := shardV.(map[string]interface{}); ok { + shardM := util.MapStr(v) + shardInfo["docs"], _ = shardM.GetValue("docs.count") + primary, _ := shardM.GetValue("routing.primary") + if primary == true { + shardInfo["prirep"] = "p" + }else{ + shardInfo["prirep"] = "r" + } + shardInfo["state"], _ = shardM.GetValue("routing.state") + shardInfo["store_in_bytes"], _ = shardM.GetValue("store.size_in_bytes") + } + if v, ok := shardInfo["shard_id"].(string); ok { + shardInfo["index_qps"] = qps[v]["index"] + shardInfo["query_qps"] = qps[v]["query"] + shardInfo["index_bytes_qps"] = qps[v]["index_bytes"] + } + shards = append(shards, shardInfo) + } + } + } + + h.WriteJSON(w, shards, http.StatusOK) +} + +func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + if GetMonitorState(clusterID) == Console { + h.APIHandler.GetSingleIndexMetrics(w, req, ps) + return + } + indexName := ps.MustGetParameter("index") + if !h.IsIndexAllowed(req, clusterID, indexName) { + h.WriteJSON(w, util.MapStr{ + "error": http.StatusText(http.StatusForbidden), + }, http.StatusForbidden) + return + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + shardID := h.GetParameterOrDefault(req, "shard_id", "") + + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }, + } + if shardID != "" { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "metadata.labels.shard_id": util.MapStr{ + "value": shardID, + }, + }, + }) + } + resBody := map[string]interface{}{} + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 60) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if bucketSize <= 60 { + min = min - int64(2 * bucketSize * 1000) + } + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem:=newMetricItem("index_throughput", 1, OperationGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + if shardID == "" { + metricItem.AddLine("Indexing Rate","Primary Indexing","Number of documents being indexed for node.","group1","payload.elasticsearch.shard_stats.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Deleting Rate","Primary Deleting","Number of documents being deleted for node.","group1","payload.elasticsearch.shard_stats.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.OnlyPrimary = true + metricItem.Lines[1].Metric.OnlyPrimary = true + }else{ + metricItem.AddLine("Indexing Rate","Indexing Rate","Number of documents being indexed for node.","group1","payload.elasticsearch.shard_stats.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Deleting Rate","Deleting Rate","Number of documents being deleted for node.","group1","payload.elasticsearch.shard_stats.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + } + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("search_throughput", 2, OperationGroupKey) + metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + metricItem.AddLine("Search Rate","Search Rate", + "Number of search requests being executed.", + "group1","payload.elasticsearch.shard_stats.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("index_latency", 3, LatencyGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + if shardID == "" { //index level + metricItem.AddLine("Indexing Latency","Primary Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.shard_stats.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Deleting Latency","Primary Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.shard_stats.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.OnlyPrimary = true + metricItem.Lines[1].Metric.OnlyPrimary = true + }else{ // shard level + metricItem.AddLine("Indexing Latency","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.shard_stats.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Deleting Latency","Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.shard_stats.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + } + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("search_latency", 4, LatencyGroupKey) + metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + + metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.shard_stats.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.shard_stats.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.shard_stats.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.shard_stats.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.shard_stats.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.shard_stats.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + metrics := h.getSingleIndexMetrics(metricItems,query, bucketSize) + shardStateMetric, err := h.getIndexShardsMetric(clusterID, indexName, min, max, bucketSize) + if err != nil { + log.Error(err) + } + metrics["shard_state"] = shardStateMetric + resBody["metrics"] = metrics + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) getIndexShardsMetric(id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "groups": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.routing.state", + "size": 10, + }, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("shard_state", 0, "") + metricItem.AddLine("Shard State","Shard State","","group1","payload.elasticsearch.shard_stats.routing.state","max",bucketSizeStr,"","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + metricData, err = parseGroupMetricData(response.Aggregations["dates"].Buckets, false) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func (h *APIHandler) getIndexHealthMetric(id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "groups": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.index_stats.index_info.health", + "size": 5, + }, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("index_health", 1, "") + metricItem.AddLine("health","Health","","group1","payload.elasticsearch.index_stats.index_info.health","max",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + metricData, err = parseGroupMetricData(response.Aggregations["dates"].Buckets, true) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + + +func getIndexStatusOfRecentDay(indexIDs []interface{})(map[string][]interface{}, error){ + q := orm.Query{ + WildcardIndex: true, + } + query := util.MapStr{ + "aggs": util.MapStr{ + "group_by_index_id": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_id", + "size": 100, + }, + "aggs": util.MapStr{ + "time_histogram": util.MapStr{ + "date_range": util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + },{ + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }, + "aggs": util.MapStr{ + "term_health": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.index_stats.index_info.health", + }, + }, + }, + }, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-15d", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.index_id": indexIDs, + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + return nil, err + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + recentStatus := map[string][]interface{}{} + for _, bk := range response.Aggregations["group_by_index_id"].Buckets { + indexKey := bk["key"].(string) + recentStatus[indexKey] = []interface{}{} + if histogramAgg, ok := bk["time_histogram"].(map[string]interface{}); ok { + if bks, ok := histogramAgg["buckets"].([]interface{}); ok { + for _, bkItem := range bks { + if bkVal, ok := bkItem.(map[string]interface{}); ok { + if termHealth, ok := bkVal["term_health"].(map[string]interface{}); ok { + if healthBks, ok := termHealth["buckets"].([]interface{}); ok { + if len(healthBks) == 0 { + continue + } + healthMap := map[string]int{} + status := "unknown" + for _, hbkItem := range healthBks { + if hitem, ok := hbkItem.(map[string]interface{}); ok { + healthMap[hitem["key"].(string)] = 1 + } + } + if _, ok = healthMap["red"]; ok { + status = "red" + }else if _, ok = healthMap["yellow"]; ok { + status = "yellow" + }else if _, ok = healthMap["green"]; ok { + status = "green" + } + recentStatus[indexKey] = append(recentStatus[indexKey], []interface{}{bkVal["key"], status}) + } + } + } + } + } + } + } + return recentStatus, nil +} + +func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string] interface{}{} + id := ps.ByName("id") + indexName := ps.ByName("index") + if !h.IsIndexAllowed(req, id, indexName) { + h.WriteJSON(w, util.MapStr{ + "error": http.StatusText(http.StatusForbidden), + }, http.StatusForbidden) + return + } + q := &orm.Query{ Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.labels.cluster_id", id), + orm.Eq("metadata.labels.index_name", indexName), + orm.Eq("metadata.name", "index_routing_table"), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + namesM := util.MapStr{} + if len(result.Result) > 0 { + if data, ok := result.Result[0].(map[string]interface{}); ok { + if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists { + if table, ok := routingTable.(map[string]interface{}); ok{ + if shardsM, ok := table["shards"].(map[string]interface{}); ok { + for _, rows := range shardsM { + if rowsArr, ok := rows.([]interface{}); ok { + for _, rowsInner := range rowsArr { + if rowsInnerM, ok := rowsInner.(map[string]interface{}); ok { + if v, ok := rowsInnerM["node"].(string); ok { + namesM[v] = true + } + } + } + } + + } + } + + } + } + } + } + + //node uuid + nodeIds := make([]interface{}, 0, len(namesM) ) + for name, _ := range namesM { + nodeIds = append(nodeIds, name) + } + + q1 := &orm.Query{ Size: 100} + q1.AddSort("timestamp", orm.DESC) + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + orm.In("metadata.node_id", nodeIds), + ) + err, result = orm.Search(elastic.NodeConfig{}, q1) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + nodes := []interface{}{} + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nodeId, _ := util.GetMapValueByKeys([]string{"metadata", "node_id"}, hitM) + nodeName, _ := util.GetMapValueByKeys([]string{"metadata", "node_name"}, hitM) + status, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "status"}, hitM) + ip, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "ip"}, hitM) + transportAddress, _ := util.GetMapValueByKeys([]string{"payload", "node_state", "transport_address"}, hitM) + var port string + if v, ok := transportAddress.(string); ok { + parts := strings.Split(v, ":") + if len(parts) > 1 { + port = parts[1] + } + } + + if v, ok := nodeId.(string); ok { + ninfo := util.MapStr{ + "id": v, + "name": nodeName, + "ip": ip, + "port": port, + "status": status, + "timestamp": hitM["timestamp"], + } + nodes = append(nodes, ninfo) + } + } + } + + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterIds := h.GetParameterOrDefault(req, "ids", "") + keyword := h.GetParameterOrDefault(req, "keyword", "") + ids := strings.Split(clusterIds, ",") + if len(ids) == 0 { + h.Error400(w, "cluster id is required") + return + } + var must = []util.MapStr{} + + if !util.StringInArray(ids, "*"){ + + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "metadata.cluster_id": ids, + }, + }) + } + + if keyword != "" { + must = append(must, util.MapStr{ + "wildcard":util.MapStr{ + "metadata.index_name": + util.MapStr{"value": fmt.Sprintf("*%s*", keyword)}, + }, + }) + } + var dsl = util.MapStr{ + "_source": []string{"metadata.index_name"}, + "collapse": util.MapStr{ + "field": "metadata.index_name", + }, + "size": 100, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": must, + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.state": util.MapStr{ + "value": "delete", + }, + }, + }, + }, + }, + }, + } + + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + indexName := orm.GetIndexName(elastic.IndexConfig{}) + resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl)) + if err != nil { + + return + } + list := resp.Hits.Hits + var indexNames []string + for _, v := range list { + m := v.Source["metadata"].(map[string]interface{}) + indexNames = append(indexNames, m["index_name"].(string)) + + } + m := make(map[string]interface{}) + m["indexnames"] = indexNames + h.WriteOKJSON(w, m) + + return +} diff --git a/modules/elastic/api/init.go b/modules/elastic/api/init.go new file mode 100644 index 00000000..7a135e0d --- /dev/null +++ b/modules/elastic/api/init.go @@ -0,0 +1,111 @@ +package api + +import ( + "infini.sh/console/core/security/enum" + "infini.sh/framework/core/api" +) + +var clusterAPI APIHandler + +func init() { + clusterAPI = APIHandler{} + + InitTestAPI() + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/health", clusterAPI.RequireClusterPermission(clusterAPI.GetClusterHealth)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleMetricsSummaryAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/cluster_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleClusterMetricsAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleNodeMetricsAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleIndexMetricsAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/queue_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleQueueMetricsAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/storage_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetStorageMetricAction)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/", clusterAPI.RequirePermission(clusterAPI.HandleCreateClusterAction, enum.PermissionElasticsearchClusterWrite)) + api.HandleAPIMethod(api.GET, "/elasticsearch/indices", clusterAPI.RequireLogin(clusterAPI.ListIndex)) + api.HandleAPIMethod(api.GET, "/elasticsearch/status", clusterAPI.RequireLogin(clusterAPI.GetClusterStatusAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetClusterAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleUpdateClusterAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleDeleteClusterAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.GET, "/elasticsearch/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchClusterAction, enum.PermissionElasticsearchClusterRead)) + api.HandleAPIMethod(api.POST, "/elasticsearch/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchClusterAction, enum.PermissionElasticsearchClusterRead)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/search_template", clusterAPI.HandleCreateSearchTemplateAction) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleUpdateSearchTemplateAction) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleDeleteSearchTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template", clusterAPI.HandleSearchSearchTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleGetSearchTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template_history/_search", clusterAPI.HandleSearchSearchTemplateHistoryAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_render/template", clusterAPI.HandleRenderTemplateAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_search/template", clusterAPI.HandleSearchTemplateAction) + + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/alias", clusterAPI.RequireClusterPermission(clusterAPI.HandleAliasAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/alias", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetAliasAction)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/saved_objects/view", clusterAPI.RequirePermission(clusterAPI.HandleCreateViewAction, enum.PermissionViewWrite)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/saved_objects/_find", clusterAPI.RequirePermission(clusterAPI.HandleGetViewListAction, enum.PermissionViewRead)) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/saved_objects/view/:view_id", clusterAPI.RequirePermission(clusterAPI.HandleDeleteViewAction, enum.PermissionViewWrite)) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/saved_objects/view/:view_id", clusterAPI.RequirePermission(clusterAPI.HandleUpdateViewAction, enum.PermissionViewWrite)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/internal/view-management/resolve_index/:wild", clusterAPI.RequireLogin(clusterAPI.HandleResolveIndexAction)) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/saved_objects/_bulk_get", clusterAPI.RequirePermission(clusterAPI.HandleBulkGetViewAction, enum.PermissionViewRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/view/_fields_for_wildcard", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetFieldCapsAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/saved_objects/view/:view_id", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetViewAction)) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/view/:view_id/_set_default_layout", clusterAPI.RequireClusterPermission(clusterAPI.SetDefaultLayout)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/search/ese", clusterAPI.RequireClusterPermission(clusterAPI.HandleEseSearchAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search/trace_id", clusterAPI.HandleTraceIDSearchAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/suggestions/values/:index", clusterAPI.RequireClusterPermission(clusterAPI.HandleValueSuggestionAction)) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/setting", clusterAPI.RequireClusterPermission(clusterAPI.HandleSettingAction)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/setting/:key", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetSettingAction)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_proxy", clusterAPI.RequireClusterPermission(clusterAPI.HandleProxyAction)) + + api.HandleAPIMethod(api.POST, "/elasticsearch/cluster/_search", clusterAPI.RequirePermission(clusterAPI.SearchClusterMetadata, enum.PermissionElasticsearchClusterRead)) + api.HandleAPIMethod(api.POST, "/elasticsearch/cluster/info", clusterAPI.RequirePermission(clusterAPI.FetchClusterInfo, enum.PermissionElasticsearchMetricRead)) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/info", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetClusterInfo, enum.PermissionElasticsearchMetricRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/node/_search", clusterAPI.RequirePermission(clusterAPI.SearchNodeMetadata, enum.PermissionElasticsearchNodeRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/nodes", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetClusterNodes, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/nodes/realtime", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetRealtimeClusterNodes, enum.PermissionElasticsearchMetricRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/node/info", clusterAPI.RequirePermission(clusterAPI.FetchNodeInfo, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/indices", clusterAPI.RequirePermission(clusterAPI.GetClusterIndices, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchIndexRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/indices/realtime", clusterAPI.RequireLogin(clusterAPI.GetRealtimeClusterIndices)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/shards", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetNodeShards, enum.PermissionElasticsearchMetricRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/info", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetNodeInfo, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/metrics", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.GetSingleNodeMetrics, enum.PermissionElasticsearchMetricRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/indices", clusterAPI.RequirePermission(clusterAPI.getNodeIndices, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchIndexRead)) + api.HandleAPIMethod(api.POST, "/elasticsearch/index/_search", clusterAPI.RequirePermission(clusterAPI.SearchIndexMetadata, enum.PermissionElasticsearchIndexRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index/:index/metrics", clusterAPI.RequirePermission(clusterAPI.GetSingleIndexMetrics, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index/:index/info", clusterAPI.RequirePermission(clusterAPI.GetIndexInfo, enum.PermissionElasticsearchIndexRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index/:index/shards", clusterAPI.RequirePermission(clusterAPI.GetIndexShards, enum.PermissionElasticsearchIndexRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index/:index/nodes", clusterAPI.RequirePermission(clusterAPI.getIndexNodes, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead)) + api.HandleAPIMethod(api.POST, "/elasticsearch/index/info", clusterAPI.RequirePermission(clusterAPI.FetchIndexInfo, enum.PermissionElasticsearchMetricRead)) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template", clusterAPI.HandleSearchTraceTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleGetTraceTemplateAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/trace_template", clusterAPI.HandleCrateTraceTemplateAction) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleSaveTraceTemplateAction) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleDeleteTraceTemplateAction) + + api.HandleAPIMethod(api.POST, "/elasticsearch/activity/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchActivityAction, enum.PermissionActivityRead)) + + api.HandleAPIMethod(api.GET, "/host/_discover", clusterAPI.getDiscoverHosts) + api.HandleAPIMethod(api.POST, "/host/_search", clusterAPI.SearchHostMetadata) + api.HandleAPIMethod(api.POST, "/host/info", clusterAPI.FetchHostInfo) + api.HandleAPIMethod(api.GET, "/host/:host_id/metrics", clusterAPI.GetSingleHostMetrics) + api.HandleAPIMethod(api.GET, "/host/:host_id/metric/_stats", clusterAPI.GetHostMetricStats) + api.HandleAPIMethod(api.GET, "/host/:host_id", clusterAPI.GetHostInfo) + api.HandleAPIMethod(api.PUT, "/host/:host_id", clusterAPI.updateHost) + api.HandleAPIMethod(api.GET, "/host/:host_id/info", clusterAPI.GetHostOverviewInfo) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_ilm/policy", clusterAPI.HandleGetILMPolicyAction) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.HandleSaveILMPolicyAction) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.HandleDeleteILMPolicyAction) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_template", clusterAPI.HandleGetTemplateAction) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_template/:template_name", clusterAPI.HandleSaveTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/shard/:shard_id/info", clusterAPI.RequirePermission(clusterAPI.GetShardInfo, enum.PermissionElasticsearchMetricRead)) + + api.HandleAPIMethod(api.GET, "/elasticsearch/metadata", clusterAPI.RequireLogin(clusterAPI.GetMetadata)) + api.HandleAPIMethod(api.GET, "/elasticsearch/hosts", clusterAPI.RequireLogin(clusterAPI.GetHosts)) + +} diff --git a/modules/elastic/api/manage.go b/modules/elastic/api/manage.go new file mode 100644 index 00000000..fa2075a8 --- /dev/null +++ b/modules/elastic/api/manage.go @@ -0,0 +1,1431 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/core" + v1 "infini.sh/console/modules/elastic/api/v1" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "math" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +type APIHandler struct { + core.Handler + v1.APIHandler +} + +func (h *APIHandler) Client() elastic.API { + return elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) +} + +func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var conf = &elastic.ElasticsearchConfig{} + err := h.DecodeJSON(req, conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + // TODO validate data format + conf.Enabled = true + conf.Host = strings.TrimSpace(conf.Host) + conf.Endpoint = fmt.Sprintf("%s://%s", conf.Schema, conf.Host) + conf.ID = util.GetUUID() + ctx := &orm.Context{ + Refresh: "wait_for", + } + if conf.CredentialID == "" && conf.BasicAuth != nil && conf.BasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(conf.Name+"_platform("+conf.ID+")", conf.BasicAuth) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + conf.CredentialID = credentialID + } + conf.BasicAuth = nil + + if conf.AgentCredentialID == "" && conf.AgentBasicAuth != nil && conf.AgentBasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(conf.Name+"_agent("+conf.ID+")", conf.AgentBasicAuth) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + conf.AgentCredentialID = credentialID + } + conf.AgentBasicAuth = nil + + if conf.Distribution == "" { + conf.Distribution = elastic.Elasticsearch + } + err = orm.Create(ctx, conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + basicAuth, err := common.GetBasicAuth(conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + conf.BasicAuth = basicAuth + conf.Source = elastic.ElasticsearchConfigSourceElasticsearch + _, err = common.InitElasticInstance(*conf) + if err != nil { + log.Warn("error on init elasticsearch:", err) + } + + h.WriteCreatedOKJSON(w, conf.ID) +} + +func saveBasicAuthToCredential(name string, auth *model.BasicAuth) (string, error) { + cred := credential.Credential{ + Name: name, + Type: credential.BasicAuth, + Tags: []string{"ES"}, + Payload: map[string]interface{}{ + "basic_auth": map[string]interface{}{ + "username": auth.Username, + "password": auth.Password.Get(), + }, + }, + } + cred.ID = util.GetUUID() + err := cred.Encode() + if err != nil { + return "", err + } + err = orm.Create(nil, &cred) + if err != nil { + return "", err + } + return cred.ID, nil +} + +func (h *APIHandler) HandleGetClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + clusterConf := elastic.ElasticsearchConfig{} + clusterConf.ID = id + exists, err := orm.Get(&clusterConf) + if err != nil || !exists { + log.Error(err) + h.Error404(w) + return + } + h.WriteGetOKJSON(w, id, clusterConf) +} + +func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var conf = map[string]interface{}{} + err := h.DecodeJSON(req, &conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + id := ps.MustGetParameter("id") + originConf := elastic.ElasticsearchConfig{} + originConf.ID = id + exists, err := orm.Get(&originConf) + if err != nil || !exists { + log.Error(err) + h.Error404(w) + return + } + buf := util.MustToJSONBytes(originConf) + source := map[string]interface{}{} + util.MustFromJSONBytes(buf, &source) + for k, v := range conf { + if k == "id" { + continue + } + if k == "basic_auth" { + if authMap, ok := v.(map[string]interface{}); ok { + if pwd, ok := authMap["password"]; !ok || (ok && pwd == "") { + if sourceM, ok := source[k].(map[string]interface{}); ok { + authMap["password"] = sourceM["password"] + } + } + } + } + source[k] = v + } + + if host, ok := conf["host"].(string); ok { + host = strings.TrimSpace(host) + if schema, ok := conf["schema"].(string); ok { + source["endpoint"] = fmt.Sprintf("%s://%s", schema, host) + source["host"] = host + } + } + + conf["updated"] = time.Now() + ctx := &orm.Context{ + Refresh: "wait_for", + } + confBytes, _ := json.Marshal(source) + newConf := &elastic.ElasticsearchConfig{} + json.Unmarshal(confBytes, newConf) + newConf.ID = id + + if conf["credential_id"] == nil { + if newConf.BasicAuth != nil && newConf.BasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(newConf.Name+"_platform("+newConf.ID+")", newConf.BasicAuth) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + newConf.CredentialID = credentialID + newConf.BasicAuth = nil + } else { + newConf.CredentialID = "" + } + } + + if conf["agent_credential_id"] == nil { + if newConf.AgentBasicAuth != nil && newConf.AgentBasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(newConf.Name+"_agent("+newConf.ID+")", newConf.AgentBasicAuth) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + newConf.AgentCredentialID = credentialID + newConf.AgentBasicAuth = nil + } else { + newConf.AgentCredentialID = "" + } + } + + err = orm.Update(ctx, newConf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + basicAuth, err := common.GetBasicAuth(newConf) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + newConf.BasicAuth = basicAuth + + //update config in heap + newConf.Source = elastic.ElasticsearchConfigSourceElasticsearch + _, err = common.InitElasticInstance(*newConf) + if err != nil { + log.Warn("error on init elasticsearch:", err) + } + + h.WriteUpdatedOKJSON(w, id) +} + +func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.MustGetParameter("id") + + esConfig := elastic.ElasticsearchConfig{} + esConfig.ID = id + ok, err := orm.Get(&esConfig) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if ok { + if esConfig.Reserved { + resBody["error"] = "this cluster is reserved" + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + } + ctx := &orm.Context{ + Refresh: "wait_for", + } + err = orm.Delete(ctx, &esConfig) + + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + delDsl := util.MapStr{ + "query": util.MapStr{ + "match": util.MapStr{ + "metadata.cluster_id": id, + }, + }, + } + err = orm.DeleteBy(elastic.NodeConfig{}, util.MustToJSONBytes(delDsl)) + if err != nil { + log.Error(err) + } + err = orm.DeleteBy(elastic.IndexConfig{}, util.MustToJSONBytes(delDsl)) + if err != nil { + log.Error(err) + } + + elastic.RemoveInstance(id) + elastic.RemoveHostsByClusterID(id) + h.WriteDeletedOKJSON(w, id) +} + +func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + name = h.GetParameterOrDefault(req, "name", "") + sortField = h.GetParameterOrDefault(req, "sort_field", "") + sortOrder = h.GetParameterOrDefault(req, "sort_order", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + if name != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name.text": "%s"}}`, name)) + } + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + if !hasAllPrivilege && clusterFilter == nil { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + if !hasAllPrivilege { + if mustBuilder.String() != "" { + mustBuilder.WriteString(",") + } + mustBuilder.Write(util.MustToJSONBytes(clusterFilter)) + } + + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + var sort = "" + if sortField != "" && sortOrder != "" { + sort = fmt.Sprintf(`,"sort":[{"%s":{"order":"%s"}}]`, sortField, sortOrder) + } + + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from, sort) + q := orm.Query{ + RawQuery: []byte(queryDSL), + } + err, result := orm.Search(elastic.ElasticsearchConfig{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + searchRes := elastic.SearchResponse{} + util.MustFromJSONBytes(result.Raw, &searchRes) + if len(searchRes.Hits.Hits) > 0 { + for _, hit := range searchRes.Hits.Hits { + if basicAuth, ok := hit.Source["basic_auth"]; ok { + if authMap, ok := basicAuth.(map[string]interface{}); ok { + delete(authMap, "password") + } + } + } + } + + h.WriteJSON(w, searchRes, http.StatusOK) +} + +func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.MustGetParameter("id") + + summary := map[string]interface{}{} + var query = util.MapStr{ + "sort": util.MapStr{ + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + "size": 1, + } + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + }, + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(query), + WildcardIndex: true, + } + err, result := orm.Search(event.Event{}, &q) + if err != nil { + resBody["error"] = err.Error() + log.Error("MetricsSummary search error: ", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if len(result.Result) > 0 { + if v, ok := result.Result[0].(map[string]interface{}); ok { + sourceMap := util.MapStr(v) + summary["timestamp"], _ = sourceMap.GetValue("timestamp") + status, _ := sourceMap.GetValue("payload.elasticsearch.cluster_stats") + statusMap := util.MapStr(status.(map[string]interface{})) + summary["cluster_name"], _ = statusMap.GetValue("cluster_name") + summary["status"], _ = statusMap.GetValue("status") + summary["indices_count"], _ = statusMap.GetValue("indices.count") + summary["total_shards"], _ = statusMap.GetValue("indices.shards.total") + summary["primary_shards"], _ = statusMap.GetValue("indices.shards.primaries") + summary["replication_shards"], _ = statusMap.GetValue("indices.shards.replication") + //summary["unassigned_shards"]=status.Indices["shards"].(map[string]interface{})["primaries"] + + summary["document_count"], _ = statusMap.GetValue("indices.docs.count") + summary["deleted_document_count"], _ = statusMap.GetValue("indices.docs.deleted") + + summary["used_store_bytes"], _ = statusMap.GetValue("indices.store.size_in_bytes") + + summary["max_store_bytes"], _ = statusMap.GetValue("nodes.fs.total_in_bytes") + summary["available_store_bytes"], _ = statusMap.GetValue("nodes.fs.available_in_bytes") + + summary["fielddata_bytes"], _ = statusMap.GetValue("indices.fielddata.memory_size_in_bytes") + summary["fielddata_evictions"], _ = statusMap.GetValue("indices.fielddata.evictions") + + summary["query_cache_bytes"], _ = statusMap.GetValue("indices.query_cache.memory_size_in_bytes") + summary["query_cache_total_count"], _ = statusMap.GetValue("indices.query_cache.total_count") + summary["query_cache_hit_count"], _ = statusMap.GetValue("indices.query_cache.hit_count") + summary["query_cache_miss_count"], _ = statusMap.GetValue("indices.query_cache.miss_count") + summary["query_cache_evictions"], _ = statusMap.GetValue("indices.query_cache.evictions") + + summary["segments_count"], _ = statusMap.GetValue("indices.segments.count") + summary["segments_memory_in_bytes"], _ = statusMap.GetValue("indices.segments.memory_in_bytes") + + summary["nodes_count"], _ = statusMap.GetValue("nodes.count.total") + summary["version"], _ = statusMap.GetValue("nodes.versions") + + summary["mem_total_in_bytes"], _ = statusMap.GetValue("nodes.os.mem.total_in_bytes") + summary["mem_used_in_bytes"], _ = statusMap.GetValue("nodes.os.mem.used_in_bytes") + summary["mem_used_percent"], _ = statusMap.GetValue("nodes.os.mem.used_percent") + + summary["uptime"], _ = statusMap.GetValue("nodes.jvm.max_uptime_in_millis") + summary["used_jvm_bytes"], _ = statusMap.GetValue("nodes.jvm.mem.heap_used_in_bytes") + summary["max_jvm_bytes"], _ = statusMap.GetValue("nodes.jvm.mem.heap_max_in_bytes") + } + } + + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + err, result = orm.Search(event.Event{}, &q) + if err != nil { + log.Error("MetricsSummary search error: ", err) + } else { + if len(result.Result) > 0 { + if v, ok := result.Result[0].(map[string]interface{}); ok { + health, _ := util.MapStr(v).GetValue("payload.elasticsearch.cluster_health") + healthMap := util.MapStr(health.(map[string]interface{})) + summary["unassigned_shards"], _ = healthMap.GetValue("unassigned_shards") + } + } + } + + resBody["summary"] = summary + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +// new +func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + if GetMonitorState(id) == Console { + h.APIHandler.HandleClusterMetricsAction(w, req, ps) + return + } + + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + panic(err) + return + } + meta := elastic.GetMetadata(id) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.IndexStats.Enabled && meta.Config.MonitorConfigs.IndexStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.IndexStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + + //fmt.Println(min," vs ",max,",",rangeFrom,rangeTo,"range hours:",hours) + + //metrics:=h.GetClusterMetrics(id,bucketSize,min,max) + isOverview := h.GetIntOrDefault(req, "overview", 0) + var metrics interface{} + if bucketSize <= 60 { + min = min - int64(2*bucketSize*1000) + } + if isOverview == 1 { + metrics = h.GetClusterIndexMetrics(id, bucketSize, min, max) + } else { + metrics = h.GetClusterMetrics(id, bucketSize, min, max) + } + + resBody["metrics"] = metrics + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } + +} + +func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + nodeName := h.Get(req, "node_name", "") + top := h.GetIntOrDefault(req, "top", 5) + if bucketSize <= 60 { + min = min - int64(2*bucketSize*1000) + } + resBody["metrics"], err = h.getNodeMetrics(id, bucketSize, min, max, nodeName, top) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + if GetMonitorState(id) == Console { + h.APIHandler.HandleIndexMetricsAction(w, req, ps) + return + } + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + indexName := h.Get(req, "index_name", "") + top := h.GetIntOrDefault(req, "top", 5) + shardID := h.Get(req, "shard_id", "") + if bucketSize <= 60 { + min = min - int64(2*bucketSize*1000) + } + metrics, err := h.getIndexMetrics(req, id, bucketSize, min, max, indexName, top, shardID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 { + metricA := metrics["doc_count"] + metricB := metrics["docs_deleted"] + if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok { + if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok { + data := make([]map[string]interface{}, 0, len(dataA)*2) + var ( + x1 float64 + x2 float64 + ) + for i := 0; i < len(dataA); i++ { + x1 = dataA[i][1].(float64) + x2 = dataB[i][1].(float64) + if x1+x2 == 0 { + continue + } + data = append(data, map[string]interface{}{ + "x": dataA[i][0], + "y": x1 / (x1 + x2) * 100, + "g": "Doc Count", + }) + data = append(data, map[string]interface{}{ + "x": dataA[i][0], + "y": x2 / (x1 + x2) * 100, + "g": "Doc Deleted", + }) + } + metricDocPercent := &common.MetricItem{ + Axis: []*common.MetricAxis{}, + Key: "doc_percent", + Group: metricA.Group, + Order: 18, + Lines: []*common.MetricLine{ + { + TimeRange: metricA.Lines[0].TimeRange, + Data: data, + Type: common.GraphTypeBar, + }, + }, + } + metrics["doc_percent"] = metricDocPercent + } + } + + } + resBody["metrics"] = metrics + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top index may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} +func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + nodeName := h.Get(req, "node_name", "") + top := h.GetIntOrDefault(req, "top", 5) + if bucketSize <= 60 { + min = min - int64(2*bucketSize*1000) + } + resBody["metrics"], err = h.getThreadPoolMetrics(id, bucketSize, min, max, nodeName, top) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +// TODO, use expired hash +var clusters = map[string]elastic.ElasticsearchConfig{} +var clustersMutex = &sync.RWMutex{} + +// TODO use prefered client +func (h *APIHandler) GetClusterClient(id string) (bool, elastic.API, error) { + clustersMutex.RLock() + config, ok := clusters[id] + clustersMutex.RUnlock() + + var client elastic.API + + if !ok { + client = elastic.GetClientNoPanic(id) + } + + if client == nil { + indexName := orm.GetIndexName(elastic.ElasticsearchConfig{}) + getResponse, err := h.Client().Get(indexName, "", id) + if err != nil { + return false, nil, err + } + + bytes := util.MustToJSONBytes(getResponse.Source) + cfg := elastic.ElasticsearchConfig{} + err = util.FromJSONBytes(bytes, &cfg) + if err != nil { + return false, nil, err + } + + if getResponse.StatusCode == http.StatusNotFound { + return false, nil, err + } + + cfg.ID = id + clustersMutex.Lock() + clusters[id] = cfg + clustersMutex.Unlock() + config = cfg + + client, _ = common.InitClientWithConfig(config) + + } + + return true, client, nil +} + +func (h *APIHandler) GetClusterHealth(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + exists, client, err := h.GetClusterClient(id) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", id) + log.Warn(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + health, _ := client.ClusterHealth(context.Background()) + + h.WriteJSON(w, health, 200) +} + +func (h *APIHandler) HandleGetNodesAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + metaData := elastic.GetMetadata(id) + result := util.MapStr{} + if metaData == nil || metaData.Nodes == nil { + h.WriteError(w, "nodes metadata not found", 500) + return + } + for k, nodeInfo := range *metaData.Nodes { + result[k] = util.MapStr{ + "name": nodeInfo.Name, + "transport_address": nodeInfo.TransportAddress, + } + } + h.WriteJSON(w, result, 200) +} + +const ( + SystemGroupKey = "system" + OperationGroupKey = "operations" + LatencyGroupKey = "latency" + CacheGroupKey = "cache" + HttpGroupKey = "http" + MemoryGroupKey = "memory" + StorageGroupKey = "storage" + JVMGroupKey = "JVM" + TransportGroupKey = "transport" + DocumentGroupKey = "document" + IOGroupKey = "io" + CircuitBreakerGroupKey = "circuit_breaker" +) + +func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem { + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterMetricItems := []*common.MetricItem{} + metricItem := newMetricItem("cluster_storage", 8, StorageGroupKey) + metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true) + + metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("cluster_documents", 4, StorageGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("cluster_indices", 6, StorageGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("node_count", 5, MemoryGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + meta := elastic.GetMetadata(id) + if meta == nil { + err := fmt.Errorf("metadata of cluster [%s] is not found", id) + panic(err) + } + majorVersion := meta.GetMajorVersion() + + metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + + //TODO check version difference + if majorVersion < 5 { + metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + } else { + metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + } + + clusterMetricItems = append(clusterMetricItems, metricItem) + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + //todo: since there is four queries, we can change these query to async + indexMetricsResult := h.GetClusterIndexMetrics(id, bucketSize, min, max) + clusterMetricsResult := h.getSingleMetrics(clusterMetricItems, query, bucketSize) + for k, v := range clusterMetricsResult { + indexMetricsResult[k] = v + } + statusMetric, err := h.getClusterStatusMetric(id, min, max, bucketSize) + if err == nil { + indexMetricsResult["cluster_health"] = statusMetric + } else { + log.Error("get cluster status metric error: ", err) + } + clusterHealthMetricsResult := h.getShardsMetric(id, min, max, bucketSize) + for k, v := range clusterHealthMetricsResult { + indexMetricsResult[k] = v + } + // get CircuitBreaker metric + circuitBreakerMetricsResult := h.getCircuitBreakerMetric(id, min, max, bucketSize) + for k, v := range circuitBreakerMetricsResult { + indexMetricsResult[k] = v + } + + return indexMetricsResult +} +func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem := newMetricItem("index_throughput", 2, OperationGroupKey) + metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true) + metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("search_throughput", 2, OperationGroupKey) + metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Search Rate", "Total Query", + "Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!", + "group1", "payload.elasticsearch.node_stats.indices.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("index_latency", 3, LatencyGroupKey) + metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true) + + metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("search_latency", 3, LatencyGroupKey) + metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + + metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItems = append(metricItems, metricItem) + query := map[string]interface{}{} + clusterUUID, err := adapter.GetClusterUUID(id) + if err != nil { + panic(err) + } + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + return h.getSingleIndexMetricsByNodeStats(metricItems, query, bucketSize) +} + +func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + "interval": bucketSizeStr, + }, + }, + }, + } + metricItem := newMetricItem("shard_count", 7, StorageGroupKey) + metricItem.AddAxi("counts", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Active Primary Shards", "Active Primary Shards", "", "group1", "payload.elasticsearch.cluster_health.active_primary_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Active Shards", "Active Shards", "", "group1", "payload.elasticsearch.cluster_health.active_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Relocating Shards", "Relocating Shards", "", "group1", "payload.elasticsearch.cluster_health.relocating_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Initializing Shards", "Initializing Shards", "", "group1", "payload.elasticsearch.cluster_health.initializing_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Unassigned Shards", "Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Delayed Unassigned Shards", "Delayed Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.delayed_unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + var clusterHealthMetrics []*common.MetricItem + clusterHealthMetrics = append(clusterHealthMetrics, metricItem) + return h.getSingleMetrics(clusterHealthMetrics, query, bucketSize) +} + +func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + "interval": bucketSizeStr, + }, + }, + }, + } + metricItem := newMetricItem("circuit_breaker", 7, StorageGroupKey) + metricItem.AddAxi("Circuit Breaker", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Parent Breaker Tripped", "Parent Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.parent.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Fieldaata Breaker Tripped", "Fielddata Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.fielddata.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Accounting Breaker Tripped", "Accounting Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.accounting.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Request Breaker Tripped", "Request Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.request.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("In Flight Requests Breaker Tripped", "In Flight Requests Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + var circuitBreakerMetrics []*common.MetricItem + circuitBreakerMetrics = append(circuitBreakerMetrics, metricItem) + return h.getSingleMetrics(circuitBreakerMetrics, query, bucketSize) +} + +func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSize int) (*common.MetricItem, error) { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "groups": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.cluster_stats.status", + "size": 5, + }, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + metricData := []interface{}{} + metricItem := newMetricItem("cluster_health", 1, MemoryGroupKey) + metricItem.AddLine("status", "Status", "", "group1", "payload.elasticsearch.cluster_stats.status", "max", bucketSizeStr, "%", "ratio", "0.[00]", "0.[00]", false, false) + + if response.StatusCode == 200 { + metricData, err = parseGroupMetricData(response.Aggregations["dates"].Buckets, true) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func (h *APIHandler) GetClusterStatusAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var status = map[string]interface{}{} + clusterIDs, hasAllPrivilege := h.GetAllowedClusters(req) + if !hasAllPrivilege && len(clusterIDs) == 0 { + h.WriteJSON(w, status, http.StatusOK) + return + } + + elastic.WalkConfigs(func(k, value interface{}) bool { + key := k.(string) + if !hasAllPrivilege && !util.StringInArray(clusterIDs, key) { + return true + } + cfg, ok := value.(*elastic.ElasticsearchConfig) + if ok && cfg != nil { + meta := elastic.GetOrInitMetadata(cfg) + status[key] = map[string]interface{}{ + "health": meta.Health, + "available": meta.IsAvailable(), + "config": map[string]interface{}{ + "monitored": meta.Config.Monitored, + }, + } + } + return true + }) + h.WriteJSON(w, status, http.StatusOK) +} + +func (h *APIHandler) GetMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + clusterIDs, hasAllPrivilege := h.GetAllowedClusters(req) + if !hasAllPrivilege && len(clusterIDs) == 0 { + h.WriteJSON(w, result, http.StatusOK) + return + } + + elastic.WalkMetadata(func(key, value interface{}) bool { + m := util.MapStr{} + k := key.(string) + if !hasAllPrivilege && !util.StringInArray(clusterIDs, k) { + return true + } + if value == nil { + return true + } + + v, ok := value.(*elastic.ElasticsearchMetadata) + if ok { + m["major_version"] = v.GetMajorVersion() + m["seed_hosts"] = v.GetSeedHosts() + m["state"] = v.ClusterState + m["topology_version"] = v.NodesTopologyVersion + m["nodes"] = v.Nodes + //m["indices"]=v.Indices + m["health"] = v.Health + m["aliases"] = v.Aliases + //m["primary_shards"]=v.PrimaryShards + m["available"] = v.IsAvailable() + m["schema"] = v.GetSchema() + m["config"] = v.Config + m["last_success"] = v.LastSuccess() + result[k] = m + } + return true + }) + + h.WriteJSON(w, result, http.StatusOK) + +} + +func (h *APIHandler) GetMetadataByID(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + + id := ps.MustGetParameter("id") + + v := elastic.GetMetadata(id) + m := util.MapStr{} + if v != nil { + m["major_version"] = v.GetMajorVersion() + m["seed_hosts"] = v.GetSeedHosts() + m["state"] = v.ClusterState + m["topology_version"] = v.NodesTopologyVersion + m["nodes"] = v.Nodes + //m["indices"]=v.Indices + m["health"] = v.Health + m["aliases"] = v.Aliases + //m["primary_shards"]=v.PrimaryShards + m["available"] = v.IsAvailable() + m["schema"] = v.GetSchema() + m["config"] = v.Config + m["last_success"] = v.LastSuccess() + result[id] = m + } + + h.WriteJSON(w, result, http.StatusOK) + +} + +func (h *APIHandler) GetHosts(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + + elastic.WalkHosts(func(key, value interface{}) bool { + k := key.(string) + if value == nil { + return true + } + + v, ok := value.(*elastic.NodeAvailable) + if ok { + result[k] = util.MapStr{ + "host": v.Host, + "available": v.IsAvailable(), + "dead": v.IsDead(), + "last_check": v.LastCheck(), + "last_success": v.LastSuccess(), + "failure_tickets": v.FailureTickets(), + } + } + return true + }) + + h.WriteJSON(w, result, http.StatusOK) + +} + +func getAllMetricsIndex() string { + return orm.GetWildcardIndexName(event.Event{}) +} + +func (h *APIHandler) HandleGetStorageMetricAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := util.MapStr{} + clusterID := ps.ByName("id") + client := elastic.GetClient(clusterID) + shardRes, err := client.CatShards() + if err != nil { + resBody["error"] = fmt.Sprintf("cat shards error: %v", err) + log.Errorf("cat shards error: %v", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var metricData = TreeMapNode{ + Name: fmt.Sprintf("%s:Storage", clusterID), + SubKeys: map[string]int{}, + } + for _, shardInfo := range shardRes { + if shardInfo.ShardType != "p" { + continue + } + nodeName := fmt.Sprintf("%s:%s", shardInfo.NodeIP, shardInfo.NodeName) + //node level + if _, ok := metricData.SubKeys[nodeName]; !ok { + metricData.Children = append(metricData.Children, &TreeMapNode{ + Name: nodeName, + SubKeys: map[string]int{}, + }) + metricData.SubKeys[nodeName] = len(metricData.Children) - 1 + } + //index level + nodeIdx := metricData.SubKeys[nodeName] + if _, ok := metricData.Children[nodeIdx].SubKeys[shardInfo.Index]; !ok { + metricData.Children[nodeIdx].Children = append(metricData.Children[nodeIdx].Children, &TreeMapNode{ + Name: shardInfo.Index, + SubKeys: map[string]int{}, + }) + metricData.Children[nodeIdx].SubKeys[shardInfo.Index] = len(metricData.Children[nodeIdx].Children) - 1 + } + //shard level + indexIdx := metricData.Children[nodeIdx].SubKeys[shardInfo.Index] + value, err := util.ConvertBytesFromString(shardInfo.Store) + if err != nil { + log.Warn(err) + } + metricData.Children[nodeIdx].Children[indexIdx].Children = append(metricData.Children[nodeIdx].Children[indexIdx].Children, &TreeMapNode{ + Name: fmt.Sprintf("shard %s", shardInfo.ShardID), + Value: value, + }) + } + var ( + totalStoreSize float64 = 0 + nodeSize float64 = 0 + indexSize float64 = 0 + ) + for _, node := range metricData.Children { + nodeSize = 0 + for _, index := range node.Children { + indexSize = 0 + for _, shard := range index.Children { + indexSize += shard.Value + } + index.Value = math.Trunc(indexSize*100) / 100 + nodeSize += indexSize + } + node.Value = math.Trunc(nodeSize*100) / 100 + totalStoreSize += nodeSize + } + metricData.Value = math.Trunc(totalStoreSize*100) / 100 + h.WriteJSON(w, metricData, http.StatusOK) +} + +func getDateHistogramIntervalField(clusterID string, bucketSize string) (string, error) { + esClient := elastic.GetClient(clusterID) + ver := esClient.GetVersion() + return elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSize) +} diff --git a/modules/elastic/api/metrics_util.go b/modules/elastic/api/metrics_util.go new file mode 100644 index 00000000..c305936c --- /dev/null +++ b/modules/elastic/api/metrics_util.go @@ -0,0 +1,1204 @@ +package api + +import ( + "fmt" + "infini.sh/framework/core/env" + "net/http" + "strings" + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" +) + +func newMetricItem(metricKey string, order int, group string) *common.MetricItem { + metricItem := common.MetricItem{ + Order: order, + Key: metricKey, + Group: group, + } + + //axis + metricItem.Axis = []*common.MetricAxis{} + + //lines + metricItem.Lines = []*common.MetricLine{} + + return &metricItem +} + +type GroupMetricItem struct { + Key string + Field string + ID string + IsDerivative bool + Units string + FormatType string + MetricItem *common.MetricItem + Field2 string + Calc func(value, value2 float64) float64 +} + +type TreeMapNode struct { + Name string `json:"name"` + Value float64 `json:"value,omitempty"` + Children []*TreeMapNode `json:"children,omitempty"` + SubKeys map[string]int `json:"-"` +} + +type MetricData map[string][][]interface{} + +func generateGroupAggs(nodeMetricItems []GroupMetricItem) map[string]interface{} { + aggs := map[string]interface{}{} + + for _, metricItem := range nodeMetricItems { + aggs[metricItem.ID] = util.MapStr{ + "max": util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID+"_field2"] = util.MapStr{ + "max": util.MapStr{ + "field": metricItem.Field2, + }, + } + } + + if metricItem.IsDerivative { + aggs[metricItem.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + return aggs +} + +func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + grpMetricItemsIndex := map[string]int{} + for i, item := range grpMetricItems { + grpMetricItemsIndex[item.ID] = i + } + grpMetricData := map[string]MetricData{} + + var minDate, maxDate int64 + if response.StatusCode == 200 { + if nodeAgg, ok := response.Aggregations["group_by_level"]; ok { + for _, bucket := range nodeAgg.Buckets { + grpKey := bucket["key"].(string) + for _, metricItem := range grpMetricItems { + metricItem.MetricItem.AddLine(metricItem.Key, grpKey, "", "group1", metricItem.Field, "max", bucketSizeStr, metricItem.Units, metricItem.FormatType, "0.[00]", "0.[00]", false, false) + dataKey := metricItem.ID + if metricItem.IsDerivative { + dataKey = dataKey + "_deriv" + } + if _, ok := grpMetricData[dataKey]; !ok { + grpMetricData[dataKey] = map[string][][]interface{}{} + } + grpMetricData[dataKey][grpKey] = [][]interface{}{} + } + if datesAgg, ok := bucket["dates"].(map[string]interface{}); ok { + if datesBuckets, ok := datesAgg["buckets"].([]interface{}); ok { + for _, dateBucket := range datesBuckets { + if bucketMap, ok := dateBucket.(map[string]interface{}); ok { + v, ok := bucketMap["key"].(float64) + if !ok { + panic("invalid bucket key") + } + dateTime := (int64(v)) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + + for mk1, mv1 := range grpMetricData { + v1, ok := bucketMap[mk1] + if ok { + v2, ok := v1.(map[string]interface{}) + if ok { + v3, ok := v2["value"].(float64) + if ok { + metricID := mk1 + if strings.HasSuffix(mk1, "_deriv") { + metricID = strings.TrimSuffix(mk1, "_deriv") + if _, ok := bucketMap[mk1+"_field2"]; !ok { + v3 = v3 / float64(bucketSize) + } + } + if field2, ok := bucketMap[mk1+"_field2"]; ok { + if idx, ok := grpMetricItemsIndex[metricID]; ok { + if field2Map, ok := field2.(map[string]interface{}); ok { + v4 := field2Map["value"].(float64) + if v4 == 0 { + v3 = 0 + } else { + v3 = grpMetricItems[idx].Calc(v3, v4) + } + } + } + } + if v3 < 0 { + continue + } + points := []interface{}{dateTime, v3} + mv1[grpKey] = append(mv1[grpKey], points) + } + } + } + } + } + } + } + + } + } + } + } + + result := map[string]*common.MetricItem{} + + for _, metricItem := range grpMetricItems { + for _, line := range metricItem.MetricItem.Lines { + line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + dataKey := metricItem.ID + if metricItem.IsDerivative { + dataKey = dataKey + "_deriv" + } + line.Data = grpMetricData[dataKey][line.Metric.Label] + if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { + // remove first metric dot + temp := v[1:] + // // remove first last dot + if len(temp) > 0 { + temp = temp[0: len(temp)-1] + } + line.Data = temp + } + } + result[metricItem.Key] = metricItem.MetricItem + } + return result +} + +func GetMinBucketSize() int { + metricsCfg := struct { + MinBucketSizeInSeconds int `config:"min_bucket_size_in_seconds"` + }{ + MinBucketSizeInSeconds: 20, + } + _, _ = env.ParseConfig("insight", &metricsCfg) + if metricsCfg.MinBucketSizeInSeconds < 20 { + metricsCfg.MinBucketSizeInSeconds = 20 + } + return metricsCfg.MinBucketSizeInSeconds +} + +// defaultBucketSize 也就是每次聚合的时间间隔 +func (h *APIHandler) getMetricRangeAndBucketSize(req *http.Request, defaultBucketSize, defaultMetricCount int) (int, int64, int64, error) { + minBucketSizeInSeconds := GetMinBucketSize() + if defaultBucketSize <= 0 { + defaultBucketSize = minBucketSizeInSeconds + } + if defaultMetricCount <= 0 { + defaultMetricCount = 15 * 60 + } + bucketSize := defaultBucketSize + + bucketSizeStr := h.GetParameterOrDefault(req, "bucket_size", "") //默认 10，每个 bucket 的时间范围，单位秒 + if bucketSizeStr != "" { + du, err := util.ParseDuration(bucketSizeStr) + if err != nil { + return 0, 0, 0, err + } + bucketSize = int(du.Seconds()) + }else { + bucketSize = 0 + } + metricCount := h.GetIntOrDefault(req, "metric_count", defaultMetricCount) //默认 15分钟的区间，每分钟15个指标，也就是 15*6 个 bucket //90 + //min,max are unix nanoseconds + + minStr := h.Get(req, "min", "") + maxStr := h.Get(req, "max", "") + + return GetMetricRangeAndBucketSize(minStr, maxStr, bucketSize, metricCount) +} + +func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, metricCount int) (int, int64, int64, error) { + var min, max int64 + var rangeFrom, rangeTo time.Time + var err error + var useMinMax = bucketSize == 0 + now := time.Now() + if minStr == "" { + rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + } else { + //try 2021-08-21T14:06:04.818Z + rangeFrom, err = util.ParseStandardTime(minStr) + if err != nil { + //try 1629637500000 + v, err := util.ToInt64(minStr) + if err != nil { + log.Error("invalid timestamp:", minStr, err) + rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + } else { + rangeFrom = util.FromUnixTimestamp(v / 1000) + } + } + } + + if maxStr == "" { + rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + } else { + rangeTo, err = util.ParseStandardTime(maxStr) + if err != nil { + v, err := util.ToInt64(maxStr) + if err != nil { + log.Error("invalid timestamp:", maxStr, err) + rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + } else { + rangeTo = util.FromUnixTimestamp(int64(v) / 1000) + } + } + } + + min = rangeFrom.UnixNano() / 1e6 + max = rangeTo.UnixNano() / 1e6 + hours := rangeTo.Sub(rangeFrom).Hours() + + if useMinMax { + + if hours <= 0.25 { + bucketSize = GetMinBucketSize() + } else if hours <= 0.5 { + bucketSize = 30 + } else if hours <= 2 { + bucketSize = 60 + } else if hours < 3 { + bucketSize = 90 + } else if hours < 6 { + bucketSize = 120 + } else if hours < 12 { + bucketSize = 60 * 3 + } else if hours < 25 { //1day + bucketSize = 60 * 5 * 2 + } else if hours <= 7*24+1 { //7days + bucketSize = 60 * 15 * 2 + } else if hours <= 15*24+1 { //15days + bucketSize = 60 * 30 * 2 + } else if hours < 30*24+1 { //<30 days + bucketSize = 60 * 60 //hourly + } else if hours <= 30*24+1 { //<30days + bucketSize = 12 * 60 * 60 //half daily + } else if hours >= 30*24+1 { //>30days + bucketSize = 60 * 60 * 24 //daily bucket + } + } + + return bucketSize, min, max, nil +} + +// 获取单个指标，可以包含多条曲线 +func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem { + metricData := map[string][][]interface{}{} + + aggs := map[string]interface{}{} + metricItemsMap := map[string]*common.MetricLine{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + metricItemsMap[line.Metric.GetDataKey()] = line + metricData[line.Metric.GetDataKey()] = [][]interface{}{} + + aggs[line.Metric.ID] = util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field, + }, + } + if line.Metric.Field2 != "" { + aggs[line.Metric.ID+"_field2"] = util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field2, + }, + } + } + + if line.Metric.IsDerivative { + //add which metric keys to extract + aggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + if line.Metric.Field2 != "" { + aggs[line.Metric.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID + "_field2", + }, + } + } + } + } + } + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + query["size"] = 0 + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": aggs, + }, + } + response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + + var minDate, maxDate int64 + if response.StatusCode == 200 { + for _, v := range response.Aggregations { + for _, bucket := range v.Buckets { + v, ok := bucket["key"].(float64) + if !ok { + panic("invalid bucket key") + } + dateTime := (int64(v)) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + for mk1, mv1 := range metricData { + v1, ok := bucket[mk1] + if ok { + v2, ok := v1.(map[string]interface{}) + if ok { + v3, ok := v2["value"].(float64) + if ok { + if strings.HasSuffix(mk1, "_deriv") { + if _, ok := bucket[mk1+"_field2"]; !ok { + v3 = v3 / float64(bucketSize) + } + } + if field2, ok := bucket[mk1+"_field2"]; ok { + if line, ok := metricItemsMap[mk1]; ok { + if field2Map, ok := field2.(map[string]interface{}); ok { + v4 := field2Map["value"].(float64) + if v4 == 0 { + v3 = 0 + } else { + v3 = line.Metric.Calc(v3, v4) + } + } + } + } + if v3 < 0 { + continue + } + points := []interface{}{dateTime, v3} + metricData[mk1] = append(mv1, points) + } + + } + } + } + } + } + } + + result := map[string]*common.MetricItem{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + line.Data = metricData[line.Metric.GetDataKey()] + if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { + // remove first metric dot + temp := v[1:] + // // remove first last dot + if len(temp) > 0 { + temp = temp[0: len(temp)-1] + } + line.Data = temp + } + } + result[metricItem.Key] = metricItem + } + + return result +} + +//func (h *APIHandler) executeQuery(query map[string]interface{}, bucketItems *[]common.BucketItem, bucketSize int) map[string]*common.MetricItem { +// response, err := elastic.GetClient(h.Config.Elasticsearch).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) +// +//} + +func (h *APIHandler) getBucketMetrics(query map[string]interface{}, bucketItems *[]common.BucketItem, bucketSize int) map[string]*common.MetricItem { + //bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + //grpMetricItemsIndex := map[string]int{} + for _, item := range *bucketItems { + //grpMetricItemsIndex[item.Key] = i + + agg, ok := response.Aggregations[item.Key] + if ok { + fmt.Println(len(agg.Buckets)) + } + + } + //grpMetricData := map[string]MetricData{} + + //var minDate, maxDate int64 + //if response.StatusCode == 200 { + // if nodeAgg, ok := response.Aggregations["group_by_level"]; ok { + // for _, bucket := range nodeAgg.Buckets { + // grpKey := bucket["key"].(string) + // for _, metricItem := range *bucketItems { + // metricItem.MetricItem.AddLine(metricItem.Key, grpKey, "", "group1", metricItem.Field, "max", bucketSizeStr, metricItem.Units, metricItem.FormatType, "0.[00]", "0.[00]", false, false) + // dataKey := metricItem.Key + // if metricItem.IsDerivative { + // dataKey = dataKey + "_deriv" + // } + // if _, ok := grpMetricData[dataKey]; !ok { + // grpMetricData[dataKey] = map[string][][]interface{}{} + // } + // grpMetricData[dataKey][grpKey] = [][]interface{}{} + // } + // if datesAgg, ok := bucket["dates"].(map[string]interface{}); ok { + // if datesBuckets, ok := datesAgg["buckets"].([]interface{}); ok { + // for _, dateBucket := range datesBuckets { + // if bucketMap, ok := dateBucket.(map[string]interface{}); ok { + // v, ok := bucketMap["key"].(float64) + // if !ok { + // panic("invalid bucket key") + // } + // dateTime := (int64(v)) + // minDate = util.MinInt64(minDate, dateTime) + // maxDate = util.MaxInt64(maxDate, dateTime) + // + // for mk1, mv1 := range grpMetricData { + // v1, ok := bucketMap[mk1] + // if ok { + // v2, ok := v1.(map[string]interface{}) + // if ok { + // v3, ok := v2["value"].(float64) + // if ok { + // if strings.HasSuffix(mk1, "_deriv") { + // v3 = v3 / float64(bucketSize) + // } + // if field2, ok := bucketMap[mk1+"_field2"]; ok { + // if idx, ok := grpMetricItemsIndex[mk1]; ok { + // if field2Map, ok := field2.(map[string]interface{}); ok { + // v3 = grpMetricItems[idx].Calc(v3, field2Map["value"].(float64)) + // } + // } + // } + // if v3 < 0 { + // continue + // } + // points := []interface{}{dateTime, v3} + // mv1[grpKey] = append(mv1[grpKey], points) + // } + // } + // } + // } + // } + // } + // } + // + // } + // } + // } + //} + // + //result := map[string]*common.MetricItem{} + // + //for _, metricItem := range grpMetricItems { + // for _, line := range metricItem.MetricItem.Lines { + // line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + // dataKey := metricItem.ID + // if metricItem.IsDerivative { + // dataKey = dataKey + "_deriv" + // } + // line.Data = grpMetricData[dataKey][line.ElasticsearchMetric.Label] + // } + // result[metricItem.Key] = metricItem.MetricItem + //} + return nil +} + +func ConvertMetricItemsToAggQuery(metricItems []*common.MetricItem) map[string]interface{} { + aggs := map[string]interface{}{} + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + aggs[line.Metric.ID] = util.MapStr{ + "max": util.MapStr{ + "field": line.Metric.Field, + }, + } + if line.Metric.IsDerivative { + //add which metric keys to extract + aggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + } + } + } + return aggs +} + +func ConvertBucketItemsToAggQuery(bucketItems []*common.BucketItem, metricItems []*common.MetricItem) util.MapStr { + aggs := util.MapStr{} + + var currentAgg = util.MapStr{} + for _, bucketItem := range bucketItems { + + bucketAgg := util.MapStr{} + + switch bucketItem.Type { + case "terms": + bucketAgg = util.MapStr{ + "terms": bucketItem.Parameters, + } + break + case "date_histogram": + bucketAgg = util.MapStr{ + "date_histogram": bucketItem.Parameters, + } + break + case "date_range": + bucketAgg = util.MapStr{ + "date_range": bucketItem.Parameters, + } + break + } + + //if bucketItem.Buckets!=nil&&len(bucketItem.Buckets)>0{ + nestedAggs := ConvertBucketItemsToAggQuery(bucketItem.Buckets, bucketItem.Metrics) + if len(nestedAggs) > 0 { + util.MergeFields(bucketAgg, nestedAggs, true) + } + //} + currentAgg[bucketItem.Key] = bucketAgg + } + + if metricItems != nil && len(metricItems) > 0 { + metricAggs := ConvertMetricItemsToAggQuery(metricItems) + util.MergeFields(currentAgg, metricAggs, true) + } + + aggs = util.MapStr{ + "aggs": currentAgg, + } + + return aggs +} + +type BucketBase map[string]interface{} + +func (receiver BucketBase) GetChildBucket(name string) (map[string]interface{}, bool) { + bks, ok := receiver[name] + if ok { + bks2, ok := bks.(map[string]interface{}) + return bks2, ok + } + return nil, false +} + +type Bucket struct { + BucketBase //子 buckets + + KeyAsString string `json:"key_as_string,omitempty"` + Key interface{} `json:"key,omitempty"` + DocCount int64 `json:"doc_count,omitempty"` + DocCountErrorUpperBound int64 `json:"doc_count_error_upper_bound,omitempty"` + SumOtherDocCount int64 `json:"sum_other_doc_count,omitempty"` + + Buckets []Bucket `json:"buckets,omitempty"` //本 buckets +} + +type SearchResponse struct { + Took int `json:"took"` + TimedOut bool `json:"timed_out"` + Hits struct { + Total interface{} `json:"total"` + MaxScore float32 `json:"max_score"` + } `json:"hits"` + Aggregations util.MapStr `json:"aggregations,omitempty"` +} + +func ParseAggregationBucketResult(bucketSize int, aggsData util.MapStr, groupKey, resultLabelKey, resultValueKey string, resultItemHandle func()) MetricData { + + metricData := MetricData{} + for k, v := range aggsData { + if k == groupKey { + //start to collect metric for each bucket + objcs, ok := v.(map[string]interface{}) + if ok { + + bks, ok := objcs["buckets"].([]interface{}) + if ok { + for _, bk := range bks { + //check each bucket, collecting metrics + bkMap, ok := bk.(map[string]interface{}) + if ok { + + groupKeyValue, ok := bkMap["key"] + if ok { + } + bkHitMap, ok := bkMap[resultLabelKey] + if ok { + //hit label, 说明匹配到时间范围了 + labelMap, ok := bkHitMap.(map[string]interface{}) + if ok { + labelBks, ok := labelMap["buckets"] + if ok { + labelBksMap, ok := labelBks.([]interface{}) + if ok { + for _, labelItem := range labelBksMap { + metrics, ok := labelItem.(map[string]interface{}) + + labelKeyValue, ok := metrics["to"] //TODO config + if !ok { + labelKeyValue, ok = metrics["from"] //TODO config + } + if !ok { + labelKeyValue, ok = metrics["key"] //TODO config + } + + metric, ok := metrics[resultValueKey] + if ok { + metricMap, ok := metric.(map[string]interface{}) + if ok { + t := "bucket" //metric, bucket + if t == "metric" { + metricValue, ok := metricMap["value"] + if ok { + saveMetric(&metricData, groupKeyValue.(string), labelKeyValue, metricValue, bucketSize) + continue + } + } else { + metricValue, ok := metricMap["buckets"] + if ok { + buckets, ok := metricValue.([]interface{}) + if ok { + var result string = "unavailable" + for _, v := range buckets { + x, ok := v.(map[string]interface{}) + if ok { + if x["key"] == "red" { + result = "red" + break + } + if x["key"] == "yellow" { + result = "yellow" + } else { + if result != "yellow" { + result = x["key"].(string) + } + } + } + } + + v, ok := (metricData)[groupKeyValue.(string)] + if !ok { + v = [][]interface{}{} + } + v2 := []interface{}{} + v2 = append(v2, labelKeyValue) + v2 = append(v2, result) + v = append(v, v2) + + (metricData)[groupKeyValue.(string)] = v + } + + continue + } + } + } + } + } + } + + } + } + + } + } + } + } + } + + } + + } + + return metricData +} + +func ParseAggregationResult(bucketSize int, aggsData util.MapStr, groupKey, metricLabelKey, metricValueKey string) MetricData { + + metricData := MetricData{} + //group bucket key: key1, 获取 key 的 buckets 作为分组的内容 map[group][]{Label，MetricValue} + //metric Label Key: key2, 获取其 key 作为 时间指标 + //metric Value Key: c7qgjrqi4h92sqdaa9b0, 获取其 value 作为 point 内容 + + //groupKey:="key1" + //metricLabelKey:="key2" + //metricValueKey:="c7qi5hii4h935v9bs920" + + //fmt.Println(groupKey," => ",metricLabelKey," => ",metricValueKey) + + for k, v := range aggsData { + //fmt.Println("k:",k) + //fmt.Println("v:",v) + + if k == groupKey { + //fmt.Println("hit group key") + //start to collect metric for each bucket + objcs, ok := v.(map[string]interface{}) + if ok { + + bks, ok := objcs["buckets"].([]interface{}) + if ok { + for _, bk := range bks { + //check each bucket, collecting metrics + //fmt.Println("check bucket:",bk) + + bkMap, ok := bk.(map[string]interface{}) + if ok { + + groupKeyValue, ok := bkMap["key"] + if ok { + //fmt.Println("collecting bucket::",groupKeyValue) + } + bkHitMap, ok := bkMap[metricLabelKey] + if ok { + //hit label, 说明匹配到时间范围了 + labelMap, ok := bkHitMap.(map[string]interface{}) + if ok { + //fmt.Println("bkHitMap",bkHitMap) + + labelBks, ok := labelMap["buckets"] + if ok { + + labelBksMap, ok := labelBks.([]interface{}) + //fmt.Println("get label buckets",ok) + if ok { + //fmt.Println("get label buckets",ok) + + for _, labelItem := range labelBksMap { + metrics, ok := labelItem.(map[string]interface{}) + + //fmt.Println(labelItem) + labelKeyValue, ok := metrics["key"] + if ok { + //fmt.Println("collecting metric label::",int64(labelKeyValue.(float64))) + } + + metric, ok := metrics[metricValueKey] + if ok { + metricMap, ok := metric.(map[string]interface{}) + if ok { + metricValue, ok := metricMap["value"] + if ok { + //fmt.Println("collecting metric value::",metricValue.(float64)) + + saveMetric(&metricData, groupKeyValue.(string), labelKeyValue, metricValue, bucketSize) + continue + } + } + } + } + } + + } + } + + } + } + } + } + } + + } + + } + + //for k,v:=range bucketItems{ + // fmt.Println("k:",k) + // fmt.Println("v:",v) + // aggObect:=aggsData[v.Key] + // fmt.Println("",aggObect) + // //fmt.Println(len(aggObect.Buckets)) + // //for _,bucket:=range aggObect.Buckets{ + // // fmt.Println(bucket.Key) + // // fmt.Println(bucket.GetChildBucket("key2")) + // // //children,ok:=bucket.GetChildBucket() + // // //if ok{ + // // // + // // //} + // //} + //} + + return metricData +} + +func saveMetric(metricData *MetricData, group string, label, value interface{}, bucketSize int) { + + if value == nil { + return + } + + v3, ok := value.(float64) + if ok { + value = v3 / float64(bucketSize) + } + + v, ok := (*metricData)[group] + if !ok { + v = [][]interface{}{} + } + v2 := []interface{}{} + v2 = append(v2, label) + v2 = append(v2, value) + v = append(v, v2) + + (*metricData)[group] = v + //fmt.Printf("save:%v, %v=%v\n",group,label,value) +} + +func parseGroupMetricData(buckets []elastic.BucketBase, isPercent bool) ([]interface{}, error) { + metricData := []interface{}{} + var minDate, maxDate int64 + for _, bucket := range buckets { + v, ok := bucket["key"].(float64) + if !ok { + log.Error("invalid bucket key") + return nil, fmt.Errorf("invalid bucket key") + } + dateTime := int64(v) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + totalCount := bucket["doc_count"].(float64) + if grpStatus, ok := bucket["groups"].(map[string]interface{}); ok { + if statusBks, ok := grpStatus["buckets"].([]interface{}); ok { + for _, statusBk := range statusBks { + if bkMap, ok := statusBk.(map[string]interface{}); ok { + statusKey := bkMap["key"].(string) + count := bkMap["doc_count"].(float64) + if isPercent{ + metricData = append(metricData, map[string]interface{}{ + "x": dateTime, + "y": count / totalCount * 100, + "g": statusKey, + }) + }else{ + metricData = append(metricData, map[string]interface{}{ + "x": dateTime, + "y": count, + "g": statusKey, + }) + } + } + } + } + } + } + return metricData, nil +} + +func (h *APIHandler) getSingleIndexMetricsByNodeStats(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem { + metricData := map[string][][]interface{}{} + + aggs := util.MapStr{} + metricItemsMap := map[string]*common.MetricLine{} + sumAggs := util.MapStr{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + dk := line.Metric.GetDataKey() + metricItemsMap[dk] = line + metricData[dk] = [][]interface{}{} + leafAgg := util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field, + }, + } + var sumBucketPath = "term_node>"+ line.Metric.ID + aggs[line.Metric.ID] = leafAgg + + sumAggs[line.Metric.ID] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath, + }, + } + if line.Metric.Field2 != "" { + leafAgg2 := util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field2, + }, + } + + aggs[line.Metric.ID+"_field2"] = leafAgg2 + sumAggs[line.Metric.ID + "_field2"] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath+"_field2", + }, + } + } + + if line.Metric.IsDerivative { + //add which metric keys to extract + sumAggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + if line.Metric.Field2 != "" { + sumAggs[line.Metric.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID + "_field2", + }, + } + } + } + } + } + + sumAggs["term_node"]= util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 1000, + }, + "aggs": aggs, + } + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + panic(err) + } + query["size"] = 0 + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": sumAggs, + }, + } + return parseSingleIndexMetrics(clusterID, metricItems, query, bucketSize,metricData, metricItemsMap) +} + +func (h *APIHandler) getSingleIndexMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem { + metricData := map[string][][]interface{}{} + + aggs := util.MapStr{} + metricItemsMap := map[string]*common.MetricLine{} + sumAggs := util.MapStr{} + var filterSubAggs = util.MapStr{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + dk := line.Metric.GetDataKey() + metricItemsMap[dk] = line + metricData[dk] = [][]interface{}{} + leafAgg := util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field, + }, + } + var sumBucketPath = "term_shard>"+ line.Metric.ID + if line.Metric.OnlyPrimary { + filterSubAggs[line.Metric.ID] = leafAgg + aggs["filter_pri"]=util.MapStr{ + "filter": util.MapStr{ + "term": util.MapStr{ + "payload.elasticsearch.shard_stats.routing.primary": util.MapStr{ + "value": true, + }, + }, + }, + "aggs": filterSubAggs, + } + sumBucketPath = "term_shard>filter_pri>"+ line.Metric.ID + }else{ + aggs[line.Metric.ID] = leafAgg + } + + sumAggs[line.Metric.ID] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath, + }, + } + if line.Metric.Field2 != "" { + leafAgg2 := util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field2, + }, + } + if line.Metric.OnlyPrimary { + filterSubAggs[line.Metric.ID+"_field2"] = leafAgg2 + }else{ + aggs[line.Metric.ID+"_field2"] = leafAgg2 + } + + sumAggs[line.Metric.ID + "_field2"] = util.MapStr{ + "sum_bucket": util.MapStr{ + "buckets_path": sumBucketPath+"_field2", + }, + } + } + + if line.Metric.IsDerivative { + //add which metric keys to extract + sumAggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + if line.Metric.Field2 != "" { + sumAggs[line.Metric.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID + "_field2", + }, + } + } + } + } + } + + sumAggs["term_shard"]= util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.shard_id", + "size": 100000, + }, + "aggs": aggs, + } + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + panic(err) + } + query["size"] = 0 + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": sumAggs, + }, + } + return parseSingleIndexMetrics(clusterID, metricItems, query, bucketSize,metricData, metricItemsMap) +} + +func parseSingleIndexMetrics(clusterID string, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int, metricData map[string][][]interface{}, metricItemsMap map[string]*common.MetricLine) map[string]*common.MetricItem { + response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + panic(err) + } + + var minDate, maxDate int64 + if response.StatusCode == 200 { + for _, v := range response.Aggregations { + for _, bucket := range v.Buckets { + v, ok := bucket["key"].(float64) + if !ok { + panic("invalid bucket key") + } + dateTime := (int64(v)) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + for mk1, mv1 := range metricData { + v1, ok := bucket[mk1] + if ok { + v2, ok := v1.(map[string]interface{}) + if ok { + v3, ok := v2["value"].(float64) + if ok { + if strings.HasSuffix(mk1, "_deriv") { + if _, ok := bucket[mk1+"_field2"]; !ok { + v3 = v3 / float64(bucketSize) + } + } + if field2, ok := bucket[mk1+"_field2"]; ok { + if line, ok := metricItemsMap[mk1]; ok { + if field2Map, ok := field2.(map[string]interface{}); ok { + v4 := field2Map["value"].(float64) + if v4 == 0 { + v3 = 0 + } else { + v3 = line.Metric.Calc(v3, v4) + } + } + } + } + if v3 < 0 { + continue + } + points := []interface{}{dateTime, v3} + metricData[mk1] = append(mv1, points) + } + + } + } + } + } + } + } + + result := map[string]*common.MetricItem{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + line.Data = metricData[line.Metric.GetDataKey()] + if v, ok := line.Data.([][]interface{}); ok && len(v)> 0 && bucketSize <= 60 { + // remove first metric dot + temp := v[1:] + // // remove first last dot + if len(temp) > 0 { + temp = temp[0: len(temp)-1] + } + line.Data = temp + } + } + result[metricItem.Key] = metricItem + } + + return result +} \ No newline at end of file diff --git a/modules/elastic/api/metrics_util_test.go b/modules/elastic/api/metrics_util_test.go new file mode 100644 index 00000000..83e483ba --- /dev/null +++ b/modules/elastic/api/metrics_util_test.go @@ -0,0 +1,92 @@ +package api + +import ( + "fmt" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "testing" + "time" +) + +func TestGetMetricParams(t *testing.T) { + handler:=APIHandler{} + req:=http.Request{} + bucketSize, min, max, err:=handler.getMetricRangeAndBucketSize(&req,60,15) + + fmt.Println(bucketSize) + fmt.Println(util.FormatUnixTimestamp(min/1000))//2022-01-27 15:28:57 + fmt.Println(util.FormatUnixTimestamp(max/1000))//2022-01-27 15:28:57 + fmt.Println(time.Now())//2022-01-27 15:28:57 + + fmt.Println(bucketSize, min, max, err) +} + +func TestConvertBucketItemsToAggQueryParams(t *testing.T) { + bucketItem:=common.BucketItem{} + bucketItem.Key="key1" + bucketItem.Type=common.TermsBucket + bucketItem.Parameters=map[string]interface{}{} + bucketItem.Parameters["field"]="metadata.labels.cluster_id" + bucketItem.Parameters["size"]=2 + + + nestBucket:=common.BucketItem{} + nestBucket.Key="key2" + nestBucket.Type=common.DateHistogramBucket + nestBucket.Parameters=map[string]interface{}{} + nestBucket.Parameters["field"]="timestamp" + nestBucket.Parameters["calendar_interval"]="1d" + nestBucket.Parameters["time_zone"]="+08:00" + + leafBucket:=common.NewBucketItem(common.TermsBucket,util.MapStr{ + "size":5, + "field":"payload.elasticsearch.cluster_health.status", + }) + + leafBucket.Key="key3" + + metricItems:=[]*common.MetricItem{} + var bucketSizeStr ="10s" + metricItem:=newMetricItem("cluster_summary", 2, "cluster") + metricItem.Key="key4" + metricItem.AddLine("Indexing","Total Indexing","Number of documents being indexed for primary and replica shards.","group1", + "payload.elasticsearch.index_stats.total.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Search","Total Search","Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!","group1", + "payload.elasticsearch.index_stats.total.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + nestBucket.AddNestBucket(leafBucket) + nestBucket.Metrics=metricItems + + bucketItem.Buckets=[]*common.BucketItem{} + bucketItem.Buckets=append(bucketItem.Buckets,&nestBucket) + + + aggs:=ConvertBucketItemsToAggQuery([]*common.BucketItem{&bucketItem},nil) + fmt.Println(util.MustToJSON(aggs)) + + response:="{ \"took\": 37, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 10000, \"relation\": \"gte\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"key1\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7pqhptj69a0sg3rn05g\", \"doc_count\": 80482, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-28T00:00:00.000+08:00\", \"key\": 1643299200000, \"doc_count\": 14310, \"c7qi5hii4h935v9bs91g\": { \"value\": 15680 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 2985 } }, { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 66172, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 }, \"c7qi5hii4h935v9bs91g_deriv\": { \"value\": 90526 }, \"c7qi5hii4h935v9bs920_deriv\": { \"value\": 17219 } } ] } }, { \"key\": \"c7qi42ai4h92sksk979g\", \"doc_count\": 660, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 660, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 } } ] } } ] } } }" + res:=SearchResponse{} + util.FromJSONBytes([]byte(response),&res) + fmt.Println(response) + groupKey:="key1" + metricLabelKey:="key2" + metricValueKey:="c7qi5hii4h935v9bs920" + data:=ParseAggregationResult(int(10),res.Aggregations,groupKey,metricLabelKey,metricValueKey) + fmt.Println(data) + +} + +func TestConvertBucketItems(t *testing.T) { + response:="{ \"took\": 8, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 81, \"relation\": \"eq\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"c7v2gm3i7638vvo4pv80\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7uv7p3i76360kgdmpb0\", \"doc_count\": 81, \"c7v2gm3i7638vvo4pv8g\": { \"buckets\": [ { \"key_as_string\": \"2022-02-05T00:00:00.000+08:00\", \"key\": 1643990400000, \"doc_count\": 81, \"c7v2gm3i7638vvo4pv90\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"yellow\", \"doc_count\": 81 } ] } } ] } } ] } } }" + res:=SearchResponse{} + util.FromJSONBytes([]byte(response),&res) + + data:=ParseAggregationBucketResult(int(10),res.Aggregations,"c7v2gm3i7638vvo4pv80","c7v2gm3i7638vvo4pv8g","c7v2gm3i7638vvo4pv90", func() { + + }) + + fmt.Println(data) + +} diff --git a/modules/elastic/api/monitor_state.go b/modules/elastic/api/monitor_state.go new file mode 100644 index 00000000..c23f84bf --- /dev/null +++ b/modules/elastic/api/monitor_state.go @@ -0,0 +1,26 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "fmt" + "infini.sh/framework/core/elastic" +) + +type MonitorState int +const ( + Console MonitorState = iota + Agent +) +func GetMonitorState(clusterID string) MonitorState { + conf := elastic.GetConfig(clusterID) + if conf == nil { + panic(fmt.Errorf("config of cluster [%s] is not found", clusterID)) + } + if conf.MonitorConfigs != nil && !conf.MonitorConfigs.NodeStats.Enabled && !conf.MonitorConfigs.IndexStats.Enabled { + return Agent + } + return Console +} diff --git a/modules/elastic/api/node_metrics.go b/modules/elastic/api/node_metrics.go new file mode 100644 index 00000000..44c61077 --- /dev/null +++ b/modules/elastic/api/node_metrics.go @@ -0,0 +1,1185 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "sort" + "strings" + "time" +) + +func (h *APIHandler) getNodeMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) (map[string]*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + } + var ( + nodeNames []string + ) + if nodeName != "" { + nodeNames = strings.Split(nodeName, ",") + top = len(nodeNames) + }else{ + nodeNames, err = h.getTopNodeName(clusterID, top, 15) + if err != nil { + log.Error(err) + } + } + if len(nodeNames) > 0 { + must = append(must, util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.transport_address": nodeNames, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeNames, + }, + }, + }, + }, + + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + cpuMetric := newMetricItem("cpu", 1, SystemGroupKey) + cpuMetric.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + + nodeMetricItems := []GroupMetricItem{ + { + Key: "cpu", + Field: "payload.elasticsearch.node_stats.process.cpu.percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: cpuMetric, + FormatType: "ratio", + Units: "%", + }, + } + + osCpuMetric := newMetricItem("os_cpu", 2, SystemGroupKey) + osCpuMetric.AddAxi("OS CPU Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_cpu", + Field: "payload.elasticsearch.node_stats.os.cpu.percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osCpuMetric, + FormatType: "ratio", + Units: "%", + }) + + osMemMetric := newMetricItem("os_used_mem", 2, SystemGroupKey) + osMemMetric.AddAxi("OS Mem Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_used_mem", + Field: "payload.elasticsearch.node_stats.os.mem.used_percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osMemMetric, + FormatType: "ratio", + Units: "%", + }) + osLoadMetric := newMetricItem("os_load_average_1m", 2, SystemGroupKey) + osLoadMetric.AddAxi("OS Load 1m Average","group1",common.PositionLeft,"","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_load_average_1m", + Field: "payload.elasticsearch.node_stats.os.cpu.load_average.1m", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osLoadMetric, + FormatType: "num", + Units: "", + }) + //swap usage + osSwapMetric := newMetricItem("os_used_swap", 3, SystemGroupKey) + osSwapMetric.AddAxi("OS Swap Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_used_swap", + Field: "payload.elasticsearch.node_stats.os.swap.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + Field2: "payload.elasticsearch.node_stats.os.swap.total_in_bytes", + Calc: func(value, value2 float64) float64 { + return util.ToFixed((value / value2)*100, 2) + }, + MetricItem: osSwapMetric, + FormatType: "ratio", + Units: "%", + }) + openFileMetric := newMetricItem("open_file", 2, SystemGroupKey) + openFileMetric.AddAxi("Open File Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "open_file", + Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: openFileMetric, + FormatType: "num", + Units: "", + }) + openFilePercentMetric := newMetricItem("open_file_percent", 2, SystemGroupKey) + openFilePercentMetric.AddAxi("Open File Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "open_file_percent", + Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", + ID: util.GetUUID(), + IsDerivative: false, + Field2: "payload.elasticsearch.node_stats.process.max_file_descriptors", + Calc: func(value, value2 float64) float64 { + if value < 0 { + return value + } + return util.ToFixed((value / value2)*100, 2) + }, + MetricItem: openFilePercentMetric, + FormatType: "ratio", + Units: "%", + }) + + diskMetric := newMetricItem("disk", 2, SystemGroupKey) + diskMetric.AddAxi("disk available percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "disk", + Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: diskMetric, + FormatType: "ratio", + Units: "%", + Field2: "payload.elasticsearch.node_stats.fs.total.available_in_bytes", + Calc: func(value, value2 float64) float64 { + return util.ToFixed((value2 / value)*100, 2) + }, + }) + // 索引速率 + indexMetric:=newMetricItem("indexing_rate", 1, OperationGroupKey) + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_rate", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "doc/s", + }) + + indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey) + indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_bytes", + Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingBytesMetric, + FormatType: "bytes", + Units: "bytes/s", + }) + + // 索引延时 + indexLatencyMetric:=newMetricItem("indexing_latency", 1, LatencyGroupKey) + indexLatencyMetric.AddAxi("indexing latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_latency", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.indexing.index_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + queryMetric:=newMetricItem("query_rate", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_rate", + Field: "payload.elasticsearch.node_stats.indices.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "requests/s", + }) + + // 查询延时 + queryLatencyMetric:=newMetricItem("query_latency", 2, LatencyGroupKey) + queryLatencyMetric.AddAxi("query latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_latency", + Field: "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.query_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + fetchMetric:=newMetricItem("fetch_rate", 3, OperationGroupKey) + fetchMetric.AddAxi("fetch rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fetch_rate", + Field: "payload.elasticsearch.node_stats.indices.search.fetch_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchMetric, + FormatType: "num", + Units: "requests/s", + }) + scrollMetric:=newMetricItem("scroll_rate", 4, OperationGroupKey) + scrollMetric.AddAxi("scroll rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_rate", + Field: "payload.elasticsearch.node_stats.indices.search.scroll_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollMetric, + FormatType: "num", + Units: "requests/s", + }) + + refreshMetric:=newMetricItem("refresh_rate", 5, OperationGroupKey) + refreshMetric.AddAxi("refresh rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "refresh_rate", + Field: "payload.elasticsearch.node_stats.indices.refresh.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshMetric, + FormatType: "num", + Units: "requests/s", + }) + flushMetric:=newMetricItem("flush_rate", 6, OperationGroupKey) + flushMetric.AddAxi("flush rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "flush_rate", + Field: "payload.elasticsearch.node_stats.indices.flush.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushMetric, + FormatType: "num", + Units: "requests/s", + }) + mergeMetric:=newMetricItem("merges_rate", 7, OperationGroupKey) + mergeMetric.AddAxi("merges rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "merges_rate", + Field: "payload.elasticsearch.node_stats.indices.merges.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeMetric, + FormatType: "num", + Units: "requests/s", + }) + + // fetch延时 + fetchLatencyMetric:=newMetricItem("fetch_latency", 3, LatencyGroupKey) + fetchLatencyMetric.AddAxi("fetch latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fetch_latency", + Field: "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.fetch_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // scroll 延时 + scrollLatencyMetric:=newMetricItem("scroll_latency", 4, LatencyGroupKey) + scrollLatencyMetric.AddAxi("scroll latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_latency", + Field: "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.scroll_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + // merge 延时 + mergeLatencyMetric:=newMetricItem("merge_latency", 7, LatencyGroupKey) + mergeLatencyMetric.AddAxi("merge latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "merge_latency", + Field: "payload.elasticsearch.node_stats.indices.merges.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.merges.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + // refresh 延时 + refreshLatencyMetric:=newMetricItem("refresh_latency", 5, LatencyGroupKey) + refreshLatencyMetric.AddAxi("refresh latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "refresh_latency", + Field: "payload.elasticsearch.node_stats.indices.refresh.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.refresh.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // flush 时延 + flushLatencyMetric:=newMetricItem("flush_latency", 6, LatencyGroupKey) + flushLatencyMetric.AddAxi("flush latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "flush_latency", + Field: "payload.elasticsearch.node_stats.indices.flush.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.flush.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // Query Cache 内存占用大小 + queryCacheMetric:=newMetricItem("query_cache", 1, CacheGroupKey) + queryCacheMetric.AddAxi("query cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache", + Field: "payload.elasticsearch.node_stats.indices.query_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: queryCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache 内存占用大小 + requestCacheMetric:=newMetricItem("request_cache", 2, CacheGroupKey) + requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache", + Field: "payload.elasticsearch.node_stats.indices.request_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: requestCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache Hit + requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey) + requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache_hit", + Field: "payload.elasticsearch.node_stats.indices.request_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + // Request Cache Miss + requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey) + requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache_miss", + Field: "payload.elasticsearch.node_stats.indices.request_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Query Cache Count + queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey) + queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_count", + Field: "payload.elasticsearch.node_stats.indices.query_cache.cache_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheCountMetric, + FormatType: "num", + Units: "", + }) + // Query Cache Miss + queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey) + queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_hit", + Field: "payload.elasticsearch.node_stats.indices.query_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + + //// Query Cache evictions + //queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 5, CacheGroupKey) + //queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + //nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + // Key: "query_cache_evictions", + // Field: "payload.elasticsearch.node_stats.indices.query_cache.evictions", + // ID: util.GetUUID(), + // IsDerivative: true, + // MetricItem: queryCacheEvictionsMetric, + // FormatType: "num", + // Units: "evictions", + //}) + + // Query Cache Miss + queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey) + queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_miss", + Field: "payload.elasticsearch.node_stats.indices.query_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + + // Fielddata内存占用大小 + fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey) + fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fielddata_cache", + Field: "payload.elasticsearch.node_stats.indices.fielddata.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldDataCacheMetric, + FormatType: "bytes", + Units: "", + }) + + // http 活跃连接数 + httpActiveMetric:=newMetricItem("http_connect_num", 12, HttpGroupKey) + httpActiveMetric.AddAxi("http connect number","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "http_connect_num", + Field: "payload.elasticsearch.node_stats.http.current_open", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: httpActiveMetric, + FormatType: "num", + Units: "conns", + }) + // http 活跃连接数速率 + httpRateMetric:=newMetricItem("http_rate", 12, HttpGroupKey) + httpRateMetric.AddAxi("http rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "http_rate", + Field: "payload.elasticsearch.node_stats.http.total_opened", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: httpRateMetric, + FormatType: "num", + Units: "conns/s", + }) + + // segment 数量 + segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey) + segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_count", + Field: "payload.elasticsearch.node_stats.indices.segments.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentCountMetric, + FormatType: "num", + Units: "", + }) + + // segment memory + segmentMemoryMetric:=newMetricItem("segment_memory", 16, MemoryGroupKey) + segmentMemoryMetric.AddAxi("segment memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment stored fields memory + segmentStoredFieldsMemoryMetric:=newMetricItem("segment_stored_fields_memory", 16, MemoryGroupKey) + segmentStoredFieldsMemoryMetric.AddAxi("segment stored fields memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_stored_fields_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.stored_fields_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentStoredFieldsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment terms fields memory + segmentTermsMemoryMetric:=newMetricItem("segment_terms_memory", 16, MemoryGroupKey) + segmentTermsMemoryMetric.AddAxi("segment terms memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_terms_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.terms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment doc values memory + segmentDocValuesMemoryMetric:=newMetricItem("segment_doc_values_memory", 16, MemoryGroupKey) + segmentDocValuesMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_doc_values_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.doc_values_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentDocValuesMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment index writer memory + segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey) + segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_index_writer_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.index_writer_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentIndexWriterMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment term vectors memory + segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey) + segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_term_vectors_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.term_vectors_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermVectorsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + // docs 数量 + docsCountMetric:=newMetricItem("docs_count", 17, DocumentGroupKey) + docsCountMetric.AddAxi("docs count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "docs_count", + Field: "payload.elasticsearch.node_stats.indices.docs.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsCountMetric, + FormatType: "num", + Units: "", + }) + // docs 删除数量 + docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey) + docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "docs_deleted", + Field: "payload.elasticsearch.node_stats.indices.docs.deleted", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsDeletedMetric, + FormatType: "num", + Units: "", + }) + + // index store size + indexStoreMetric:=newMetricItem("index_storage", 18, StorageGroupKey) + indexStoreMetric.AddAxi("indices storage","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "index_storage", + Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexStoreMetric, + FormatType: "bytes", + Units: "", + }) + + // jvm used heap + jvmUsedPercentMetric:=newMetricItem("jvm_heap_used_percent", 1, JVMGroupKey) + jvmUsedPercentMetric.AddAxi("JVM heap used percent","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_heap_used_percent", + Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: jvmUsedPercentMetric, + FormatType: "num", + Units: "%", + }) + //JVM mem Young pools used + youngPoolsUsedMetric:=newMetricItem("jvm_mem_young_used", 2, JVMGroupKey) + youngPoolsUsedMetric.AddAxi("Mem Pools Young Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_young_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: youngPoolsUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM mem Young pools peak used + youngPoolsUsedPeakMetric:=newMetricItem("jvm_mem_young_peak_used", 2, JVMGroupKey) + youngPoolsUsedPeakMetric.AddAxi("Mem Pools Young Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_young_peak_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.peak_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: youngPoolsUsedPeakMetric, + FormatType: "bytes", + Units: "", + }) + + //JVM mem old pools used + oldPoolsUsedMetric:=newMetricItem("jvm_mem_old_used", 3, JVMGroupKey) + oldPoolsUsedMetric.AddAxi("Mem Pools Old Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_old_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: oldPoolsUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM mem old pools peak used + oldPoolsUsedPeakMetric:=newMetricItem("jvm_mem_old_peak_used", 3, JVMGroupKey) + oldPoolsUsedPeakMetric.AddAxi("Mem Pools Old Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_old_peak_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.peak_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: oldPoolsUsedPeakMetric, + FormatType: "bytes", + Units: "", + }) + + //JVM used heap + heapUsedMetric:=newMetricItem("jvm_used_heap", 1, JVMGroupKey) + heapUsedMetric.AddAxi("JVM Used Heap","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_used_heap", + Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: heapUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM Young GC Rate + gcYoungRateMetric:=newMetricItem("jvm_young_gc_rate", 2, JVMGroupKey) + gcYoungRateMetric.AddAxi("JVM Young GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_young_gc_rate", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcYoungRateMetric, + FormatType: "num", + Units: "times/s", + }) + //JVM Young GC Latency + gcYoungLatencyMetric:=newMetricItem("jvm_young_gc_latency", 2, JVMGroupKey) + gcYoungLatencyMetric.AddAxi("JVM Young GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_young_gc_latency", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_time_in_millis", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcYoungLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //JVM old GC Rate + gcOldRateMetric:=newMetricItem("jvm_old_gc_rate", 3, JVMGroupKey) + gcOldRateMetric.AddAxi("JVM Old GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_old_gc_rate", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcOldRateMetric, + FormatType: "num", + Units: "times/s", + }) + //JVM old GC Latency + gcOldLatencyMetric:=newMetricItem("jvm_old_gc_latency", 3, JVMGroupKey) + gcOldLatencyMetric.AddAxi("JVM Old GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_old_gc_latency", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_time_in_millis", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcOldLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //Transport 发送速率 + transTxRateMetric:=newMetricItem("transport_tx_rate", 19, TransportGroupKey) + transTxRateMetric.AddAxi("Transport Send Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_tx_rate", + Field: "payload.elasticsearch.node_stats.transport.tx_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transTxRateMetric, + FormatType: "num", + Units: "times/s", + }) + //Transport 接收速率 + transRxRateMetric:=newMetricItem("transport_rx_rate", 19, TransportGroupKey) + transRxRateMetric.AddAxi("Transport Receive Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_rx_rate", + Field: "payload.elasticsearch.node_stats.transport.rx_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transRxRateMetric, + FormatType: "num", + Units: "times/s", + }) + + //Transport 发送流量 + transTxBytesMetric:=newMetricItem("transport_tx_bytes", 19, TransportGroupKey) + transTxBytesMetric.AddAxi("Transport Send Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_tx_bytes", + Field: "payload.elasticsearch.node_stats.transport.tx_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transTxBytesMetric, + FormatType: "bytes", + Units: "s", + }) + //Transport 接收流量 + transRxBytesMetric:=newMetricItem("transport_rx_bytes", 19, TransportGroupKey) + transRxBytesMetric.AddAxi("Transport Receive Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_rx_bytes", + Field: "payload.elasticsearch.node_stats.transport.rx_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transRxBytesMetric, + FormatType: "bytes", + Units: "s", + }) + + //Transport tcp 连接数 + tcpNumMetric:=newMetricItem("transport_outbound_connections", 20, TransportGroupKey) + tcpNumMetric.AddAxi("Transport Outbound Connections","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_outbound_connections", + Field: "payload.elasticsearch.node_stats.transport.total_outbound_connections", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: tcpNumMetric, + FormatType: "num", + Units: "", + }) + + //IO total + totalOperationsMetric:=newMetricItem("total_io_operations", 1, IOGroupKey) + totalOperationsMetric.AddAxi("Total I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: totalOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //IO total + readOperationsMetric:=newMetricItem("total_read_io_operations", 2, IOGroupKey) + readOperationsMetric.AddAxi("Total Read I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_read_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.read_operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: readOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //IO total + writeOperationsMetric:=newMetricItem("total_write_io_operations", 3, IOGroupKey) + writeOperationsMetric.AddAxi("Total Write I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_write_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.write_operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: writeOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //scroll context + openContextMetric:=newMetricItem("scroll_open_contexts", 7, OperationGroupKey) + openContextMetric.AddAxi("Scroll Open Contexts","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_open_contexts", + Field: "payload.elasticsearch.node_stats.indices.search.open_contexts", + ID: util.GetUUID(), + MetricItem: openContextMetric, + FormatType: "num", + Units: "", + }) + + // Circuit Breaker + parentBreakerMetric := newMetricItem("parent_breaker", 1, CircuitBreakerGroupKey) + parentBreakerMetric.AddAxi("Parent Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "parent_breaker", + Field: "payload.elasticsearch.node_stats.breakers.parent.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: parentBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + accountingBreakerMetric := newMetricItem("accounting_breaker", 2, CircuitBreakerGroupKey) + accountingBreakerMetric.AddAxi("Accounting Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "accounting_breaker", + Field: "payload.elasticsearch.node_stats.breakers.accounting.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: accountingBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + fielddataBreakerMetric := newMetricItem("fielddata_breaker", 3, CircuitBreakerGroupKey) + fielddataBreakerMetric.AddAxi("Fielddata Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "fielddata_breaker", + Field: "payload.elasticsearch.node_stats.breakers.fielddata.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fielddataBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + requestBreakerMetric := newMetricItem("request_breaker", 4, CircuitBreakerGroupKey) + requestBreakerMetric.AddAxi("Request Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "request_breaker", + Field: "payload.elasticsearch.node_stats.breakers.request.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + inFlightRequestBreakerMetric := newMetricItem("in_flight_requests_breaker", 5, CircuitBreakerGroupKey) + inFlightRequestBreakerMetric.AddAxi("In Flight Requests Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "in_flight_requests_breaker", + Field: "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: inFlightRequestBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + modelInferenceBreakerMetric := newMetricItem("model_inference_breaker", 6, CircuitBreakerGroupKey) + modelInferenceBreakerMetric.AddAxi("Model Inference Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "model_inference_breaker", + Field: "payload.elasticsearch.node_stats.breakers.model_inference.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: modelInferenceBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + + aggs := generateGroupAggs(nodeMetricItems) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"] = 0 + query["aggs"] = util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": top, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": aggs, + }, + }, + }, + } + return h.getMetrics(query, nodeMetricItems, bucketSize), nil + +} + +func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int) ([]string, error){ + ver := h.Client().GetVersion() + cr, _ := util.VersionCompare(ver.Number, "6.1") + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { + return nil, nil + } + var ( + now = time.Now() + max = now.UnixNano()/1e6 + min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 + bucketSizeStr = "60s" + ) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + + query := util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "group_by_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>search_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}}, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "search_query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + }, + }, + "search_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "search_query_total", + }, + }, + }, + }, + }, + }, + "group_by_index1": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>index_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}, + }, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + }, + }, + "index_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + }, + }, + }, + }, + }, + } + response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) + if err!=nil{ + log.Error(err) + return nil, err + } + var maxQpsKVS = map[string] float64{} + for _, agg := range response.Aggregations { + for _, bk := range agg.Buckets { + key := bk["key"].(string) + if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { + val := maxQps["value"].(float64) + if _, ok = maxQpsKVS[key] ; ok { + maxQpsKVS[key] = maxQpsKVS[key] + val + }else{ + maxQpsKVS[key] = val + } + } + } + } + var ( + qpsValues TopTermOrder + ) + for k, v := range maxQpsKVS { + qpsValues = append(qpsValues, TopTerm{ + Key: k, + Value: v, + }) + } + sort.Sort(qpsValues) + var length = top + if top > len(qpsValues) { + length = len(qpsValues) + } + nodeNames := []string{} + for i := 0; i 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + w.Write(util.MustToJSONBytes(response)) +} + +func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var nodeIDs = []string{} + h.DecodeJSON(req, &nodeIDs) + + if len(nodeIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + q1 := orm.Query{WildcardIndex: true} + query := util.MapStr{ + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "collapse": util.MapStr{ + "field": "metadata.labels.node_id", + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + }, + }, + } + q1.RawQuery = util.MustToJSONBytes(query) + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + statusMap := map[string]interface{}{} + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) + if ok { + source := map[string]interface{}{} + //timestamp, ok := result["timestamp"].(string) + uptime, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, result) + if ok { + source["uptime"] = uptime + } + + fsTotal, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total"}, result) + if ok { + source["fs"] = util.MapStr{ + "total": fsTotal, + } + } + + jvmMem, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem"}, result) + if ok { + source["jvm"] = util.MapStr{ + "mem": jvmMem, + } + } + indices := util.MapStr{} + docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "docs"}, result) + if ok { + indices["docs"] = docs + } + source["indices"] = indices + shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, result) + if ok { + source["shard_info"] = shardInfo + } + if tempClusterID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "cluster_id"}, result); ok { + if clusterID, ok := tempClusterID.(string); ok { + if meta := elastic.GetMetadata(clusterID); meta != nil && meta.ClusterState != nil { + source["is_master_node"] = meta.ClusterState.MasterNode == nodeID + } + } + } + + statusMap[util.ToString(nodeID)] = source + } + } + } + statusMetric, err := getNodeOnlineStatusOfRecentDay(nodeIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, (15)) + if err != nil { + panic(err) + return + } + // 索引速率 + indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems := []GroupMetricItem{} + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "Indexing/s", + }) + queryMetric:=newMetricItem("search", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "search", + Field: "payload.elasticsearch.node_stats.indices.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "Search/s", + }) + + aggs:=map[string]interface{}{} + query=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + for _,metricItem:=range nodeMetricItems{ + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + } + } + + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + panic(err) + } + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 100, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + }, + }, + } + metrics := h.getMetrics(query, nodeMetricItems, bucketSize) + indexMetrics := map[string]util.MapStr{} + for key, item := range metrics { + for _, line := range item.Lines { + if _, ok := indexMetrics[line.Metric.Label]; !ok{ + indexMetrics[line.Metric.Label] = util.MapStr{ + } + } + indexMetrics[line.Metric.Label][key] = line.Data + } + } + result := util.MapStr{} + for _, nodeID := range nodeIDs { + source := util.MapStr{} + + source["summary"] = statusMap[nodeID] + source["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Node Status", + "units": "day", + }, + "data": statusMetric[nodeID], + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": indexMetrics[nodeID]["indexing"], + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": indexMetrics[nodeID]["search"], + }, + } + result[nodeID] = source + } + h.WriteJSON(w, result, http.StatusOK) +} + +func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + nodeID := ps.MustGetParameter("node_id") + + q := orm.Query{ + Size: 1, + } + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And(orm.Eq("metadata.node_id", nodeID), orm.Eq("metadata.cluster_id", clusterID)) + + err, res := orm.Search(&elastic.NodeConfig{}, &q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + //if len(response.Hits.Hits) == 0 { + // h.WriteError(w, "", http.StatusNotFound) + // return + //} + q1 := orm.Query{ + Size: 1, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "node_stats"), + orm.Eq("metadata.labels.node_id", nodeID), + ) + q1.Collapse("metadata.labels.node_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + kvs := util.MapStr{} + if len(result.Result) > 0 { + vresult, ok := result.Result[0].(map[string]interface{}) + if ok { + transportAddress, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "transport_address"}, vresult) + if ok { + kvs["transport_address"] = transportAddress + } + kvs["timestamp"] = vresult["timestamp"] + if vresult["timestamp"] != nil { + if ts, ok := vresult["timestamp"].(string); ok { + tt, _ := time.Parse(time.RFC3339, ts) + if time.Now().Sub(tt).Seconds() > 30 { + kvs["status"] = "unavailable" + }else{ + kvs["status"] = "available" + } + } + } + roles, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "roles"}, vresult) + if ok { + kvs["roles"] = roles + } + fsTotal, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total"}, vresult) + if ok { + kvs["fs"] = util.MapStr{ + "total": fsTotal, + } + } + + jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult) + if ok { + if jvmVal, ok := jvm.(map[string]interface{});ok { + kvs["jvm"] = util.MapStr{ + "mem": jvmVal["mem"], + "uptime": jvmVal["uptime_in_millis"], + } + } + } + indices := util.MapStr{} + docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "docs"}, vresult) + if ok { + indices["docs"] = docs + } + store, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "store"}, vresult) + if ok { + indices["store"] = store + } + kvs["indices"] = indices + shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, vresult) + if ok { + kvs["shard_info"] = shardInfo + } + } + } + if len( response.Hits.Hits) > 0 { + hit := response.Hits.Hits[0] + innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source) + if mp, ok := innerMetaData.(map[string]interface{}); ok { + kvs["transport_address"] = mp["transport_address"] + kvs["roles"] = mp["roles"] + if kvs["status"] != "available" { + kvs["status"] = mp["status"] + kvs["timestamp"] = hit.Source["timestamp"] + } + } + } + + if meta := elastic.GetMetadata(clusterID); meta != nil && meta.ClusterState != nil { + kvs["is_master_node"] = meta.ClusterState.MasterNode == nodeID + } + h.WriteJSON(w, kvs, http.StatusOK) +} + +func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + nodeID := ps.MustGetParameter("node_id") + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeID, + }, + }, + }, + } + resBody := map[string]interface{}{} + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req,10,60) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + metricItems:=[]*common.MetricItem{} + metricItem:=newMetricItem("cpu", 1, SystemGroupKey) + metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) + metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) + metricItems=append(metricItems,metricItem) + metricItem =newMetricItem("jvm", 2, SystemGroupKey) + metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) + metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("index_throughput", 3, OperationGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("search_throughput", 4, OperationGroupKey) + metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + metricItem.AddLine("Search Rate","Total Shards", + "Number of search requests being executed.", + "group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("index_latency", 5, LatencyGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + + metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("search_latency", 6, LatencyGroupKey) + metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + + metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + metricItem =newMetricItem("parent_breaker", 8, SystemGroupKey) + metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + metrics := h.getSingleMetrics(metricItems,query, bucketSize) + healthMetric, err := getNodeHealthMetric(query, bucketSize) + if err != nil { + log.Error(err) + } + query = util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeID, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + } + shardStateMetric, err := getNodeShardStateMetric(query, bucketSize) + if err != nil { + log.Error(err) + } + metrics["node_health"] = healthMetric + metrics["shard_state"] = shardStateMetric + resBody["metrics"] = metrics + h.WriteJSON(w, resBody, http.StatusOK) +} + +func getNodeShardStateMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "groups": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.shard_stats.routing.state", + "size": 10, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("shard_state", 0, "") + metricItem.AddLine("Shard State","Shard State","","group1","payload.elasticsearch.shard_stats.routing.state","count",bucketSizeStr,"","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + metricData, err = parseGroupMetricData(response.Aggregations["dates"].Buckets, false) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "min_uptime": util.MapStr{ + "min": util.MapStr{ + "field": "payload.elasticsearch.node_stats.jvm.uptime_in_millis", + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("node_health", 0, "") + metricItem.AddLine("Node health","Node Health","","group1","payload.elasticsearch.node_stats.jvm.uptime_in_millis","min",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + for _, bucket := range response.Aggregations["dates"].Buckets { + v, ok := bucket["key"].(float64) + if !ok { + log.Error("invalid bucket key") + return nil, fmt.Errorf("invalid bucket key") + } + dateTime := int64(v) + statusKey := "available" + if uptimeAgg, ok := bucket["min_uptime"].(map[string]interface{}); ok { + if _, ok = uptimeAgg["value"].(float64); !ok { + statusKey = "unavailable" + } + metricData = append(metricData, map[string]interface{}{ + "x": dateTime, + "y": 100, + "g": statusKey, + }) + } + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{}, error){ + q := orm.Query{ + WildcardIndex: true, + } + query := util.MapStr{ + "aggs": util.MapStr{ + "group_by_node_id": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 100, + }, + "aggs": util.MapStr{ + "uptime_histogram": util.MapStr{ + "date_range": util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + },{ + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }, + "aggs": util.MapStr{ + "min_uptime": util.MapStr{ + "min": util.MapStr{ + "field": "payload.elasticsearch.node_stats.jvm.uptime_in_millis", + }, + }, + }, + }, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte":"now-15d", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + return nil, err + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + recentStatus := map[string][]interface{}{} + for _, bk := range response.Aggregations["group_by_node_id"].Buckets { + nodeKey := bk["key"].(string) + recentStatus[nodeKey] = []interface{}{} + if histogramAgg, ok := bk["uptime_histogram"].(map[string]interface{}); ok { + if bks, ok := histogramAgg["buckets"].([]interface{}); ok { + for _, bkItem := range bks { + if bkVal, ok := bkItem.(map[string]interface{}); ok { + if minUptime, ok := util.GetMapValueByKeys([]string{"min_uptime", "value"}, bkVal); ok { + //mark node status as offline when uptime less than 10m + if v, ok := minUptime.(float64); ok && v >= 600000 { + recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"}) + }else{ + recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"}) + } + } + } + } + } + } + } + return recentStatus, nil +} + +func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + + resBody := map[string] interface{}{} + id := ps.ByName("id") + nodeUUID := ps.ByName("node_id") + q := &orm.Query{ Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.labels.cluster_id", id), + orm.Eq("metadata.labels.node_id", nodeUUID), + orm.Eq("metadata.name", "node_routing_table"), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + namesM := util.MapStr{} + if len(result.Result) > 0 { + if data, ok := result.Result[0].(map[string]interface{}); ok { + if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { + if rows, ok := routingTable.([]interface{}); ok{ + for _, row := range rows { + if v, ok := row.(map[string]interface{}); ok { + if indexName, ok := v["index"].(string); ok{ + namesM[indexName] = true + } + } + } + } + } + } + } + + indexNames := make([]interface{}, 0, len(namesM) ) + for name, _ := range namesM { + indexNames = append(indexNames, name) + } + + q1 := &orm.Query{ Size: 100} + q1.AddSort("timestamp", orm.DESC) + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + orm.In("metadata.index_name", indexNames), + orm.NotEq("metadata.labels.index_status", "deleted"), + ) + err, result = orm.Search(elastic.IndexConfig{}, q1) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + + indices, err := h.getLatestIndices(req, min, max, id, &result) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + + h.WriteJSON(w, indices, http.StatusOK) +} + +type ShardsSummary struct { + Index string `json:"index"` + Shards int `json:"shards"` + Replicas int `json:"replicas"` + DocsCount int64 `json:"docs_count"` + DocsDeleted int64 `json:"docs_deleted"` + StoreInBytes int64 `json:"store_in_bytes"` + PriStoreInBytes int64 `json:"pri_store_in_bytes"` + Timestamp interface{} `json:"timestamp"` +} +func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) { + //filter indices + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return []interface{}{}, nil + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + + query := util.MapStr{ + "size": 10000, + "_source": []string{"metadata.labels.index_name", "payload.elasticsearch.shard_stats.docs","payload.elasticsearch.shard_stats.store", "payload.elasticsearch.shard_stats.routing", "timestamp"}, + "collapse": util.MapStr{ + "field": "metadata.labels.shard_id", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "shard_stats", + }, + }, + }, + }, + }, + }, + } + q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + err, searchResult := orm.Search(event.Event{}, q) + if err != nil { + return nil, err + } + indexInfos := map[string]*ShardsSummary{} + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardDocCount, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, hitM) + storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) + if v, ok := indexName.(string); ok { + if _, ok = indexInfos[v]; !ok { + indexInfos[v] = &ShardsSummary{} + } + indexInfo := indexInfos[v] + indexInfo.Index = v + if count, ok := shardDocCount.(float64); ok && primary == true { + indexInfo.DocsCount += int64(count) + } + if storeSize, ok := storeInBytes.(float64); ok { + indexInfo.StoreInBytes += int64(storeSize) + } + if primary == true { + indexInfo.Shards++ + }else{ + indexInfo.Replicas++ + } + indexInfo.Timestamp = hitM["timestamp"] + } + } + } + indices := []interface{}{} + var indexPattern *radix.Pattern + if !hasAllPrivilege{ + indexPattern = radix.Compile(allowedIndices...) + } + + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "index_name"}, hitM) + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, hitM) + health, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, hitM) + if state == "delete" { + health = "N/A" + } + shards, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_shards"}, hitM) + replicas, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_replicas"}, hitM) + shardsNum, _ := util.ToInt(shards.(string)) + replicasNum, _ := util.ToInt(replicas.(string)) + if v, ok := indexName.(string); ok { + if indexPattern != nil { + if !indexPattern.Match(v) { + continue + } + } + if indexInfos[v] != nil { + indices = append(indices, util.MapStr{ + "index": v, + "status": state, + "health": health, + "timestamp": indexInfos[v].Timestamp, + "docs_count": indexInfos[v].DocsCount, + "shards": indexInfos[v].Shards, + "replicas": replicasNum, + "unassigned_shards": (replicasNum + 1) * shardsNum - indexInfos[v].Shards - replicasNum, + "store_size": util.FormatBytes(float64(indexInfos[v].StoreInBytes), 1), + }) + } else { + indices = append(indices, util.MapStr{ + "index": v, + "status": state, + "health": health, + "timestamp": hitM["timestamp"], + }) + } + } + } + } + return indices, nil +} + + +func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + if GetMonitorState(clusterID) == Console { + h.APIHandler.GetNodeShards(w, req, ps) + return + } + nodeID := ps.MustGetParameter("node_id") + q1 := orm.Query{ + Size: 1000, + WildcardIndex: true, + CollapseField: "metadata.labels.shard_id", + } + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "shard_stats"), + orm.Eq("metadata.labels.node_id", nodeID), + orm.Eq("metadata.labels.cluster_uuid", clusterUUID), + orm.Ge("timestamp", "now-15m"), + ) + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError ) + return + } + var shards = []interface{}{} + if len(result.Result) > 0 { + qps, err := h.getShardQPS(clusterID, nodeID, "", 20) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + for _, item := range result.Result { + row, ok := item.(map[string]interface{}) + if ok { + source := util.MapStr(row) + shardV, err := source.GetValue("payload.elasticsearch.shard_stats") + if err != nil { + log.Error(err) + continue + } + shardInfo := util.MapStr{} + shardInfo["id"], _ = source.GetValue("metadata.labels.node_id") + shardInfo["index"], _ = source.GetValue("metadata.labels.index_name") + shardInfo["ip"], _ = source.GetValue("metadata.labels.ip") + shardInfo["node"], _ = source.GetValue("metadata.labels.node_name") + shardInfo["shard"], _ = source.GetValue("metadata.labels.shard") + shardInfo["shard_id"], _ = source.GetValue("metadata.labels.shard_id") + if v, ok := shardV.(map[string]interface{}); ok { + shardM := util.MapStr(v) + shardInfo["docs"], _ = shardM.GetValue("docs.count") + primary, _ := shardM.GetValue("routing.primary") + if primary == true { + shardInfo["prirep"] = "p" + }else{ + shardInfo["prirep"] = "r" + } + shardInfo["state"], _ = shardM.GetValue("routing.state") + shardInfo["store_in_bytes"], _ = shardM.GetValue("store.size_in_bytes") + } + + if v, ok := shardInfo["shard_id"].(string); ok { + shardInfo["index_qps"] = qps[v]["index"] + shardInfo["query_qps"] = qps[v]["query"] + shardInfo["index_bytes_qps"] = qps[v]["index_bytes"] + } + shards = append(shards, shardInfo) + } + } + } + + h.WriteJSON(w, shards, http.StatusOK) +} \ No newline at end of file diff --git a/modules/elastic/api/proxy.go b/modules/elastic/api/proxy.go new file mode 100644 index 00000000..56f463b8 --- /dev/null +++ b/modules/elastic/api/proxy.go @@ -0,0 +1,283 @@ +package api + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "github.com/buger/jsonparser" + log "github.com/cihub/seelog" + "github.com/segmentio/encoding/json" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/fasthttp" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +var httpPool = fasthttp.NewRequestResponsePool("proxy_search") + +func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + targetClusterID := ps.ByName("id") + method := h.GetParameterOrDefault(req, "method", "") + path := h.GetParameterOrDefault(req, "path", "") + if method == "" || path == "" { + resBody["error"] = fmt.Errorf("parameter method and path is required") + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + exists, esClient, err := h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + authPath, _ := url.PathUnescape(path) + var realPath = authPath + newURL, err := url.Parse(realPath) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if strings.Trim(newURL.Path, "/") == "_sql" { + distribution := esClient.GetVersion().Distribution + indexName, err := rewriteTableNamesOfSqlRequest(req, distribution) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !h.IsIndexAllowed(req, targetClusterID, indexName) { + h.WriteError(w, fmt.Sprintf("forbidden to access index %s", indexName), http.StatusForbidden) + return + } + q, _ := url.ParseQuery(newURL.RawQuery) + hasFormat := q.Has("format") + switch distribution { + case elastic.Opensearch: + path = "_plugins/_sql?format=raw" + case elastic.Easysearch: + if !hasFormat { + q.Add("format", "raw") + } + path = "_sql?" + q.Encode() + default: + if !hasFormat { + q.Add("format", "txt") + } + path = "_sql?" + q.Encode() + } + } + //ccs search + if parts := strings.SplitN(authPath, "/", 2); strings.Contains(parts[0], ":") { + ccsParts := strings.SplitN(parts[0], ":", 2) + realPath = fmt.Sprintf("%s/%s", ccsParts[1], parts[1]) + } + newReq := req.Clone(context.Background()) + newReq.URL = newURL + newReq.Method = method + isSuperAdmin, permission, err := h.ValidateProxyRequest(newReq, targetClusterID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusForbidden) + return + } + if permission == "" && api.IsAuthEnable() && !isSuperAdmin { + resBody["error"] = "unknown request path" + h.WriteJSON(w, resBody, http.StatusForbidden) + return + } + //if permission != "" { + // if permission == "cat.indices" || permission == "cat.shards" { + // reqUrl.Path + // } + //} + + var ( + freq = httpPool.AcquireRequest() + fres = httpPool.AcquireResponse() + ) + defer func() { + httpPool.ReleaseRequest(freq) + httpPool.ReleaseResponse(fres) + }() + metadata := elastic.GetMetadata(targetClusterID) + if metadata == nil { + resBody["error"] = fmt.Sprintf("cluster [%s] metadata not found", targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + if metadata.Config.BasicAuth != nil { + freq.SetBasicAuth(metadata.Config.BasicAuth.Username, metadata.Config.BasicAuth.Password.Get()) + } + + endpoint := util.JoinPath(metadata.GetActivePreferredSeedEndpoint(), path) + + freq.SetRequestURI(endpoint) + method = strings.ToUpper(method) + freq.Header.SetMethod(method) + freq.Header.SetUserAgent(req.Header.Get("user-agent")) + freq.Header.SetReferer(endpoint) + rurl, _ := url.Parse(endpoint) + + if rurl != nil { + freq.Header.SetHost(rurl.Host) + freq.Header.SetRequestURI(rurl.RequestURI()) + } + + clonedURI := freq.CloneURI() + defer fasthttp.ReleaseURI(clonedURI) + clonedURI.SetScheme(metadata.GetSchema()) + freq.SetURI(clonedURI) + + if permission == "cluster.search" { + indices, hasAll := h.GetAllowedIndices(req, targetClusterID) + if !hasAll && len(indices) == 0 { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + if hasAll { + freq.SetBodyStream(req.Body, int(req.ContentLength)) + } else { + body, err := io.ReadAll(req.Body) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(body) == 0 { + body = []byte("{}") + } + v, _, _, _ := jsonparser.Get(body, "query") + newQ := bytes.NewBuffer([]byte(`{"bool": {"must": [{"terms": {"_index":`)) + indicesBytes := util.MustToJSONBytes(indices) + newQ.Write(indicesBytes) + newQ.Write([]byte("}}")) + if len(v) > 0 { + newQ.Write([]byte(",")) + newQ.Write(v) + } + newQ.Write([]byte(`]}}`)) + body, _ = jsonparser.Set(body, newQ.Bytes(), "query") + freq.SetBody(body) + } + } else { + freq.SetBodyStream(req.Body, int(req.ContentLength)) + } + defer req.Body.Close() + + err = getHttpClient().Do(freq, fres) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + okBody := struct { + RequestHeader string `json:"request_header"` + ResponseHeader string `json:"response_header"` + ResponseBody string `json:"response_body"` + }{ + RequestHeader: freq.Header.String(), + ResponseHeader: fres.Header.String(), + ResponseBody: string(fres.GetRawBody()), + } + + w.Header().Set("Content-type", string(fres.Header.ContentType())) + w.WriteHeader(fres.StatusCode()) + json.NewEncoder(w).Encode(okBody) + +} + +func rewriteTableNamesOfSqlRequest(req *http.Request, distribution string) (string, error) { + var buf bytes.Buffer + if _, err := buf.ReadFrom(req.Body); err != nil { + return "", err + } + if err := req.Body.Close(); err != nil { + return "", err + } + req.Body = io.NopCloser(bytes.NewReader(buf.Bytes())) + sqlQuery, err := jsonparser.GetString(buf.Bytes(), "query") + if err != nil { + return "", fmt.Errorf("parse query from request body error: %w", err) + } + q := util.NewSQLQueryString(sqlQuery) + tableNames, err := q.TableNames() + if err != nil { + return "", err + } + rewriteBody := false + switch distribution { + case elastic.Elasticsearch: + for _, tname := range tableNames { + if strings.ContainsAny(tname, "-.") && !strings.HasPrefix(tname, "\"") { + //append quotes from table name + sqlQuery = strings.Replace(sqlQuery, tname, fmt.Sprintf(`\"%s\"`, tname), -1) + rewriteBody = true + } + } + case elastic.Opensearch, elastic.Easysearch: + for _, tname := range tableNames { + //remove quotes from table name + if strings.HasPrefix(tname, "\"") || strings.HasSuffix(tname, "\"") { + sqlQuery = strings.Replace(sqlQuery, tname, strings.Trim(tname, "\""), -1) + rewriteBody = true + } + } + } + if rewriteBody { + sqlQuery = fmt.Sprintf(`"%s"`, sqlQuery) + reqBody, _ := jsonparser.Set(buf.Bytes(), []byte(sqlQuery), "query") + req.Body = io.NopCloser(bytes.NewReader(reqBody)) + req.ContentLength = int64(len(reqBody)) + } + var unescapedTableNames []string + for _, tname := range tableNames { + unescapedTableNames = append(unescapedTableNames, strings.Trim(tname, "\"")) + } + return strings.Join(unescapedTableNames, ","), nil +} + +var ( + client *fasthttp.Client + clientOnce sync.Once +) + +func getHttpClient() *fasthttp.Client { + clientOnce.Do(func() { + clientCfg := global.Env().SystemConfig.HTTPClientConfig + client = &fasthttp.Client{ + MaxConnsPerHost: clientCfg.MaxConnectionPerHost, + TLSConfig: &tls.Config{InsecureSkipVerify: clientCfg.TLSConfig.TLSInsecureSkipVerify}, + ReadTimeout: util.GetDurationOrDefault(clientCfg.ReadTimeout, 60*time.Second), + WriteTimeout: util.GetDurationOrDefault(clientCfg.ReadTimeout, 60*time.Second), + DialDualStack: true, + ReadBufferSize: clientCfg.ReadBufferSize, + WriteBufferSize: clientCfg.WriteBufferSize, + //Dial: fasthttpproxy.FasthttpProxyHTTPDialerTimeout(time.Second * 2), + } + }) + return client +} diff --git a/modules/elastic/api/search.go b/modules/elastic/api/search.go new file mode 100644 index 00000000..c7aab878 --- /dev/null +++ b/modules/elastic/api/search.go @@ -0,0 +1,410 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "github.com/segmentio/encoding/json" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "strconv" + "strings" + "time" +) + +func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + var template = &elastic.SearchTemplate{} + + err = h.DecodeJSON(req, template) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var body = map[string]interface{}{ + "script": map[string]interface{}{ + "lang": "mustache", + "source": template.Source, + }, + } + bodyBytes, _ := json.Marshal(body) + + //fmt.Println(client) + err = client.SetSearchTemplate(template.Name, bodyBytes) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + id := util.GetUUID() + template.Created = time.Now() + template.Updated = template.Created + template.ClusterID = targetClusterID + index:=orm.GetIndexName(elastic.SearchTemplate{}) + insertRes, err := esClient.Index(index, "", id, template, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + resBody["_source"] = template + resBody["_id"] = id + resBody["result"] = insertRes.Result + + h.WriteJSON(w, resBody,http.StatusOK) +} + +func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + var template = &elastic.SearchTemplate{} + + err = h.DecodeJSON(req, template) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + templateID := ps.ByName("template_id") + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + index:=orm.GetIndexName(elastic.SearchTemplate{}) + getRes, err := esClient.Get(index, "",templateID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if getRes.Found == false { + resBody["error"] = fmt.Sprintf("template %s can not be found", templateID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + originTemplate := getRes.Source + targetTemplate := make(map[string]interface{}, len(originTemplate)) + for k, v := range originTemplate { + targetTemplate[k] = v + } + targetName := originTemplate["name"].(string) + if template.Name != "" && template.Name != targetName { + err = client.DeleteSearchTemplate(targetName) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + targetTemplate["name"] = template.Name + targetName = template.Name + } + if template.Source != "" { + targetTemplate["source"] = template.Source + } + var body = map[string]interface{}{ + "script": map[string]interface{}{ + "lang": "mustache", + "source": targetTemplate["source"], + }, + } + bodyBytes, _ := json.Marshal(body) + + err = client.SetSearchTemplate(targetName, bodyBytes) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + targetTemplate["updated"] = time.Now() + insertRes, err := esClient.Index(index, "", templateID, targetTemplate, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + ht := &elastic.SearchTemplateHistory{ + TemplateID: templateID, + Action: "update", + Content: originTemplate, + Created: time.Now(), + } + esClient.Index(orm.GetIndexName(ht), "", util.GetUUID(), ht, "") + + resBody["_source"] = originTemplate + resBody["_id"] = templateID + resBody["result"] = insertRes.Result + + h.WriteJSON(w, resBody,http.StatusOK) +} + +func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + templateID := ps.ByName("template_id") + + index:=orm.GetIndexName(elastic.SearchTemplate{}) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + res, err := esClient.Get(index, "", templateID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + err = client.DeleteSearchTemplate(res.Source["name"].(string)) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + delRes, err := esClient.Delete(index, "", res.ID) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + ht := &elastic.SearchTemplateHistory{ + TemplateID: templateID, + Action: "delete", + Content: res.Source, + Created: time.Now(), + } + _, err = esClient.Index(orm.GetIndexName(ht), "", util.GetUUID(), ht, "wait_for") + if err != nil { + log.Error(err) + } + + resBody["_id"] = templateID + resBody["result"] = delRes.Result + h.WriteJSON(w, resBody, delRes.StatusCode) + +} + +func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + var ( + name = h.GetParameterOrDefault(req, "name", "") + strFrom = h.GetParameterOrDefault(req, "from", "0") + strSize = h.GetParameterOrDefault(req, "size", "20") + queryDSL = `{"query":{"bool":{"must":[%s]}},"from": %d, "size": %d}` + mustBuilder = &strings.Builder{} + ) + from, _ := strconv.Atoi(strFrom) + size, _ := strconv.Atoi(strSize) + targetClusterID := ps.ByName("id") + mustBuilder.WriteString(fmt.Sprintf(`{"match":{"cluster_id": "%s"}}`, targetClusterID)) + if name != ""{ + mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"name": "%s"}}`, name)) + } + + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), from, size) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.SearchTemplate{}), []byte(queryDSL)) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + h.WriteJSON(w, res, http.StatusOK) +} + +func (h *APIHandler) HandleGetSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{} + + id := ps.ByName("template_id") + indexName := orm.GetIndexName(elastic.SearchTemplate{}) + getResponse, err := h.Client().Get(indexName, "", id) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + if getResponse!=nil{ + h.WriteJSON(w, resBody, getResponse.StatusCode) + }else{ + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + return + } + h.WriteJSON(w,getResponse,200) +} + +func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + var ( + templateID = h.GetParameterOrDefault(req, "template_id", "") + strFrom = h.GetParameterOrDefault(req, "from", "0") + strSize = h.GetParameterOrDefault(req, "size", "20") + queryDSL = `{"query":{"bool":{"must":[%s]}},"from": %d, "size": %d}` + mustBuilder = &strings.Builder{} + ) + from, _ := strconv.Atoi(strFrom) + size, _ := strconv.Atoi(strSize) + targetClusterID := ps.ByName("id") + mustBuilder.WriteString(fmt.Sprintf(`{"match":{"content.cluster_id": "%s"}}`, targetClusterID)) + if templateID != ""{ + mustBuilder.WriteString(fmt.Sprintf(`,{"match":{"template_id": "%s"}}`, templateID)) + } + + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), from, size) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.SearchTemplateHistory{}), []byte(queryDSL)) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + h.WriteJSON(w, res, http.StatusOK) +} + +func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + reqBody := map[string]interface{}{} + err = h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + res, err := client.RenderTemplate(reqBody) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + h.WriteJSON(w, string(res), http.StatusOK) +} + +func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + reqBody := map[string]interface{}{} + err = h.DecodeJSON(req, &reqBody) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + res, err := client.SearchTemplate(reqBody) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + h.WriteJSON(w, string(res), http.StatusOK) +} \ No newline at end of file diff --git a/modules/elastic/api/setting.go b/modules/elastic/api/setting.go new file mode 100644 index 00000000..b10e154b --- /dev/null +++ b/modules/elastic/api/setting.go @@ -0,0 +1,78 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "time" +) + +func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{ + } + targetClusterID := ps.ByName("id") + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + var reqParams = elastic.Setting{ + UpdatedAt: time.Now(), + ClusterID: targetClusterID, + } + + err := h.DecodeJSON(req, &reqParams) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + indexName := orm.GetIndexName(reqParams) + queryDSL := fmt.Sprintf(`{"size":1,"query":{"bool":{"must":[{"match":{"key":"%s"}},{"match":{"cluster_id":"%s"}}]}}}`, reqParams.Key, targetClusterID) + searchRes, err := esClient.SearchWithRawQueryDSL(indexName, []byte(queryDSL)) + if len(searchRes.Hits.Hits) > 0 { + _, err = esClient.Index(indexName, "", searchRes.Hits.Hits[0].ID, reqParams, "wait_for") + }else{ + reqParams.ID = util.GetUUID() + _, err = esClient.Index(indexName, "", reqParams.ID, reqParams, "wait_for") + } + + + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + resBody["acknowledged"] = true + h.WriteJSON(w, resBody ,http.StatusOK) +} + +func (h *APIHandler) HandleGetSettingAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{ + } + targetClusterID := ps.ByName("id") + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + var key = ps.ByName("key") + + queryDSL := fmt.Sprintf(`{"size":1,"query":{"bool":{"must":[{"match":{"key":"%s"}},{"match":{"cluster_id":"%s"}}]}}}`, key, targetClusterID) + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.Setting{}), []byte(queryDSL)) + + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var value interface{} + if len(searchRes.Hits.Hits) > 0 { + value = searchRes.Hits.Hits[0].Source["value"] + }else{ + value = "" + } + h.WriteJSON(w, value ,http.StatusOK) +} diff --git a/modules/elastic/api/shard.go b/modules/elastic/api/shard.go new file mode 100644 index 00000000..718a6be8 --- /dev/null +++ b/modules/elastic/api/shard.go @@ -0,0 +1,46 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "infini.sh/framework/core/event" + "infini.sh/framework/core/orm" + "infini.sh/framework/modules/elastic/adapter" + "net/http" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" +) + +func (h *APIHandler) GetShardInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + shardID := ps.MustGetParameter("shard_id") + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + q := orm.Query{ + Size: 1, + } + q.Conds = orm.And( + orm.Eq("metadata.labels.shard_id", shardID), + orm.Eq("metadata.labels.cluster_uuid", clusterUUID), + ) + q.AddSort("timestamp", orm.DESC) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(res.Result) == 0 { + h.WriteJSON(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + h.WriteJSON(w, res.Result[0], http.StatusOK) +} diff --git a/modules/elastic/api/template.go b/modules/elastic/api/template.go new file mode 100644 index 00000000..92ea8a97 --- /dev/null +++ b/modules/elastic/api/template.go @@ -0,0 +1,52 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "io" + "net/http" + "src/github.com/buger/jsonparser" +) + +func (h *APIHandler) HandleGetTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + clusterID := ps.MustGetParameter("id") + esClient := elastic.GetClient(clusterID) + templates, err := esClient.GetTemplate("") + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, templates, http.StatusOK) +} + +func (h *APIHandler) HandleSaveTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + clusterID := ps.MustGetParameter("id") + templateName := ps.MustGetParameter("template_name") + esClient := elastic.GetClient(clusterID) + reqBody, err := io.ReadAll(req.Body) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + esResBody, err := esClient.PutTemplate(templateName, reqBody) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + resErr, _, _, _ := jsonparser.Get(esResBody, "error") + if resErr != nil { + errStr := string(resErr) + log.Errorf("put template error: %s", errStr) + h.WriteError(w, errStr, http.StatusInternalServerError) + return + } + h.WriteAckOKJSON(w) +} \ No newline at end of file diff --git a/modules/elastic/api/test_connection.go b/modules/elastic/api/test_connection.go new file mode 100644 index 00000000..8a098d53 --- /dev/null +++ b/modules/elastic/api/test_connection.go @@ -0,0 +1,162 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "fmt" + "github.com/segmentio/encoding/json" + util2 "infini.sh/agent/lib/util" + "infini.sh/console/core" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "strings" + "time" +) + +type TestAPI struct { + core.Handler +} + +var testAPI = TestAPI{} + +var testInited bool + +func InitTestAPI() { + if !testInited { + api.HandleAPIMethod(api.POST, "/elasticsearch/try_connect", testAPI.HandleTestConnectionAction) + testInited = true + } +} + +func (h TestAPI) HandleTestConnectionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + freq = httpPool.AcquireRequest() + fres = httpPool.AcquireResponse() + resBody = map[string]interface{}{} + ) + defer func() { + httpPool.ReleaseRequest(freq) + httpPool.ReleaseResponse(fres) + }() + var config = &elastic.ElasticsearchConfig{} + err := h.DecodeJSON(req, &config) + if err != nil { + panic(err) + } + defer req.Body.Close() + var url string + if config.Endpoint != "" { + url = config.Endpoint + } else if config.Host != "" && config.Schema != "" { + url = fmt.Sprintf("%s://%s", config.Schema, config.Host) + config.Endpoint = url + } else { + resBody["error"] = fmt.Sprintf("invalid config: %v", util.MustToJSON(config)) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if url == "" { + panic(errors.Error("invalid url: " + util.MustToJSON(config))) + } + + if !util.SuffixStr(url, "/") { + url = fmt.Sprintf("%s/", url) + } + + freq.SetRequestURI(url) + freq.Header.SetMethod("GET") + + if (config.BasicAuth == nil || (config.BasicAuth != nil && config.BasicAuth.Username == "")) && + config.CredentialID != "" && config.CredentialID != "manual" { + credential, err := common.GetCredential(config.CredentialID) + if err != nil { + panic(err) + } + var dv interface{} + dv, err = credential.Decode() + if err != nil { + panic(err) + } + if auth, ok := dv.(model.BasicAuth); ok { + config.BasicAuth = &auth + } + } + + if config.BasicAuth != nil && strings.TrimSpace(config.BasicAuth.Username) != "" { + freq.SetBasicAuth(config.BasicAuth.Username, config.BasicAuth.Password.Get()) + } + + err = getHttpClient().DoTimeout(freq, fres, 10*time.Second) + + if err != nil { + panic(err) + } + + var statusCode = fres.StatusCode() + if statusCode > 300 || statusCode == 0 { + resBody["error"] = fmt.Sprintf("invalid status code: %d", statusCode) + h.WriteJSON(w, resBody, 500) + return + } + + b := fres.Body() + clusterInfo := &elastic.ClusterInformation{} + err = json.Unmarshal(b, clusterInfo) + if err != nil { + panic(err) + } + + resBody["version"] = clusterInfo.Version.Number + resBody["cluster_uuid"] = clusterInfo.ClusterUUID + resBody["cluster_name"] = clusterInfo.ClusterName + resBody["distribution"] = clusterInfo.Version.Distribution + + //fetch cluster health info + freq.SetRequestURI(fmt.Sprintf("%s/_cluster/health", config.Endpoint)) + fres.Reset() + err = getHttpClient().Do(freq, fres) + if err != nil { + resBody["error"] = fmt.Sprintf("error on get cluster health: %v", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + healthInfo := &elastic.ClusterHealth{} + err = json.Unmarshal(fres.Body(), &healthInfo) + if err != nil { + resBody["error"] = fmt.Sprintf("error on decode cluster health info : %v", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + resBody["status"] = healthInfo.Status + resBody["number_of_nodes"] = healthInfo.NumberOfNodes + resBody["number_of_data_nodes"] = healthInfo.NumberOf_data_nodes + resBody["active_shards"] = healthInfo.ActiveShards + + //fetch local node's info + nodeID, nodeInfo, err := util2.GetLocalNodeInfo(config.GetAnyEndpoint(), config.BasicAuth) + if err != nil { + resBody["error"] = fmt.Sprintf("error on decode cluster health info : %v", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + resBody["status"] = healthInfo.Status + resBody["number_of_nodes"] = healthInfo.NumberOfNodes + resBody["number_of_data_nodes"] = healthInfo.NumberOf_data_nodes + resBody["active_shards"] = healthInfo.ActiveShards + resBody["node_uuid"] = nodeID + resBody["node_info"] = nodeInfo + + h.WriteJSON(w, resBody, http.StatusOK) + +} diff --git a/modules/elastic/api/threadpool_metrics.go b/modules/elastic/api/threadpool_metrics.go new file mode 100644 index 00000000..9e83f4a1 --- /dev/null +++ b/modules/elastic/api/threadpool_metrics.go @@ -0,0 +1,556 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "strings" +) + +const ( + ThreadPoolGetGroupKey = "thread_pool_get" + ThreadPoolSearchGroupKey = "thread_pool_search" + ThreadPoolFlushGroupKey = "thread_pool_flush" + ThreadPoolRefreshGroupKey = "thread_pool_refresh" + ThreadPoolWriteGroupKey = "thread_pool_write" + ThreadPoolForceMergeGroupKey = "thread_pool_force_merge" + ThreadPoolIndexGroupKey = "thread_pool_index" + ThreadPoolBulkGroupKey = "thread_pool_bulk" +) + +func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) (map[string]*common.MetricItem, error){ + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + } + var ( + nodeNames []string + ) + if nodeName != "" { + nodeNames = strings.Split(nodeName, ",") + top = len(nodeNames) + }else{ + nodeNames, err = h.getTopNodeName(clusterID, top, 15) + if err != nil { + log.Error(err) + } + } + if len(nodeNames) > 0 { + must = append(must, util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.transport_address": nodeNames, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeNames, + }, + }, + }, + }, + + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + searchThreadsMetric := newMetricItem("search_threads", 1, ThreadPoolSearchGroupKey) + searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems := []GroupMetricItem{ + { + Key: "search_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.search.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchThreadsMetric, + FormatType: "num", + Units: "", + }, + } + searchQueueMetric := newMetricItem("search_queue", 1, ThreadPoolSearchGroupKey) + searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.search.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchQueueMetric, + FormatType: "num", + Units: "", + }) + searchActiveMetric := newMetricItem("search_active", 1, ThreadPoolSearchGroupKey) + searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_active", + Field: "payload.elasticsearch.node_stats.thread_pool.search.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchActiveMetric, + FormatType: "num", + Units: "", + }) + searchRejectedMetric := newMetricItem("search_rejected", 1, ThreadPoolSearchGroupKey) + searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.search.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: searchRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + getThreadsMetric := newMetricItem("get_threads", 1, ThreadPoolGetGroupKey) + getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.get.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getThreadsMetric, + FormatType: "num", + Units: "", + }) + getQueueMetric := newMetricItem("get_queue", 1, ThreadPoolGetGroupKey) + getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.get.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getQueueMetric, + FormatType: "num", + Units: "", + }) + getActiveMetric := newMetricItem("get_active", 1, ThreadPoolGetGroupKey) + getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_active", + Field: "payload.elasticsearch.node_stats.thread_pool.get.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getActiveMetric, + FormatType: "num", + Units: "", + }) + getRejectedMetric := newMetricItem("get_rejected", 1, ThreadPoolGetGroupKey) + getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.get.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: getRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + flushThreadsMetric := newMetricItem("flush_threads", 1, ThreadPoolFlushGroupKey) + flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushThreadsMetric, + FormatType: "num", + Units: "", + }) + flushQueueMetric := newMetricItem("flush_queue", 1, ThreadPoolFlushGroupKey) + flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushQueueMetric, + FormatType: "num", + Units: "", + }) + flushActiveMetric := newMetricItem("flush_active", 1, ThreadPoolFlushGroupKey) + flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_active", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushActiveMetric, + FormatType: "num", + Units: "", + }) + flushRejectedMetric := newMetricItem("flush_rejected", 1, ThreadPoolFlushGroupKey) + flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + majorVersion := elastic.GetMetadata(clusterID).GetMajorVersion() + ver := elastic.GetClient(clusterID).GetVersion() + + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && majorVersion < 6{ + indexThreadsMetric := newMetricItem("index_threads", 1, ThreadPoolIndexGroupKey) + indexThreadsMetric.AddAxi("Index Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.index.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexThreadsMetric, + FormatType: "num", + Units: "", + }) + indexQueueMetric := newMetricItem("index_queue", 1, ThreadPoolIndexGroupKey) + indexQueueMetric.AddAxi("Index Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.index.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexQueueMetric, + FormatType: "num", + Units: "", + }) + indexActiveMetric := newMetricItem("index_active", 1, ThreadPoolIndexGroupKey) + indexActiveMetric.AddAxi("Index Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_active", + Field: "payload.elasticsearch.node_stats.thread_pool.index.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexActiveMetric, + FormatType: "num", + Units: "", + }) + indexRejectedMetric := newMetricItem("index_rejected", 1, ThreadPoolIndexGroupKey) + indexRejectedMetric.AddAxi("Index Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.index.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + bulkThreadsMetric := newMetricItem("bulk_threads", 1, ThreadPoolBulkGroupKey) + bulkThreadsMetric.AddAxi("Bulk Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkThreadsMetric, + FormatType: "num", + Units: "", + }) + bulkQueueMetric := newMetricItem("bulk_queue", 1, ThreadPoolBulkGroupKey) + bulkQueueMetric.AddAxi("Bulk Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkQueueMetric, + FormatType: "num", + Units: "", + }) + bulkActiveMetric := newMetricItem("bulk_active", 1, ThreadPoolBulkGroupKey) + bulkActiveMetric.AddAxi("Bulk Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_active", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkActiveMetric, + FormatType: "num", + Units: "", + }) + bulkRejectedMetric := newMetricItem("bulk_rejected", 1, ThreadPoolBulkGroupKey) + bulkRejectedMetric.AddAxi("Bulk Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: bulkRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + }else { + writeThreadsMetric := newMetricItem("write_threads", 1, ThreadPoolWriteGroupKey) + writeThreadsMetric.AddAxi("Write Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.write.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeThreadsMetric, + FormatType: "num", + Units: "", + }) + writeQueueMetric := newMetricItem("write_queue", 1, ThreadPoolWriteGroupKey) + writeQueueMetric.AddAxi("Write Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.write.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeQueueMetric, + FormatType: "num", + Units: "", + }) + writeActiveMetric := newMetricItem("write_active", 1, ThreadPoolWriteGroupKey) + writeActiveMetric.AddAxi("Write Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_active", + Field: "payload.elasticsearch.node_stats.thread_pool.write.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeActiveMetric, + FormatType: "num", + Units: "", + }) + writeRejectedMetric := newMetricItem("write_rejected", 1, ThreadPoolWriteGroupKey) + writeRejectedMetric.AddAxi("Write Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.write.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: writeRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + } + refreshThreadsMetric := newMetricItem("refresh_threads", 1, ThreadPoolRefreshGroupKey) + refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshThreadsMetric, + FormatType: "num", + Units: "", + }) + refreshQueueMetric := newMetricItem("refresh_queue", 1, ThreadPoolRefreshGroupKey) + refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshQueueMetric, + FormatType: "num", + Units: "", + }) + refreshActiveMetric := newMetricItem("refresh_active", 1, ThreadPoolRefreshGroupKey) + refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_active", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshActiveMetric, + FormatType: "num", + Units: "", + }) + refreshRejectedMetric := newMetricItem("refresh_rejected", 1, ThreadPoolRefreshGroupKey) + refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + forceMergeThreadsMetric := newMetricItem("force_merge_threads", 1, ThreadPoolForceMergeGroupKey) + forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeThreadsMetric, + FormatType: "num", + Units: "", + }) + forceMergeQueueMetric := newMetricItem("force_merge_queue", 1, ThreadPoolForceMergeGroupKey) + forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeQueueMetric, + FormatType: "num", + Units: "", + }) + forceMergeActiveMetric := newMetricItem("force_merge_active", 1, ThreadPoolForceMergeGroupKey) + forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_active", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeActiveMetric, + FormatType: "num", + Units: "", + }) + forceMergeRejectedMetric := newMetricItem("force_merge_rejected", 1, ThreadPoolForceMergeGroupKey) + forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: forceMergeRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + //Get Thread Pool queue + aggs:=map[string]interface{}{} + + for _,metricItem:=range queueMetricItems{ + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID + "_field2"]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field2, + }, + } + } + + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID + "_field2_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": top, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + }, + }, + } + return h.getMetrics(query, queueMetricItems, bucketSize), nil +} diff --git a/modules/elastic/api/trace_template.go b/modules/elastic/api/trace_template.go new file mode 100644 index 00000000..815f66a7 --- /dev/null +++ b/modules/elastic/api/trace_template.go @@ -0,0 +1,168 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/http" + "strconv" + "strings" + "time" +) + +func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string] interface{}{ + } + targetClusterID := ps.ByName("id") + exists,client,err:=h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists{ + resBody["error"] = fmt.Sprintf("cluster [%s] not found",targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + var traceReq = &elastic.TraceTemplate{ + + } + + err = h.DecodeJSON(req, traceReq) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + traceReq.Created = time.Now() + traceReq.Updated = traceReq.Created + traceReq.ClusterID = targetClusterID + + var id = util.GetUUID() + insertRes, err := client.Index(orm.GetIndexName(elastic.TraceTemplate{}), "", id, traceReq, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + resBody["_source"] = traceReq + resBody["_id"] = insertRes.ID + resBody["result"] = insertRes.Result + + h.WriteJSON(w, resBody,http.StatusOK) +} + +func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string] interface{}{ + } + var ( + name = h.GetParameterOrDefault(req, "name", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + targetClusterID := ps.ByName("id") + mustBuilder.WriteString(fmt.Sprintf(`{"term":{"cluster_id":{"value": "%s"}}}`, targetClusterID)) + if name != ""{ + mustBuilder.WriteString(fmt.Sprintf(`,{"prefix":{"name": "%s"}}`, name)) + } + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.TraceTemplate{}), []byte(queryDSL)) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + h.WriteJSON(w, res, http.StatusOK) +} + +func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{ + } + + reqParams := elastic.TraceTemplate{} + err := h.DecodeJSON(req, &reqParams) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + reqParams.ID = ps.ByName("template_id") + reqParams.Updated = time.Now() + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + _, err = esClient.Index(orm.GetIndexName(reqParams),"", reqParams.ID, reqParams, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusOK) + return + } + + resBody["_id"] = reqParams.ID + resBody["result"] = "updated" + resBody["_source"] = reqParams + + h.WriteJSON(w, resBody,http.StatusOK) +} + +func (h *APIHandler) HandleGetTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ + resBody := map[string] interface{}{} + + id := ps.ByName("template_id") + indexName := orm.GetIndexName(elastic.TraceTemplate{}) + getResponse, err := h.Client().Get(indexName, "", id) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + h.WriteJSON(w,getResponse, getResponse.StatusCode) +} + +func (h *APIHandler) HandleDeleteTraceTemplateAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("template_id") + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + delRes, err := esClient.Delete(orm.GetIndexName(elastic.TraceTemplate{}), "", id, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + if delRes!=nil{ + h.WriteJSON(w, resBody, delRes.StatusCode) + }else{ + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + } + + elastic.RemoveInstance(id) + resBody["_id"] = id + resBody["result"] = delRes.Result + h.WriteJSON(w, resBody, delRes.StatusCode) +} diff --git a/modules/elastic/api/v1/cluster_overview.go b/modules/elastic/api/v1/cluster_overview.go new file mode 100644 index 00000000..56f7390f --- /dev/null +++ b/modules/elastic/api/v1/cluster_overview.go @@ -0,0 +1,1109 @@ +package v1 + +import ( + "fmt" + "net/http" + "strings" + "time" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" +) + +func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var clusterIDs = []string{} + h.DecodeJSON(req, &clusterIDs) + + if len(clusterIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + cids := make([]interface{}, 0, len(clusterIDs)) + for _, clusterID := range clusterIDs { + cids = append(cids, clusterID) + } + healthMap := map[string]interface{}{} + + //fetch extra cluster status + q1 := orm.Query{WildcardIndex: true} + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "cluster_stats"), + orm.In("metadata.labels.cluster_id", cids), + ) + q1.Collapse("metadata.labels.cluster_id") + q1.AddSort("timestamp", orm.DESC) + q1.Size = len(clusterIDs) + 1 + + err, results := orm.Search(&event.Event{}, &q1) + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + clusterID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "cluster_id"}, result) + if ok { + source := map[string]interface{}{} + indicesCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "count"}, result) + if ok { + source["number_of_indices"] = indicesCount + } + + docsCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "docs", "count"}, result) + if ok { + source["number_of_documents"] = docsCount + } + + docsDeletedCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "docs", "deleted"}, result) + if ok { + source["number_of_deleted_documents"] = docsDeletedCount + } + fs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "fs"}, result) + if ok { + source["fs"] = fs + } + jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "jvm", "mem"}, result) + if ok { + source["jvm"] = jvm + } + nodeCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "nodes", "count", "total"}, result) + if ok { + source["number_of_nodes"] = nodeCount + } + shardCount, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "cluster_stats", "indices", "shards", "total"}, result) + if ok { + source["number_of_shards"] = shardCount + } + source["timestamp"] = result["timestamp"] + + healthMap[util.ToString(clusterID)] = source + } + } + } + + //fetch cluster metrics + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, (15)) + if err != nil { + panic(err) + return + } + + query := util.MapStr{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.cluster_id": clusterIDs, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + var top = len(clusterIDs) + 1 + + metricItems := []*common.MetricItem{} + var bucketSizeStr = fmt.Sprintf("%vs", bucketSize) + metricItem := newMetricItem("cluster_summary", 2, "cluster") + indexLine := metricItem.AddLine("Indexing", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", + "payload.elasticsearch.index_stats.total.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true) + + searchLine := metricItem.AddLine("Search", "Total Search", "Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!", "group1", + "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + bucketItem := common.NewBucketItem( + common.TermsBucket, util.MapStr{ + "field": "metadata.labels.cluster_id", + "size": top, + }) + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + panic(err) + } + histgram := common.NewBucketItem( + common.DateHistogramBucket, util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }) + histgram.AddMetricItems(metricItems...) + + bucketItem.AddNestBucket(histgram) + + query["size"] = 0 + + aggs := ConvertBucketItemsToAggQuery([]*common.BucketItem{bucketItem}, nil) + + util.MergeFields(query, aggs, true) + + searchR1, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + panic(err) + } + + searchResponse := SearchResponse{} + util.FromJSONBytes(searchR1.RawResult.Body, &searchResponse) + + m1 := ParseAggregationResult(bucketSize, searchResponse.Aggregations, bucketItem.Key, histgram.Key, indexLine.Metric.GetDataKey()) + m2 := ParseAggregationResult(bucketSize, searchResponse.Aggregations, bucketItem.Key, histgram.Key, searchLine.Metric.GetDataKey()) + + //fetch recent cluster health status + bucketItem = common.NewBucketItem( + common.TermsBucket, util.MapStr{ + "field": "metadata.labels.cluster_id", + "size": top, + }) + + bucketSizeStr = "1d" + histgram = common.NewBucketItem( + common.DateRangeBucket, util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + }, { + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }) + + termBucket := common.NewBucketItem(common.TermsBucket, util.MapStr{ + "field": "payload.elasticsearch.cluster_health.status", + "size": top, + "missing": "", + }) + histgram.AddNestBucket(termBucket) + + bucketItem.AddNestBucket(histgram) + + aggs = ConvertBucketItemsToAggQuery([]*common.BucketItem{bucketItem}, nil) + query = util.MapStr{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.cluster_id": clusterIDs, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-15d", + "lte": "now", + }, + }, + }, + }, + }, + } + query["size"] = 0 + + util.MergeFields(query, aggs, true) + + searchR1, err = elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + panic(err) + } + searchResponse = SearchResponse{} + util.FromJSONBytes(searchR1.RawResult.Body, &searchResponse) + m3 := ParseAggregationBucketResult(bucketSize, searchResponse.Aggregations, bucketItem.Key, histgram.Key, termBucket.Key, nil) + + infos := util.MapStr{} + for _, clusterID := range clusterIDs { + result := util.MapStr{} + + //TODO update last active timestamp + //source := hit.Source + //source["project"]=util.MapStr{ + // "id":"12312312", + // "name":"统一日志云平台v1.0", + //} + //source["location"]=util.MapStr{ + // "provider" : "阿里云", + // "region" : "北京", + // "dc" : "昌平机房", + // "rack" : "rack-1", + //} + //source["owner"]=[]util.MapStr{util.MapStr{ + // "department" : "运维部", + // "name":"廖石阳", + // "id":"123123123", + //}} + + //result["metadata"] = source + result["summary"] = healthMap[clusterID] + result["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Cluster Status", + "units": "day", + }, + "data": getClusterMetrics(clusterID, m3), + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": getClusterMetrics(clusterID, m1), + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": getClusterMetrics(clusterID, m2), + }, + } + infos[clusterID] = result + } + + h.WriteJSON(w, infos, 200) +} + +func getClusterMetrics(id string, data MetricData) [][]interface{} { + return data[id] +} + +func (h *APIHandler) GetClusterInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + q := &orm.Query{WildcardIndex: true, Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "cluster_health"), + orm.Eq("metadata.labels.cluster_id", id), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + var healthInfo interface{} = util.MapStr{} + if len(result.Result) > 0 { + if rowM, ok := result.Result[0].(map[string]interface{}); ok { + healthInfo, _ = util.MapStr(rowM).GetValue("payload.elasticsearch.cluster_health") + } + } + + h.WriteJSON(w, healthInfo, 200) +} + +func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + resBody := map[string]interface{}{} + id := ps.ByName("id") + q := &orm.Query{Size: 1000} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + ) + + err, result := orm.Search(elastic.NodeConfig{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + query := util.MapStr{ + "size": 1000, + "collapse": util.MapStr{ + "field": "metadata.labels.node_id", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + }, + }, + } + q = &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + err, searchResult := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + nodeInfos := map[string]util.MapStr{} + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, hitM) + var totalShards float64 + if v, ok := shardInfo.(map[string]interface{}); ok { + shardCount := v["shard_count"] + replicasCount := v["replicas_count"] + if v1, ok := shardCount.(float64); ok { + totalShards += v1 + } + if v1, ok := replicasCount.(float64); ok { + totalShards += v1 + } + } + uptime, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, hitM) + cpu, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "percent"}, hitM) + load, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "load_average", "1m"}, hitM) + heapUsage, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem", "heap_used_percent"}, hitM) + freeDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "free_in_bytes"}, hitM) + nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) + if v, ok := freeDisk.(float64); ok { + freeDisk = util.ByteSize(uint64(v)) + } + + if v, ok := nodeID.(string); ok { + nodeInfos[v] = util.MapStr{ + "timestamp": hitM["timestamp"], + "shards": totalShards, + "cpu": cpu, + "load_1m": load, + "heap.percent": heapUsage, + "disk.avail": freeDisk, + "uptime": uptime, + } + + } + } + } + nodes := []interface{}{} + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nodeId, _ := util.GetMapValueByKeys([]string{"metadata", "node_id"}, hitM) + nodeName, _ := util.GetMapValueByKeys([]string{"metadata", "node_name"}, hitM) + status, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "status"}, hitM) + ip, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "ip"}, hitM) + transportAddress, _ := util.GetMapValueByKeys([]string{"payload", "node_state", "transport_address"}, hitM) + var port string + if v, ok := transportAddress.(string); ok { + parts := strings.Split(v, ":") + if len(parts) > 1 { + port = parts[1] + } + } + + if v, ok := nodeId.(string); ok { + ninfo := util.MapStr{ + "id": v, + "name": nodeName, + "ip": ip, + "port": port, + "status": status, + } + if nodeInfos[v] != nil { + if nodeInfos[v]["timestamp"] != nil { + if ts, ok := nodeInfos[v]["timestamp"].(string); ok { + tt, _ := time.Parse(time.RFC3339, ts) + if time.Now().Sub(tt).Seconds() > 30 { + ninfo["status"] = "unavailable" + } + } + } + util.MergeFields(ninfo, nodeInfos[v], true) + } else { + ninfo["timestamp"] = hitM["timestamp"] + } + nodes = append(nodes, ninfo) + } + } + } + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h *APIHandler) GetRealtimeClusterNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + meta := elastic.GetMetadata(id) + if meta == nil || !meta.IsAvailable() { + log.Debugf("cluster [%s] is not available", id) + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + esClient := elastic.GetClient(id) + if esClient == nil { + h.WriteJSON(w, util.MapStr{ + "error": "cluster not found", + }, http.StatusNotFound) + return + } + catNodesInfo, err := esClient.CatNodes("id,name,ip,port,master,heap.percent,disk.avail,cpu,load_1m,uptime") + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + return + } + catShardsInfo, err := esClient.CatShards() + if err != nil { + log.Error(err) + } + shardCounts := map[string]int{} + nodeM := map[string]string{} + for _, shardInfo := range catShardsInfo { + nodeM[shardInfo.NodeName] = shardInfo.NodeID + if c, ok := shardCounts[shardInfo.NodeName]; ok { + shardCounts[shardInfo.NodeName] = c + 1 + } else { + shardCounts[shardInfo.NodeName] = 1 + } + } + qps, err := h.getNodeQPS(id, 20) + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + return + } + + nodeInfos := []RealtimeNodeInfo{} + for _, nodeInfo := range catNodesInfo { + if len(nodeInfo.Id) == 4 { //node short id, use nodeId from shard info isnstead + nodeInfo.Id = nodeM[nodeInfo.Name] + } + if c, ok := shardCounts[nodeInfo.Name]; ok { + nodeInfo.Shards = c + } + info := RealtimeNodeInfo{ + CatNodeResponse: CatNodeResponse(nodeInfo), + } + if _, ok := qps[nodeInfo.Id]; ok { + info.IndexQPS = qps[nodeInfo.Id]["index"] + info.QueryQPS = qps[nodeInfo.Id]["query"] + info.IndexBytesQPS = qps[nodeInfo.Id]["index_bytes"] + } + nodeInfos = append(nodeInfos, info) + } + h.WriteJSON(w, nodeInfos, http.StatusOK) +} + +func (h *APIHandler) GetClusterIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + //size = h.GetIntOrDefault(req, "size", 20) + //from = h.GetIntOrDefault(req, "from", 0) + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + resBody := map[string]interface{}{} + id := ps.ByName("id") + q := &orm.Query{Size: 2000} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + //orm.NotEq("metadata.labels.state", "delete"), + ) + + err, result := orm.Search(elastic.IndexConfig{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + indices, err := h.getLatestIndices(req, min, max, id, &result) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + + h.WriteJSON(w, indices, http.StatusOK) +} + +func (h *APIHandler) GetRealtimeClusterIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + meta := elastic.GetMetadata(id) + if meta == nil || !meta.IsAvailable() { + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + //filter indices + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, id) + if !hasAllPrivilege && len(allowedIndices) == 0 { + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return + } + + esClient := elastic.GetClient(id) + indexInfos, err := esClient.GetIndices("") + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if !hasAllPrivilege { + filterIndices := map[string]elastic.IndexInfo{} + pattern := radix.Compile(allowedIndices...) + for indexName, indexInfo := range *indexInfos { + if pattern.Match(indexName) { + filterIndices[indexName] = indexInfo + } + } + indexInfos = &filterIndices + } + + qps, err := h.getIndexQPS(id, 20) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + } + var indices []RealtimeIndexInfo + for _, item := range *indexInfos { + info := RealtimeIndexInfo{ + IndexInfo: IndexInfo(item), + } + if _, ok := qps[item.Index]; ok { + info.IndexQPS = qps[item.Index]["index"] + info.QueryQPS = qps[item.Index]["query"] + info.IndexBytesQPS = qps[item.Index]["index_bytes"] + } + indices = append(indices, info) + } + h.WriteJSON(w, indices, http.StatusOK) +} + +type IndexInfo elastic.IndexInfo +type RealtimeIndexInfo struct { + IndexQPS interface{} `json:"index_qps"` + QueryQPS interface{} `json:"query_qps"` + IndexBytesQPS interface{} `json:"index_bytes_qps"` + IndexInfo +} +type CatNodeResponse elastic.CatNodeResponse +type RealtimeNodeInfo struct { + IndexQPS interface{} `json:"index_qps"` + QueryQPS interface{} `json:"query_qps"` + IndexBytesQPS interface{} `json:"index_bytes_qps"` + CatNodeResponse +} + +func (h *APIHandler) getIndexQPS(clusterID string, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + ver := h.Client().GetVersion() + bucketSizeStr := fmt.Sprintf("%ds", bucketSizeInSeconds) + intervalField, err := elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "term_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 1000, + }, + "aggs": util.MapStr{ + "date": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: "10s", + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.primaries.indexing.index_total", + }, + }, + "query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.total.search.query_total", + }, + }, + "index_bytes_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.primaries.store.size_in_bytes", + }, + }, + "index_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + "query_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "query_total", + }, + }, + "index_bytes_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_bytes_total", + }, + }, + }, + }, + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-1m", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + }, + }, + }, + } + return h.QueryQPS(query, bucketSizeInSeconds) +} + +func (h *APIHandler) getNodeQPS(clusterID string, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + ver := h.Client().GetVersion() + bucketSizeStr := fmt.Sprintf("%ds", bucketSizeInSeconds) + intervalField, err := elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "term_node": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "date": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: "10s", + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + }, + }, + "index_bytes_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + }, + }, + "query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + }, + }, + "index_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + "index_bytes_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_bytes_total", + }, + }, + "query_rate": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "query_total", + }, + }, + }, + }, + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-1m", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + }, + }, + } + return h.QueryQPS(query, bucketSizeInSeconds) +} + +func (h *APIHandler) QueryQPS(query util.MapStr, bucketSizeInSeconds int) (map[string]util.MapStr, error) { + esClient := h.Client() + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(event.Event{}), util.MustToJSONBytes(query)) + if err != nil { + return nil, err + } + indexQPS := map[string]util.MapStr{} + for _, agg := range searchRes.Aggregations { + for _, bk := range agg.Buckets { + if k, ok := bk["key"].(string); ok { + indexQPS[k] = util.MapStr{} + if dateAgg, ok := bk["date"].(map[string]interface{}); ok { + if bks, ok := dateAgg["buckets"].([]interface{}); ok { + var ( + maxIndexRate float64 + maxQueryRate float64 + maxIndexBytesRate float64 + preIndexTotal float64 + dropNext bool + ) + for _, dateBk := range bks { + if dateBkVal, ok := dateBk.(map[string]interface{}); ok { + if indexTotal, ok := dateBkVal["index_total"].(map[string]interface{}); ok { + if indexTotalVal, ok := indexTotal["value"].(float64); ok { + if preIndexTotal > 0 { + //if value of indexTotal is decreasing, drop the next value, + //and we will drop current and next qps value + if indexTotalVal - preIndexTotal < 0 { + dropNext = true + preIndexTotal = indexTotalVal + continue + }else{ + dropNext = false + } + } + preIndexTotal = indexTotalVal + } + } + if dropNext { + continue + } + if indexRate, ok := dateBkVal["index_rate"].(map[string]interface{}); ok { + if indexRateVal, ok := indexRate["value"].(float64); ok && indexRateVal > maxIndexRate { + maxIndexRate = indexRateVal + } + } + if queryRate, ok := dateBkVal["query_rate"].(map[string]interface{}); ok { + if queryRateVal, ok := queryRate["value"].(float64); ok && queryRateVal > maxQueryRate { + maxQueryRate = queryRateVal + } + } + if indexBytesRate, ok := dateBkVal["index_bytes_rate"].(map[string]interface{}); ok { + if indexBytesRateVal, ok := indexBytesRate["value"].(float64); ok && indexBytesRateVal > maxIndexBytesRate { + maxIndexBytesRate = indexBytesRateVal + } + } + } + } + indexQPS[k]["index"] = maxIndexRate / float64(bucketSizeInSeconds) + indexQPS[k]["query"] = maxQueryRate / float64(bucketSizeInSeconds) + indexQPS[k]["index_bytes"] = maxIndexBytesRate / float64(bucketSizeInSeconds) + } + } + } + } + } + return indexQPS, nil +} + +func (h *APIHandler) SearchClusterMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := util.MapStr{} + reqBody := struct { + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + SearchField string `json:"search_field"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var should []util.MapStr + if reqBody.SearchField != "" { + should = []util.MapStr{ + { + "prefix": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "boost": 2, + }, + }, + }, + } + } else { + should = []util.MapStr{ + { + "prefix": util.MapStr{ + "name": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "prefix": util.MapStr{ + "host": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "prefix": util.MapStr{ + "version": util.MapStr{ + "value": reqBody.Keyword, + "boost": 15, + }, + }, + }, + { + "match_phrase_prefix": util.MapStr{ + "name.text": util.MapStr{ + "query": reqBody.Keyword, + "boost": 6, + }, + }, + }, + { + "match": util.MapStr{ + "search_text": util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "boost": 2, + }, + }, + }, + { + "query_string": util.MapStr{ + "fields": []string{"*"}, + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "fuzzy_prefix_length": 2, + "fuzzy_max_expansions": 10, + "allow_leading_wildcard": false, + }, + }, + } + } + + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + if !hasAllPrivilege && clusterFilter == nil { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + must := []interface{}{} + if !hasAllPrivilege && clusterFilter != nil { + must = append(must, clusterFilter) + } + query := util.MapStr{ + "aggs": elastic.BuildSearchTermAggregations(reqBody.Aggregations), + "size": reqBody.Size, + "from": reqBody.From, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": elastic.BuildSearchTermFilter(reqBody.Filter), + "should": should, + "must": must, + }, + }, + "sort": []util.MapStr{ + { + "updated": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(reqBody.Sort) > 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.ElasticsearchConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + w.Write(util.MustToJSONBytes(response)) +} diff --git a/modules/elastic/api/v1/index_metrics.go b/modules/elastic/api/v1/index_metrics.go new file mode 100644 index 00000000..a4666039 --- /dev/null +++ b/modules/elastic/api/v1/index_metrics.go @@ -0,0 +1,838 @@ +package v1 + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "sort" + "strings" + "time" +) + +func (h *APIHandler) getIndexMetrics(req *http.Request, clusterID string, bucketSize int, min, max int64, indexName string, top int) map[string]*common.MetricItem{ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_id":util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + } + var ( + indexNames []string + err error + ) + if indexName != "" { + indexNames = strings.Split(indexName, ",") + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return nil + } + if !hasAllPrivilege{ + namePattern := radix.Compile(allowedIndices...) + var filterNames []string + for _, name := range indexNames { + if namePattern.Match(name){ + filterNames = append(filterNames, name) + } + } + if len(filterNames) == 0 { + return nil + } + indexNames = filterNames + } + top = len(indexNames) + + }else{ + indexNames, err = h.getTopIndexName(req, clusterID, top, 15) + if err != nil { + log.Error(err) + } + + } + if len(indexNames) > 0 { + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "metadata.labels.index_name": indexNames, + }, + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": "_all", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + //索引存储大小 + indexStorageMetric := newMetricItem("index_storage", 1, StorageGroupKey) + indexStorageMetric.AddAxi("Index storage","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + + indexMetricItems := []GroupMetricItem{ + { + Key: "index_storage", + Field: "payload.elasticsearch.index_stats.total.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexStorageMetric, + FormatType: "bytes", + Units: "", + }, + } + // segment 数量 + segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey) + segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_count", + Field: "payload.elasticsearch.index_stats.total.segments.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentCountMetric, + FormatType: "num", + Units: "", + }) + //索引文档个数 + docCountMetric := newMetricItem("doc_count", 2, DocumentGroupKey) + docCountMetric.AddAxi("Doc count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "doc_count", + Field: "payload.elasticsearch.index_stats.total.docs.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docCountMetric, + FormatType: "num", + Units: "", + }) + // docs 删除数量 + docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey) + docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "docs_deleted", + Field: "payload.elasticsearch.index_stats.total.docs.deleted", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsDeletedMetric, + FormatType: "num", + Units: "", + }) + //查询次数 + queryTimesMetric := newMetricItem("query_times", 2, OperationGroupKey) + queryTimesMetric.AddAxi("Query times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_times", + Field: "payload.elasticsearch.index_stats.total.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + + //Fetch次数 + fetchTimesMetric := newMetricItem("fetch_times", 3, OperationGroupKey) + fetchTimesMetric.AddAxi("Fetch times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "fetch_times", + Field: "payload.elasticsearch.index_stats.total.search.fetch_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //scroll 次数 + scrollTimesMetric := newMetricItem("scroll_times", 4, OperationGroupKey) + scrollTimesMetric.AddAxi("scroll times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "scroll_times", + Field: "payload.elasticsearch.index_stats.total.search.scroll_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //Merge次数 + mergeTimesMetric := newMetricItem("merge_times", 7, OperationGroupKey) + mergeTimesMetric.AddAxi("Merge times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "merge_times", + Field: "payload.elasticsearch.index_stats.total.merges.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //Refresh次数 + refreshTimesMetric := newMetricItem("refresh_times", 5, OperationGroupKey) + refreshTimesMetric.AddAxi("Refresh times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "refresh_times", + Field: "payload.elasticsearch.index_stats.total.refresh.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + //flush 次数 + flushTimesMetric := newMetricItem("flush_times", 6, OperationGroupKey) + flushTimesMetric.AddAxi("flush times","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "flush_times", + Field: "payload.elasticsearch.index_stats.total.flush.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushTimesMetric, + FormatType: "num", + Units: "requests/s", + }) + + //写入速率 + indexingRateMetric := newMetricItem("indexing_rate", 1, OperationGroupKey) + indexingRateMetric.AddAxi("Indexing rate","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_rate", + Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingRateMetric, + FormatType: "num", + Units: "doc/s", + }) + indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey) + indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_bytes", + Field: "payload.elasticsearch.index_stats.primaries.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingBytesMetric, + FormatType: "bytes", + Units: "bytes/s", + }) + //写入时延 + indexingLatencyMetric := newMetricItem("indexing_latency", 1, LatencyGroupKey) + indexingLatencyMetric.AddAxi("Indexing latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "indexing_latency", + Field: "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", + Field2: "payload.elasticsearch.index_stats.primaries.indexing.index_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //查询时延 + queryLatencyMetric := newMetricItem("query_latency", 2, LatencyGroupKey) + queryLatencyMetric.AddAxi("Query latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_latency", + Field: "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.search.query_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //fetch时延 + fetchLatencyMetric := newMetricItem("fetch_latency", 3, LatencyGroupKey) + fetchLatencyMetric.AddAxi("Fetch latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "fetch_latency", + Field: "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.search.fetch_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //merge时延 + mergeLatencyMetric := newMetricItem("merge_latency", 7, LatencyGroupKey) + mergeLatencyMetric.AddAxi("Merge latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "merge_latency", + Field: "payload.elasticsearch.index_stats.total.merges.total_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.merges.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //refresh时延 + refreshLatencyMetric := newMetricItem("refresh_latency", 5, LatencyGroupKey) + refreshLatencyMetric.AddAxi("Refresh latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "refresh_latency", + Field: "payload.elasticsearch.index_stats.total.refresh.total_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.refresh.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //scroll时延 + scrollLatencyMetric := newMetricItem("scroll_latency", 4, LatencyGroupKey) + scrollLatencyMetric.AddAxi("Scroll Latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "scroll_latency", + Field: "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.search.scroll_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //flush 时延 + flushLatencyMetric := newMetricItem("flush_latency", 6, LatencyGroupKey) + flushLatencyMetric.AddAxi("Flush latency","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "flush_latency", + Field: "payload.elasticsearch.index_stats.total.flush.total_time_in_millis", + Field2: "payload.elasticsearch.index_stats.total.flush.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //queryCache + queryCacheMetric := newMetricItem("query_cache", 1, CacheGroupKey) + queryCacheMetric.AddAxi("Query cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "query_cache", + Field: "payload.elasticsearch.index_stats.total.query_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: queryCacheMetric, + FormatType: "bytes", + Units: "", + }) + //requestCache + requestCacheMetric := newMetricItem("request_cache", 2, CacheGroupKey) + requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "request_cache", + Field: "payload.elasticsearch.index_stats.total.request_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: requestCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache Hit + requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey) + requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "request_cache_hit", + Field: "payload.elasticsearch.index_stats.total.request_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + // Request Cache Miss + requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey) + requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "request_cache_miss", + Field: "payload.elasticsearch.index_stats.total.request_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Query Cache Count + queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey) + queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_count", + Field: "payload.elasticsearch.index_stats.total.query_cache.cache_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheCountMetric, + FormatType: "num", + Units: "", + }) + // Query Cache Miss + queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey) + queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_hit", + Field: "payload.elasticsearch.index_stats.total.query_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + + //// Query Cache evictions + //queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 11, CacheGroupKey) + //queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + //indexMetricItems=append(indexMetricItems, GroupMetricItem{ + // Key: "query_cache_evictions", + // Field: "payload.elasticsearch.index_stats.total.query_cache.evictions", + // ID: util.GetUUID(), + // IsDerivative: true, + // MetricItem: queryCacheEvictionsMetric, + // FormatType: "num", + // Units: "evictions", + //}) + + // Query Cache Miss + queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey) + queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "query_cache_miss", + Field: "payload.elasticsearch.index_stats.total.query_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Fielddata内存占用大小 + fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey) + fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "fielddata_cache", + Field: "payload.elasticsearch.index_stats.total.fielddata.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldDataCacheMetric, + FormatType: "bytes", + Units: "", + }) + //segment memory + segmentMemoryMetric := newMetricItem("segment_memory", 13, MemoryGroupKey) + segmentMemoryMetric.AddAxi("Segment memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_memory", + Field: "payload.elasticsearch.index_stats.total.segments.memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment doc values memory + docValuesMemoryMetric := newMetricItem("segment_doc_values_memory", 13, MemoryGroupKey) + docValuesMemoryMetric.AddAxi("Segment Doc values Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_doc_values_memory", + Field: "payload.elasticsearch.index_stats.total.segments.doc_values_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docValuesMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment terms memory + termsMemoryMetric := newMetricItem("segment_terms_memory", 13, MemoryGroupKey) + termsMemoryMetric.AddAxi("Segment Terms Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_terms_memory", + Field: "payload.elasticsearch.index_stats.total.segments.terms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: termsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + //segment fields memory + fieldsMemoryMetric := newMetricItem("segment_fields_memory", 13, MemoryGroupKey) + fieldsMemoryMetric.AddAxi("Segment Fields Memory","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + indexMetricItems = append(indexMetricItems, GroupMetricItem{ + Key: "segment_fields_memory", + Field: "payload.elasticsearch.index_stats.total.segments.stored_fields_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment index writer memory + segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey) + segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_index_writer_memory", + Field: "payload.elasticsearch.index_stats.total.segments.index_writer_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentIndexWriterMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment term vectors memory + segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey) + segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + indexMetricItems=append(indexMetricItems, GroupMetricItem{ + Key: "segment_term_vectors_memory", + Field: "payload.elasticsearch.index_stats.total.segments.term_vectors_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermVectorsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + aggs:=map[string]interface{}{} + + for _,metricItem:=range indexMetricItems { + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + + if metricItem.Field2 != ""{ + aggs[metricItem.ID + "_field2"]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field2, + }, + } + } + + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID + "_deriv_field2"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": top, + "order": util.MapStr{ + "max_store": "desc", + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + "max_store": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.total.store.size_in_bytes", + }, + }, + }, + }, + } + return h.getMetrics(query, indexMetricItems, bucketSize) + +} + +func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, lastMinutes int) ([]string, error){ + ver := h.Client().GetVersion() + cr, _ := util.VersionCompare(ver.Number, "6.1") + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { + return nil, nil + } + var ( + now = time.Now() + max = now.UnixNano()/1e6 + min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 + ) + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + } + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return nil, fmt.Errorf("no index permission") + } + if !hasAllPrivilege { + must = append(must, util.MapStr{ + "query_string": util.MapStr{ + "query": strings.Join(allowedIndices, " "), + "fields": []string{"metadata.labels.index_name"}, + "default_operator": "OR", + }, + }) + } + bucketSizeStr := "60s" + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + + query := util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": "_all", + }, + }, + }, + }, + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "group_by_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>search_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}}, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "search_query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.total.search.query_total", + }, + }, + "search_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "search_query_total", + }, + }, + }, + }, + }, + }, + "group_by_index1": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_name", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>index_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}, + }, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.index_stats.total.indexing.index_total", + }, + }, + "index_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + }, + }, + }, + }, + }, + } + response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) + if err!=nil{ + log.Error(err) + return nil, err + } + var maxQpsKVS = map[string] float64{} + for _, agg := range response.Aggregations { + for _, bk := range agg.Buckets { + key := bk["key"].(string) + if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { + val := maxQps["value"].(float64) + if _, ok = maxQpsKVS[key] ; ok { + maxQpsKVS[key] = maxQpsKVS[key] + val + }else{ + maxQpsKVS[key] = val + } + } + } + } + var ( + qpsValues TopTermOrder + ) + for k, v := range maxQpsKVS { + qpsValues = append(qpsValues, TopTerm{ + Key: k, + Value: v, + }) + } + sort.Sort(qpsValues) + var length = top + if top > len(qpsValues) { + length = len(qpsValues) + } + indexNames := []string{} + for i := 0; i t[j].Value //desc +} +func (t TopTermOrder) Swap(i, j int){ + t[i], t[j] = t[j], t[i] +} diff --git a/modules/elastic/api/v1/index_overview.go b/modules/elastic/api/v1/index_overview.go new file mode 100644 index 00000000..ef574541 --- /dev/null +++ b/modules/elastic/api/v1/index_overview.go @@ -0,0 +1,1159 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package v1 + +import ( + "fmt" + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "strings" + "time" +) + +func (h *APIHandler) SearchIndexMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody:=util.MapStr{} + reqBody := struct{ + Keyword string `json:"keyword"` + Size int `json:"size"` + From int `json:"from"` + Aggregations []elastic.SearchAggParam `json:"aggs"` + Highlight elastic.SearchHighlightParam `json:"highlight"` + Filter elastic.SearchFilterParam `json:"filter"` + Sort []string `json:"sort"` + SearchField string `json:"search_field"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + aggs := elastic.BuildSearchTermAggregations(reqBody.Aggregations) + aggs["term_cluster_id"] = util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.cluster_id", + "size": 1000, + }, + "aggs": util.MapStr{ + "term_cluster_name": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.cluster_name", + "size": 1, + }, + }, + }, + } + filter := elastic.BuildSearchTermFilter(reqBody.Filter) + var should []util.MapStr + if reqBody.SearchField != ""{ + should = []util.MapStr{ + { + "prefix": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + reqBody.SearchField: util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + } + }else{ + if reqBody.Keyword != ""{ + should = []util.MapStr{ + { + "prefix": util.MapStr{ + "metadata.index_name": util.MapStr{ + "value": reqBody.Keyword, + "boost": 30, + }, + }, + }, + { + "prefix": util.MapStr{ + "metadata.aliases": util.MapStr{ + "value": reqBody.Keyword, + "boost": 20, + }, + }, + }, + { + "match": util.MapStr{ + "search_text": util.MapStr{ + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "max_expansions": 10, + "prefix_length": 2, + "fuzzy_transpositions": true, + "boost": 2, + }, + }, + }, + { + "query_string": util.MapStr{ + "fields": []string{"*"}, + "query": reqBody.Keyword, + "fuzziness": "AUTO", + "fuzzy_prefix_length": 2, + "fuzzy_max_expansions": 10, + "fuzzy_transpositions": true, + "allow_leading_wildcard": false, + }, + }, + } + } + } + + must := []interface{}{ + } + hasAllPrivilege, indexPrivilege := h.GetCurrentUserIndex(req) + if !hasAllPrivilege && len(indexPrivilege) == 0 { + h.WriteJSON(w, elastic.SearchResponse{ + + }, http.StatusOK) + return + } + if !hasAllPrivilege { + indexShould := make([]interface{}, 0, len(indexPrivilege)) + for clusterID, indices := range indexPrivilege { + var ( + wildcardIndices []string + normalIndices []string + ) + for _, index := range indices { + if strings.Contains(index,"*") { + wildcardIndices = append(wildcardIndices, index) + continue + } + normalIndices = append(normalIndices, index) + } + subShould := []util.MapStr{} + if len(wildcardIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "query_string": util.MapStr{ + "query": strings.Join(wildcardIndices, " "), + "fields": []string{"metadata.index_name"}, + "default_operator": "OR", + }, + }) + } + if len(normalIndices) > 0 { + subShould = append(subShould, util.MapStr{ + "terms": util.MapStr{ + "metadata.index_name": normalIndices, + }, + }) + } + indexShould = append(indexShould, util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "wildcard": util.MapStr{ + "metadata.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": subShould, + }, + }, + }, + }, + }) + } + indexFilter := util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": indexShould, + }, + } + must = append(must, indexFilter) + } + boolQuery := util.MapStr{ + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.index_status": "deleted", + }, + }, + }, + "filter": filter, + "must": must, + } + if len(should) > 0 { + boolQuery["should"] = should + boolQuery["minimum_should_match"] = 1 + } + query := util.MapStr{ + "aggs": aggs, + "size": reqBody.Size, + "from": reqBody.From, + "highlight": elastic.BuildSearchHighlight(&reqBody.Highlight), + "query": util.MapStr{ + "bool": boolQuery, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(reqBody.Sort) > 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.IndexConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + w.Write(util.MustToJSONBytes(response)) + +} +func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + defer func() { + if err := recover(); err != nil { + log.Error(err) + } + }() + var indexIDs []interface{} + + + h.DecodeJSON(req, &indexIDs) + + if len(indexIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + q1 := orm.Query{WildcardIndex: true} + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "index_stats"), + orm.In("metadata.labels.index_id", indexIDs), + ) + q1.Collapse("metadata.labels.index_id") + q1.AddSort("timestamp", orm.DESC) + q1.Size = len(indexIDs) + 1 + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + } + + summaryMap := util.MapStr{} + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + if indexID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "index_id"}, result); ok { + summary := map[string]interface{}{} + if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "total", "docs"}, result); ok { + summary["docs"] = docs + } + if indexInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, result); ok { + summary["index_info"] = indexInfo + } + if shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "shard_info"}, result); ok { + if sinfo, ok := shardInfo.([]interface{}); ok { + unassignedCount := 0 + for _, item := range sinfo { + if itemMap, ok := item.(map[string]interface{}); ok{ + if itemMap["state"] == "UNASSIGNED" { + unassignedCount++ + } + } + + } + summary["unassigned_shards"] = unassignedCount + } + } + summaryMap[indexID.(string)] = summary + } + } + } + + statusMetric, err := getIndexStatusOfRecentDay(indexIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, (15)) + if err != nil { + panic(err) + return + } + // 索引速率 + indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems := []GroupMetricItem{} + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing", + Field: "payload.elasticsearch.index_stats.primaries.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "Indexing/s", + }) + queryMetric:=newMetricItem("search", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "search", + Field: "payload.elasticsearch.index_stats.total.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "Search/s", + }) + + aggs:=map[string]interface{}{} + query :=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.index_id": indexIDs, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + for _,metricItem:=range nodeMetricItems{ + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + } + } + + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + panic(err) + } + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_id", + "size": 100, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + }, + }, + } + metrics := h.getMetrics(query, nodeMetricItems, bucketSize) + indexMetrics := map[string]util.MapStr{} + for key, item := range metrics { + for _, line := range item.Lines { + if _, ok := indexMetrics[line.Metric.Label]; !ok{ + indexMetrics[line.Metric.Label] = util.MapStr{ + } + } + indexMetrics[line.Metric.Label][key] = line.Data + } + } + infos := util.MapStr{} + for _, tempIndexID := range indexIDs { + result := util.MapStr{} + + indexID := tempIndexID.(string) + + result["summary"] = summaryMap[indexID] + result["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Index Status", + "units": "day", + }, + "data": statusMetric[indexID], + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": indexMetrics[indexID]["indexing"], + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": indexMetrics[indexID]["search"], + }, + } + infos[indexID] = result + } + h.WriteJSON(w, infos, http.StatusOK) +} + +func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + indexID := ps.MustGetParameter("index") + parts := strings.Split(indexID, ":") + if len(parts) > 1 && !h.IsIndexAllowed(req, clusterID, parts[1]) { + h.WriteError(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return + } + if len(parts) < 2 { + h.WriteError(w, "invalid index id: "+ indexID, http.StatusInternalServerError) + return + } + + q := orm.Query{ + Size: 1, + } + q.Conds = orm.And(orm.Eq("metadata.index_name", parts[1]), orm.Eq("metadata.cluster_id", clusterID)) + q.AddSort("timestamp", orm.DESC) + + err, res := orm.Search(&elastic.IndexConfig{}, &q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + if len(response.Hits.Hits) == 0 { + log.Warnf("index [%v][%v] not found, may be you should wait several seconds", clusterID, indexID) + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + q1 := orm.Query{ + Size: 1, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "index_stats"), + orm.Eq("metadata.labels.index_name", parts[1]), + orm.Eq("metadata.labels.cluster_id", clusterID), + ) + q1.Collapse("metadata.labels.index_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + summary := util.MapStr{} + hit := response.Hits.Hits[0].Source + if aliases, ok := util.GetMapValueByKeys([]string{"metadata", "aliases"}, hit); ok { + health, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, hit) + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, hit) + summary["aliases"] = aliases + summary["timestamp"] = hit["timestamp"] + summary["index_info"] = util.MapStr{ + "health":health, + "status": state, + } + } + //if mappings, ok := util.GetMapValueByKeys([]string{"metadata", "mappings"}, hit); ok { + // summary["mappings"] = mappings + //} + //if settings, ok := util.GetMapValueByKeys([]string{"metadata", "settings"}, hit); ok { + // summary["settings"] = settings + //} + if len(result.Result) > 0 { + result, ok := result.Result[0].(map[string]interface{}) + if ok { + if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "total", "docs"}, result); ok { + summary["docs"] = docs + } + if indexInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, result); ok { + if infoM, ok := indexInfo.(map[string]interface{}); ok { + if tm, ok := result["timestamp"].(string); ok { + issueTime, _ := time.Parse(time.RFC3339, tm) + if time.Now().Sub(issueTime).Seconds() > 30 { + health, _:= util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, response.Hits.Hits[0].Source) + infoM["health"] = health + } + } + state, _:= util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, response.Hits.Hits[0].Source) + if state == "delete" { + infoM["status"] = "delete" + infoM["health"] = "N/A" + } + } + summary["index_info"] = indexInfo + } + if shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "shard_info"}, result); ok { + if sinfo, ok := shardInfo.([]interface{}); ok { + unassignedCount := 0 + for _, item := range sinfo { + if itemMap, ok := item.(map[string]interface{}); ok{ + if itemMap["state"] == "UNASSIGNED" { + unassignedCount++ + } + } + + } + summary["unassigned_shards"] = unassignedCount + } + } + } + summary["timestamp"] = result["timestamp"] + } + + h.WriteJSON(w, summary, http.StatusOK) +} + +func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + indexName := ps.MustGetParameter("index") + q1 := orm.Query{ + Size: 1, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "index_stats"), + orm.Eq("metadata.labels.index_name", indexName), + orm.Eq("metadata.labels.cluster_id", clusterID), + ) + q1.Collapse("metadata.labels.index_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + if err != nil { + h.WriteJSON(w,util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError ) + return + } + var shardInfo interface{} = []interface{}{} + if len(result.Result) > 0 { + row, ok := result.Result[0].(map[string]interface{}) + if ok { + shardInfo, ok = util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "shard_info"}, row) + } + } + + h.WriteJSON(w, shardInfo, http.StatusOK) +} + +func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + indexName := ps.MustGetParameter("index") + if !h.IsIndexAllowed(req, clusterID, indexName) { + h.WriteJSON(w, util.MapStr{ + "error": http.StatusText(http.StatusForbidden), + }, http.StatusForbidden) + return + } + var must = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }, + } + resBody := map[string]interface{}{} + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 60) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + meta := elastic.GetMetadata(clusterID) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.IndexStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.IndexStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem:=newMetricItem("index_throughput", 1, OperationGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + metricItem.AddLine("Indexing Rate","Primary Indexing","Number of documents being indexed for node.","group1","payload.elasticsearch.index_stats.primaries.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Deleting Rate","Primary Deleting","Number of documents being deleted for node.","group1","payload.elasticsearch.index_stats.primaries.indexing.delete_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("search_throughput", 2, OperationGroupKey) + metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + metricItem.AddLine("Search Rate","Search Rate", + "Number of search requests being executed.", + "group1","payload.elasticsearch.index_stats.total.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("index_latency", 3, LatencyGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + + metricItem.AddLine("Indexing Latency","Primary Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Deleting Latency","Primary Deleting Latency","Average latency for delete documents.","group1","payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("search_latency", 4, LatencyGroupKey) + metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + + metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.index_stats.total.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.index_stats.total.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.index_stats.total.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + metrics := h.getSingleMetrics(metricItems,query, bucketSize) + healthMetric, err := h.getIndexHealthMetric(clusterID, indexName, min, max, bucketSize) + if err != nil { + log.Error(err) + } + metrics["index_health"] = healthMetric + resBody["metrics"] = metrics + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) getIndexHealthMetric(id, indexName string, min, max int64, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_name": util.MapStr{ + "value": indexName, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "group_status": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.index_stats.index_info.health", + "size": 5, + }, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("index_health", 1, "") + metricItem.AddLine("health","Health","","group1","payload.elasticsearch.index_stats.index_info.health","max",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + metricData, err = parseHealthMetricData(response.Aggregations["dates"].Buckets) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + + +func getIndexStatusOfRecentDay(indexIDs []interface{})(map[string][]interface{}, error){ + q := orm.Query{ + WildcardIndex: true, + } + query := util.MapStr{ + "aggs": util.MapStr{ + "group_by_index_id": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.index_id", + "size": 100, + }, + "aggs": util.MapStr{ + "time_histogram": util.MapStr{ + "date_range": util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + },{ + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }, + "aggs": util.MapStr{ + "term_health": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.index_stats.index_info.health", + }, + }, + }, + }, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": "now-15d", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.index_id": indexIDs, + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + return nil, err + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + recentStatus := map[string][]interface{}{} + for _, bk := range response.Aggregations["group_by_index_id"].Buckets { + indexKey := bk["key"].(string) + recentStatus[indexKey] = []interface{}{} + if histogramAgg, ok := bk["time_histogram"].(map[string]interface{}); ok { + if bks, ok := histogramAgg["buckets"].([]interface{}); ok { + for _, bkItem := range bks { + if bkVal, ok := bkItem.(map[string]interface{}); ok { + if termHealth, ok := bkVal["term_health"].(map[string]interface{}); ok { + if healthBks, ok := termHealth["buckets"].([]interface{}); ok { + if len(healthBks) == 0 { + continue + } + healthMap := map[string]int{} + status := "unknown" + for _, hbkItem := range healthBks { + if hitem, ok := hbkItem.(map[string]interface{}); ok { + healthMap[hitem["key"].(string)] = 1 + } + } + if _, ok = healthMap["red"]; ok { + status = "red" + }else if _, ok = healthMap["yellow"]; ok { + status = "yellow" + }else if _, ok = healthMap["green"]; ok { + status = "green" + } + recentStatus[indexKey] = append(recentStatus[indexKey], []interface{}{bkVal["key"], status}) + } + } + } + } + } + } + } + return recentStatus, nil +} + +func (h *APIHandler) getIndexNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string] interface{}{} + id := ps.ByName("id") + indexName := ps.ByName("index") + if !h.IsIndexAllowed(req, id, indexName) { + h.WriteJSON(w, util.MapStr{ + "error": http.StatusText(http.StatusForbidden), + }, http.StatusForbidden) + return + } + q := &orm.Query{ Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.labels.cluster_id", id), + orm.Eq("metadata.labels.index_name", indexName), + orm.Eq("metadata.name", "index_routing_table"), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + namesM := util.MapStr{} + if len(result.Result) > 0 { + if data, ok := result.Result[0].(map[string]interface{}); ok { + if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_routing_table"}, data); exists { + if table, ok := routingTable.(map[string]interface{}); ok{ + if shardsM, ok := table["shards"].(map[string]interface{}); ok { + for _, rows := range shardsM { + if rowsArr, ok := rows.([]interface{}); ok { + for _, rowsInner := range rowsArr { + if rowsInnerM, ok := rowsInner.(map[string]interface{}); ok { + if v, ok := rowsInnerM["node"].(string); ok { + namesM[v] = true + } + } + } + } + + } + } + + } + } + } + } + + //node uuid + nodeIds := make([]interface{}, 0, len(namesM) ) + for name, _ := range namesM { + nodeIds = append(nodeIds, name) + } + + q1 := &orm.Query{ Size: 100} + q1.AddSort("timestamp", orm.DESC) + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + orm.In("metadata.node_id", nodeIds), + ) + err, result = orm.Search(elastic.NodeConfig{}, q1) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + nodes := []interface{}{} + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nodeId, _ := util.GetMapValueByKeys([]string{"metadata", "node_id"}, hitM) + nodeName, _ := util.GetMapValueByKeys([]string{"metadata", "node_name"}, hitM) + status, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "status"}, hitM) + ip, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "ip"}, hitM) + transportAddress, _ := util.GetMapValueByKeys([]string{"payload", "node_state", "transport_address"}, hitM) + var port string + if v, ok := transportAddress.(string); ok { + parts := strings.Split(v, ":") + if len(parts) > 1 { + port = parts[1] + } + } + + if v, ok := nodeId.(string); ok { + ninfo := util.MapStr{ + "id": v, + "name": nodeName, + "ip": ip, + "port": port, + "status": status, + "timestamp": hitM["timestamp"], + } + nodes = append(nodes, ninfo) + } + } + } + + h.WriteJSON(w, nodes, http.StatusOK) +} + +func (h APIHandler) ListIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterIds := h.GetParameterOrDefault(req, "ids", "") + keyword := h.GetParameterOrDefault(req, "keyword", "") + ids := strings.Split(clusterIds, ",") + if len(ids) == 0 { + h.Error400(w, "cluster id is required") + return + } + var must = []util.MapStr{} + + if !util.StringInArray(ids, "*"){ + + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "metadata.cluster_id": ids, + }, + }) + } + + if keyword != "" { + must = append(must, util.MapStr{ + "wildcard":util.MapStr{ + "metadata.index_name": + util.MapStr{"value": fmt.Sprintf("*%s*", keyword)}, + }, + }) + } + var dsl = util.MapStr{ + "_source": []string{"metadata.index_name"}, + "collapse": util.MapStr{ + "field": "metadata.index_name", + }, + "size": 100, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": must, + "must_not": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.state": util.MapStr{ + "value": "delete", + }, + }, + }, + }, + }, + }, + } + + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + indexName := orm.GetIndexName(elastic.IndexConfig{}) + resp, err := esClient.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(dsl)) + if err != nil { + + return + } + list := resp.Hits.Hits + var indexNames []string + for _, v := range list { + m := v.Source["metadata"].(map[string]interface{}) + indexNames = append(indexNames, m["index_name"].(string)) + + } + m := make(map[string]interface{}) + m["indexnames"] = indexNames + h.WriteOKJSON(w, m) + + return +} diff --git a/modules/elastic/api/v1/manage.go b/modules/elastic/api/v1/manage.go new file mode 100644 index 00000000..5871f775 --- /dev/null +++ b/modules/elastic/api/v1/manage.go @@ -0,0 +1,1408 @@ +package v1 + +import ( + "context" + "encoding/json" + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/core" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "math" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +type APIHandler struct { + core.Handler + Config common.ModuleConfig +} + +func (h *APIHandler) Client() elastic.API { + return elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) +} + +func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var conf = &elastic.ElasticsearchConfig{} + err := h.DecodeJSON(req, conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + // TODO validate data format + conf.Enabled = true + conf.Host = strings.TrimSpace(conf.Host) + conf.Endpoint = fmt.Sprintf("%s://%s", conf.Schema, conf.Host) + conf.ID = util.GetUUID() + ctx := &orm.Context{ + Refresh: "wait_for", + } + if conf.CredentialID == "" && conf.BasicAuth != nil && conf.BasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + conf.CredentialID = credentialID + } + conf.BasicAuth = nil + if conf.Distribution == "" { + conf.Distribution = elastic.Elasticsearch + } + err = orm.Create(ctx, conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + basicAuth, err := common.GetBasicAuth(conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + conf.BasicAuth = basicAuth + conf.Source = elastic.ElasticsearchConfigSourceElasticsearch + _, err = common.InitElasticInstance(*conf) + if err != nil { + log.Warn("error on init elasticsearch:", err) + } + + h.WriteCreatedOKJSON(w, conf.ID) +} + +func saveBasicAuthToCredential(conf *elastic.ElasticsearchConfig) (string, error) { + if conf == nil { + return "", fmt.Errorf("param elasticsearh config can not be empty") + } + cred := credential.Credential{ + Name: conf.Name, + Type: credential.BasicAuth, + Tags: []string{"ES"}, + Payload: map[string]interface{}{ + "basic_auth": map[string]interface{}{ + "username": conf.BasicAuth.Username, + "password": conf.BasicAuth.Password.Get(), + }, + }, + } + cred.ID = util.GetUUID() + err := cred.Encode() + if err != nil { + return "", err + } + err = orm.Create(nil, &cred) + if err != nil { + return "", err + } + return cred.ID, nil +} + +func (h *APIHandler) HandleGetClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + clusterConf := elastic.ElasticsearchConfig{} + clusterConf.ID = id + exists, err := orm.Get(&clusterConf) + if err != nil || !exists { + log.Error(err) + h.Error404(w) + return + } + h.WriteGetOKJSON(w, id, clusterConf) +} + +func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var conf = map[string]interface{}{} + err := h.DecodeJSON(req, &conf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + id := ps.MustGetParameter("id") + originConf := elastic.ElasticsearchConfig{} + originConf.ID = id + exists, err := orm.Get(&originConf) + if err != nil || !exists { + log.Error(err) + h.Error404(w) + return + } + buf := util.MustToJSONBytes(originConf) + source := map[string]interface{}{} + util.MustFromJSONBytes(buf, &source) + for k, v := range conf { + if k == "id" { + continue + } + if k == "basic_auth" { + if authMap, ok := v.(map[string]interface{}); ok { + if pwd, ok := authMap["password"]; !ok || (ok && pwd == "") { + if sourceM, ok := source[k].(map[string]interface{}); ok { + authMap["password"] = sourceM["password"] + } + } + } + } + source[k] = v + } + + if host, ok := conf["host"].(string); ok { + host = strings.TrimSpace(host) + if schema, ok := conf["schema"].(string); ok { + source["endpoint"] = fmt.Sprintf("%s://%s", schema, host) + source["host"] = host + } + } + + conf["updated"] = time.Now() + ctx := &orm.Context{ + Refresh: "wait_for", + } + confBytes, _ := json.Marshal(source) + newConf := &elastic.ElasticsearchConfig{} + json.Unmarshal(confBytes, newConf) + newConf.ID = id + if conf["credential_id"] == nil { + if newConf.BasicAuth != nil && newConf.BasicAuth.Username != "" { + credentialID, err := saveBasicAuthToCredential(newConf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + newConf.CredentialID = credentialID + newConf.BasicAuth = nil + } else { + newConf.CredentialID = "" + } + } + err = orm.Update(ctx, newConf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + basicAuth, err := common.GetBasicAuth(newConf) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + newConf.BasicAuth = basicAuth + + //update config in heap + newConf.Source = elastic.ElasticsearchConfigSourceElasticsearch + _, err = common.InitElasticInstance(*newConf) + if err != nil { + log.Warn("error on init elasticsearch:", err) + } + + h.WriteUpdatedOKJSON(w, id) +} + +func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.MustGetParameter("id") + + esConfig := elastic.ElasticsearchConfig{} + esConfig.ID = id + ok, err := orm.Get(&esConfig) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if ok { + if esConfig.Reserved { + resBody["error"] = "this cluster is reserved" + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + } + ctx := &orm.Context{ + Refresh: "wait_for", + } + err = orm.Delete(ctx, &esConfig) + + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + delDsl := util.MapStr{ + "query": util.MapStr{ + "match": util.MapStr{ + "metadata.cluster_id": id, + }, + }, + } + err = orm.DeleteBy(elastic.NodeConfig{}, util.MustToJSONBytes(delDsl)) + if err != nil { + log.Error(err) + } + err = orm.DeleteBy(elastic.IndexConfig{}, util.MustToJSONBytes(delDsl)) + if err != nil { + log.Error(err) + } + + elastic.RemoveInstance(id) + elastic.RemoveHostsByClusterID(id) + h.WriteDeletedOKJSON(w, id) +} + +func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + name = h.GetParameterOrDefault(req, "name", "") + sortField = h.GetParameterOrDefault(req, "sort_field", "") + sortOrder = h.GetParameterOrDefault(req, "sort_order", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + if name != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name.text": "%s"}}`, name)) + } + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + if !hasAllPrivilege && clusterFilter == nil { + h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) + return + } + if !hasAllPrivilege { + if mustBuilder.String() != "" { + mustBuilder.WriteString(",") + } + mustBuilder.Write(util.MustToJSONBytes(clusterFilter)) + } + + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + var sort = "" + if sortField != "" && sortOrder != "" { + sort = fmt.Sprintf(`,"sort":[{"%s":{"order":"%s"}}]`, sortField, sortOrder) + } + + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from, sort) + q := orm.Query{ + RawQuery: []byte(queryDSL), + } + err, result := orm.Search(elastic.ElasticsearchConfig{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + searchRes := elastic.SearchResponse{} + util.MustFromJSONBytes(result.Raw, &searchRes) + if len(searchRes.Hits.Hits) > 0 { + for _, hit := range searchRes.Hits.Hits { + if basicAuth, ok := hit.Source["basic_auth"]; ok { + if authMap, ok := basicAuth.(map[string]interface{}); ok { + delete(authMap, "password") + } + } + } + } + + h.WriteJSON(w, searchRes, http.StatusOK) +} + +func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.MustGetParameter("id") + + summary := map[string]interface{}{} + var query = util.MapStr{ + "sort": util.MapStr{ + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + "size": 1, + } + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + }, + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(query), + WildcardIndex: true, + } + err, result := orm.Search(event.Event{}, &q) + if err != nil { + resBody["error"] = err.Error() + log.Error("MetricsSummary search error: ", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if len(result.Result) > 0 { + if v, ok := result.Result[0].(map[string]interface{}); ok { + sourceMap := util.MapStr(v) + summary["timestamp"], _ = sourceMap.GetValue("timestamp") + status, _ := sourceMap.GetValue("payload.elasticsearch.cluster_stats") + statusMap := util.MapStr(status.(map[string]interface{})) + summary["cluster_name"], _ = statusMap.GetValue("cluster_name") + summary["status"], _ = statusMap.GetValue("status") + summary["indices_count"], _ = statusMap.GetValue("indices.count") + summary["total_shards"], _ = statusMap.GetValue("indices.shards.total") + summary["primary_shards"], _ = statusMap.GetValue("indices.shards.primaries") + summary["replication_shards"], _ = statusMap.GetValue("indices.shards.replication") + //summary["unassigned_shards"]=status.Indices["shards"].(map[string]interface{})["primaries"] + + summary["document_count"], _ = statusMap.GetValue("indices.docs.count") + summary["deleted_document_count"], _ = statusMap.GetValue("indices.docs.deleted") + + summary["used_store_bytes"], _ = statusMap.GetValue("indices.store.size_in_bytes") + + summary["max_store_bytes"], _ = statusMap.GetValue("nodes.fs.total_in_bytes") + summary["available_store_bytes"], _ = statusMap.GetValue("nodes.fs.available_in_bytes") + + summary["fielddata_bytes"], _ = statusMap.GetValue("indices.fielddata.memory_size_in_bytes") + summary["fielddata_evictions"], _ = statusMap.GetValue("indices.fielddata.evictions") + + summary["query_cache_bytes"], _ = statusMap.GetValue("indices.query_cache.memory_size_in_bytes") + summary["query_cache_total_count"], _ = statusMap.GetValue("indices.query_cache.total_count") + summary["query_cache_hit_count"], _ = statusMap.GetValue("indices.query_cache.hit_count") + summary["query_cache_miss_count"], _ = statusMap.GetValue("indices.query_cache.miss_count") + summary["query_cache_evictions"], _ = statusMap.GetValue("indices.query_cache.evictions") + + summary["segments_count"], _ = statusMap.GetValue("indices.segments.count") + summary["segments_memory_in_bytes"], _ = statusMap.GetValue("indices.segments.memory_in_bytes") + + summary["nodes_count"], _ = statusMap.GetValue("nodes.count.total") + summary["version"], _ = statusMap.GetValue("nodes.versions") + + summary["mem_total_in_bytes"], _ = statusMap.GetValue("nodes.os.mem.total_in_bytes") + summary["mem_used_in_bytes"], _ = statusMap.GetValue("nodes.os.mem.used_in_bytes") + summary["mem_used_percent"], _ = statusMap.GetValue("nodes.os.mem.used_percent") + + summary["uptime"], _ = statusMap.GetValue("nodes.jvm.max_uptime_in_millis") + summary["used_jvm_bytes"], _ = statusMap.GetValue("nodes.jvm.mem.heap_used_in_bytes") + summary["max_jvm_bytes"], _ = statusMap.GetValue("nodes.jvm.mem.heap_max_in_bytes") + } + } + + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + err, result = orm.Search(event.Event{}, &q) + if err != nil { + log.Error("MetricsSummary search error: ", err) + } else { + if len(result.Result) > 0 { + if v, ok := result.Result[0].(map[string]interface{}); ok { + health, _ := util.MapStr(v).GetValue("payload.elasticsearch.cluster_health") + healthMap := util.MapStr(health.(map[string]interface{})) + summary["unassigned_shards"], _ = healthMap.GetValue("unassigned_shards") + } + } + } + + resBody["summary"] = summary + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +// new +func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + panic(err) + return + } + meta := elastic.GetMetadata(id) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.IndexStats.Enabled && meta.Config.MonitorConfigs.IndexStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.IndexStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + + //fmt.Println(min," vs ",max,",",rangeFrom,rangeTo,"range hours:",hours) + + //metrics:=h.GetClusterMetrics(id,bucketSize,min,max) + isOverview := h.GetIntOrDefault(req, "overview", 0) + var metrics interface{} + if isOverview == 1 { + metrics = h.GetClusterIndexMetrics(id, bucketSize, min, max) + } else { + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.ClusterStats.Enabled && meta.Config.MonitorConfigs.ClusterStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.ClusterStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.ClusterHealth.Enabled && meta.Config.MonitorConfigs.ClusterHealth.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.ClusterStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + metrics = h.GetClusterMetrics(id, bucketSize, min, max) + } + + resBody["metrics"] = metrics + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } + +} + +func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + meta := elastic.GetMetadata(id) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.NodeStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.NodeStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + nodeName := h.Get(req, "node_name", "") + top := h.GetIntOrDefault(req, "top", 5) + resBody["metrics"], err = h.getNodeMetrics(id, bucketSize, min, max, nodeName, top) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + meta := elastic.GetMetadata(id) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.IndexStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.IndexStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + indexName := h.Get(req, "index_name", "") + top := h.GetIntOrDefault(req, "top", 5) + metrics := h.getIndexMetrics(req, id, bucketSize, min, max, indexName, top) + if metrics["doc_count"] != nil && metrics["docs_deleted"] != nil && len(metrics["doc_count"].Lines) > 0 && len(metrics["docs_deleted"].Lines) > 0 { + metricA := metrics["doc_count"] + metricB := metrics["docs_deleted"] + if dataA, ok := metricA.Lines[0].Data.([][]interface{}); ok { + if dataB, ok := metricB.Lines[0].Data.([][]interface{}); ok { + data := make([]map[string]interface{}, 0, len(dataA)*2) + var ( + x1 float64 + x2 float64 + ) + for i := 0; i < len(dataA); i++ { + x1 = dataA[i][1].(float64) + x2 = dataB[i][1].(float64) + if x1+x2 == 0 { + continue + } + data = append(data, map[string]interface{}{ + "x": dataA[i][0], + "y": x1 / (x1 + x2) * 100, + "g": "Doc Count", + }) + data = append(data, map[string]interface{}{ + "x": dataA[i][0], + "y": x2 / (x1 + x2) * 100, + "g": "Doc Deleted", + }) + } + metricDocPercent := &common.MetricItem{ + Axis: []*common.MetricAxis{}, + Key: "doc_percent", + Group: metricA.Group, + Order: 18, + Lines: []*common.MetricLine{ + { + TimeRange: metricA.Lines[0].TimeRange, + Data: data, + Type: common.GraphTypeBar, + }, + }, + } + metrics["doc_percent"] = metricDocPercent + } + } + + } + resBody["metrics"] = metrics + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top index may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} +func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 10, 90) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + nodeName := h.Get(req, "node_name", "") + top := h.GetIntOrDefault(req, "top", 5) + meta := elastic.GetMetadata(id) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.NodeStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.NodeStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + resBody["metrics"] = h.getThreadPoolMetrics(id, bucketSize, min, max, nodeName, top) + ver := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).GetVersion() + if ver.Distribution == "" { + cr, err := util.VersionCompare(ver.Number, "6.1") + if err != nil { + log.Error(err) + } + if cr < 0 { + resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" + } + } + + err = h.WriteJSON(w, resBody, http.StatusOK) + if err != nil { + log.Error(err) + } +} + +// TODO, use expired hash +var clusters = map[string]elastic.ElasticsearchConfig{} +var clustersMutex = &sync.RWMutex{} + +// TODO use prefered client +func (h *APIHandler) GetClusterClient(id string) (bool, elastic.API, error) { + clustersMutex.RLock() + config, ok := clusters[id] + clustersMutex.RUnlock() + + var client elastic.API + + if !ok { + client = elastic.GetClientNoPanic(id) + } + + if client == nil { + indexName := orm.GetIndexName(elastic.ElasticsearchConfig{}) + getResponse, err := h.Client().Get(indexName, "", id) + if err != nil { + return false, nil, err + } + + bytes := util.MustToJSONBytes(getResponse.Source) + cfg := elastic.ElasticsearchConfig{} + err = util.FromJSONBytes(bytes, &cfg) + if err != nil { + return false, nil, err + } + + if getResponse.StatusCode == http.StatusNotFound { + return false, nil, err + } + + cfg.ID = id + clustersMutex.Lock() + clusters[id] = cfg + clustersMutex.Unlock() + config = cfg + + client, _ = common.InitClientWithConfig(config) + + } + + return true, client, nil +} + +func (h *APIHandler) GetClusterHealth(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + id := ps.ByName("id") + exists, client, err := h.GetClusterClient(id) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", id) + log.Warn(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + health, _ := client.ClusterHealth(context.Background()) + + h.WriteJSON(w, health, 200) +} + +func (h *APIHandler) HandleGetNodesAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + metaData := elastic.GetMetadata(id) + result := util.MapStr{} + if metaData == nil || metaData.Nodes == nil { + h.WriteError(w, "nodes metadata not found", 500) + return + } + for k, nodeInfo := range *metaData.Nodes { + result[k] = util.MapStr{ + "name": nodeInfo.Name, + "transport_address": nodeInfo.TransportAddress, + } + } + h.WriteJSON(w, result, 200) +} + +const ( + SystemGroupKey = "system" + OperationGroupKey = "operations" + LatencyGroupKey = "latency" + CacheGroupKey = "cache" + HttpGroupKey = "http" + MemoryGroupKey = "memory" + StorageGroupKey = "storage" + JVMGroupKey = "JVM" + TransportGroupKey = "transport" + DocumentGroupKey = "document" + IOGroupKey = "io" + CircuitBreakerGroupKey = "circuit_breaker" +) + +func (h *APIHandler) GetClusterMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem { + + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterMetricItems := []*common.MetricItem{} + metricItem := newMetricItem("cluster_storage", 8, StorageGroupKey) + metricItem.AddAxi("indices_storage", "group1", common.PositionLeft, "bytes", "0.[0]", "0.[0]", 5, true) + metricItem.AddAxi("available_storage", "group2", common.PositionRight, "bytes", "0.[0]", "0.[0]", 5, true) + + metricItem.AddLine("Disk", "Indices Storage", "", "group1", "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Disk", "Available Disk", "", "group2", "payload.elasticsearch.cluster_stats.nodes.fs.available_in_bytes", "max", bucketSizeStr, "", "bytes", "0,0.[00]", "0,0.[00]", false, false) + + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("cluster_documents", 4, StorageGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddAxi("deleted", "group2", common.PositionRight, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Documents Count", "Documents Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.docs.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Documents Deleted", "Documents Deleted", "", "group2", "payload.elasticsearch.cluster_stats.indices.docs.deleted", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("cluster_indices", 6, StorageGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Indices Count", "Indices Count", "", "group1", "payload.elasticsearch.cluster_stats.indices.count", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + clusterMetricItems = append(clusterMetricItems, metricItem) + + metricItem = newMetricItem("node_count", 5, MemoryGroupKey) + metricItem.AddAxi("count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + meta := elastic.GetMetadata(id) + if meta == nil { + err := fmt.Errorf("metadata of cluster [%s] is not found", id) + panic(err) + } + majorVersion := meta.GetMajorVersion() + + metricItem.AddLine("Total", "Total Nodes", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.total", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + + //TODO check version difference + if majorVersion < 5 { + metricItem.AddLine("Master Only", "Master Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Data Node", "Data Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Master Data", "Master Data", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master_data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + } else { + metricItem.AddLine("Master Node", "Master Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.master", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Data Node", "Data Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.data", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Coordinating Node Only", "Coordinating Node Only", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.coordinating_only", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + metricItem.AddLine("Ingest Node", "Ingest Node", "", "group1", "payload.elasticsearch.cluster_stats.nodes.count.ingest", "max", bucketSizeStr, "", "num", "0.[00]", "0.[00]", false, false) + } + + clusterMetricItems = append(clusterMetricItems, metricItem) + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + //todo: since there is four queries, we can change these query to async + indexMetricsResult := h.GetClusterIndexMetrics(id, bucketSize, min, max) + clusterMetricsResult := h.getSingleMetrics(clusterMetricItems, query, bucketSize) + for k, v := range clusterMetricsResult { + indexMetricsResult[k] = v + } + statusMetric, err := h.getClusterStatusMetric(id, min, max, bucketSize) + if err == nil { + indexMetricsResult["cluster_health"] = statusMetric + } else { + log.Error("get cluster status metric error: ", err) + } + clusterHealthMetricsResult := h.getShardsMetric(id, min, max, bucketSize) + for k, v := range clusterHealthMetricsResult { + indexMetricsResult[k] = v + } + // get CircuitBreaker metric + circuitBreakerMetricsResult := h.getCircuitBreakerMetric(id, min, max, bucketSize) + for k, v := range circuitBreakerMetricsResult { + indexMetricsResult[k] = v + } + + return indexMetricsResult +} +func (h *APIHandler) GetClusterIndexMetrics(id string, bucketSize int, min, max int64) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + metricItems := []*common.MetricItem{} + metricItem := newMetricItem("index_throughput", 2, OperationGroupKey) + metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true) + metricItem.AddLine("Indexing Rate", "Total Indexing", "Number of documents being indexed for primary and replica shards.", "group1", "payload.elasticsearch.index_stats.total.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Indexing Rate", "Primary Indexing", "Number of documents being indexed for primary shards.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_total", "max", bucketSizeStr, "doc/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("search_throughput", 2, OperationGroupKey) + metricItem.AddAxi("searching", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Search Rate", "Total Query", + "Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!", + "group1", "payload.elasticsearch.index_stats.total.search.query_total", "max", bucketSizeStr, "query/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("index_latency", 3, LatencyGroupKey) + metricItem.AddAxi("indexing", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, true) + + metricItem.AddLine("Indexing", "Indexing Latency", "Average latency for indexing documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Indexing", "Delete Latency", "Average latency for delete documents.", "group1", "payload.elasticsearch.index_stats.primaries.indexing.delete_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.primaries.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItems = append(metricItems, metricItem) + + metricItem = newMetricItem("search_latency", 3, LatencyGroupKey) + metricItem.AddAxi("searching", "group2", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + + metricItem.AddLine("Searching", "Query Latency", "Average latency for searching query.", "group2", "payload.elasticsearch.index_stats.total.search.query_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Searching", "Fetch Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.fetch_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItem.AddLine("Searching", "Scroll Latency", "Average latency for searching fetch.", "group2", "payload.elasticsearch.index_stats.total.search.scroll_time_in_millis", "max", bucketSizeStr, "ms", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.index_stats.total.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value / value2 + } + metricItems = append(metricItems, metricItem) + query := map[string]interface{}{} + query["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.index_id": util.MapStr{ + "value": "_all", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + return h.getSingleMetrics(metricItems, query, bucketSize) +} + +func (h *APIHandler) getShardsMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_health", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + "interval": bucketSizeStr, + }, + }, + }, + } + metricItem := newMetricItem("shard_count", 7, StorageGroupKey) + metricItem.AddAxi("counts", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Active Primary Shards", "Active Primary Shards", "", "group1", "payload.elasticsearch.cluster_health.active_primary_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Active Shards", "Active Shards", "", "group1", "payload.elasticsearch.cluster_health.active_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Relocating Shards", "Relocating Shards", "", "group1", "payload.elasticsearch.cluster_health.relocating_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Initializing Shards", "Initializing Shards", "", "group1", "payload.elasticsearch.cluster_health.initializing_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Unassigned Shards", "Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + metricItem.AddLine("Delayed Unassigned Shards", "Delayed Unassigned Shards", "", "group1", "payload.elasticsearch.cluster_health.delayed_unassigned_shards", "max", bucketSizeStr, "", "num", "0,0.[00]", "0,0.[00]", false, false) + var clusterHealthMetrics []*common.MetricItem + clusterHealthMetrics = append(clusterHealthMetrics, metricItem) + return h.getSingleMetrics(clusterHealthMetrics, query, bucketSize) +} + +func (h *APIHandler) getCircuitBreakerMetric(id string, min, max int64, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + "interval": bucketSizeStr, + }, + }, + }, + } + metricItem := newMetricItem("circuit_breaker", 7, StorageGroupKey) + metricItem.AddAxi("Circuit Breaker", "group1", common.PositionLeft, "num", "0,0", "0,0.[00]", 5, false) + metricItem.AddLine("Parent Breaker Tripped", "Parent Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.parent.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Fieldaata Breaker Tripped", "Fielddata Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.fielddata.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Accounting Breaker Tripped", "Accounting Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.accounting.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("Request Breaker Tripped", "Request Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.request.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + metricItem.AddLine("In Flight Requests Breaker Tripped", "In Flight Requests Tripped", "", "group1", "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", "sum", bucketSizeStr, "times/s", "num", "0,0.[00]", "0,0.[00]", false, true) + var circuitBreakerMetrics []*common.MetricItem + circuitBreakerMetrics = append(circuitBreakerMetrics, metricItem) + return h.getSingleMetrics(circuitBreakerMetrics, query, bucketSize) +} + +func (h *APIHandler) getClusterStatusMetric(id string, min, max int64, bucketSize int) (*common.MetricItem, error) { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query := util.MapStr{ + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": id, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "cluster_stats", + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "group_status": util.MapStr{ + "terms": util.MapStr{ + "field": "payload.elasticsearch.cluster_stats.status", + "size": 5, + }, + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + metricData := []interface{}{} + metricItem := newMetricItem("cluster_health", 1, MemoryGroupKey) + metricItem.AddLine("status", "Status", "", "group1", "payload.elasticsearch.cluster_stats.status", "max", bucketSizeStr, "%", "ratio", "0.[00]", "0.[00]", false, false) + + if response.StatusCode == 200 { + metricData, err = parseHealthMetricData(response.Aggregations["dates"].Buckets) + if err != nil { + return nil, err + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func (h *APIHandler) GetClusterStatusAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var status = map[string]interface{}{} + clusterIDs, hasAllPrivilege := h.GetAllowedClusters(req) + if !hasAllPrivilege && len(clusterIDs) == 0 { + h.WriteJSON(w, status, http.StatusOK) + return + } + + elastic.WalkConfigs(func(k, value interface{}) bool { + key := k.(string) + if !hasAllPrivilege && !util.StringInArray(clusterIDs, key) { + return true + } + cfg, ok := value.(*elastic.ElasticsearchConfig) + if ok && cfg != nil { + meta := elastic.GetOrInitMetadata(cfg) + status[key] = map[string]interface{}{ + "health": meta.Health, + "available": meta.IsAvailable(), + "config": map[string]interface{}{ + "monitored": meta.Config.Monitored, + }, + } + } + return true + }) + h.WriteJSON(w, status, http.StatusOK) +} + +func (h *APIHandler) GetMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + clusterIDs, hasAllPrivilege := h.GetAllowedClusters(req) + if !hasAllPrivilege && len(clusterIDs) == 0 { + h.WriteJSON(w, result, http.StatusOK) + return + } + + elastic.WalkMetadata(func(key, value interface{}) bool { + m := util.MapStr{} + k := key.(string) + if !hasAllPrivilege && !util.StringInArray(clusterIDs, k) { + return true + } + if value == nil { + return true + } + + v, ok := value.(*elastic.ElasticsearchMetadata) + if ok { + m["major_version"] = v.GetMajorVersion() + m["seed_hosts"] = v.GetSeedHosts() + m["state"] = v.ClusterState + m["topology_version"] = v.NodesTopologyVersion + m["nodes"] = v.Nodes + //m["indices"]=v.Indices + m["health"] = v.Health + m["aliases"] = v.Aliases + //m["primary_shards"]=v.PrimaryShards + m["available"] = v.IsAvailable() + m["schema"] = v.GetSchema() + m["config"] = v.Config + m["last_success"] = v.LastSuccess() + result[k] = m + } + return true + }) + + h.WriteJSON(w, result, http.StatusOK) + +} + +func (h *APIHandler) GetMetadataByID(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + + id := ps.MustGetParameter("id") + + v := elastic.GetMetadata(id) + m := util.MapStr{} + if v != nil { + m["major_version"] = v.GetMajorVersion() + m["seed_hosts"] = v.GetSeedHosts() + m["state"] = v.ClusterState + m["topology_version"] = v.NodesTopologyVersion + m["nodes"] = v.Nodes + //m["indices"]=v.Indices + m["health"] = v.Health + m["aliases"] = v.Aliases + //m["primary_shards"]=v.PrimaryShards + m["available"] = v.IsAvailable() + m["schema"] = v.GetSchema() + m["config"] = v.Config + m["last_success"] = v.LastSuccess() + result[id] = m + } + + h.WriteJSON(w, result, http.StatusOK) + +} + +func (h *APIHandler) GetHosts(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + result := util.MapStr{} + + elastic.WalkHosts(func(key, value interface{}) bool { + k := key.(string) + if value == nil { + return true + } + + v, ok := value.(*elastic.NodeAvailable) + if ok { + result[k] = util.MapStr{ + "host": v.Host, + "available": v.IsAvailable(), + "dead": v.IsDead(), + "last_check": v.LastCheck(), + "last_success": v.LastSuccess(), + "failure_tickets": v.FailureTickets(), + } + } + return true + }) + + h.WriteJSON(w, result, http.StatusOK) + +} + +func getAllMetricsIndex() string { + return orm.GetWildcardIndexName(event.Event{}) +} + +func (h *APIHandler) HandleGetStorageMetricAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := util.MapStr{} + clusterID := ps.ByName("id") + client := elastic.GetClient(clusterID) + shardRes, err := client.CatShards() + if err != nil { + resBody["error"] = fmt.Sprintf("cat shards error: %v", err) + log.Errorf("cat shards error: %v", err) + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var metricData = TreeMapNode{ + Name: fmt.Sprintf("%s:Storage", clusterID), + SubKeys: map[string]int{}, + } + for _, shardInfo := range shardRes { + if shardInfo.ShardType != "p" { + continue + } + nodeName := fmt.Sprintf("%s:%s", shardInfo.NodeIP, shardInfo.NodeName) + //node level + if _, ok := metricData.SubKeys[nodeName]; !ok { + metricData.Children = append(metricData.Children, &TreeMapNode{ + Name: nodeName, + SubKeys: map[string]int{}, + }) + metricData.SubKeys[nodeName] = len(metricData.Children) - 1 + } + //index level + nodeIdx := metricData.SubKeys[nodeName] + if _, ok := metricData.Children[nodeIdx].SubKeys[shardInfo.Index]; !ok { + metricData.Children[nodeIdx].Children = append(metricData.Children[nodeIdx].Children, &TreeMapNode{ + Name: shardInfo.Index, + SubKeys: map[string]int{}, + }) + metricData.Children[nodeIdx].SubKeys[shardInfo.Index] = len(metricData.Children[nodeIdx].Children) - 1 + } + //shard level + indexIdx := metricData.Children[nodeIdx].SubKeys[shardInfo.Index] + value, err := util.ConvertBytesFromString(shardInfo.Store) + if err != nil { + log.Warn(err) + } + metricData.Children[nodeIdx].Children[indexIdx].Children = append(metricData.Children[nodeIdx].Children[indexIdx].Children, &TreeMapNode{ + Name: fmt.Sprintf("shard %s", shardInfo.ShardID), + Value: value, + }) + } + var ( + totalStoreSize float64 = 0 + nodeSize float64 = 0 + indexSize float64 = 0 + ) + for _, node := range metricData.Children { + nodeSize = 0 + for _, index := range node.Children { + indexSize = 0 + for _, shard := range index.Children { + indexSize += shard.Value + } + index.Value = math.Trunc(indexSize*100) / 100 + nodeSize += indexSize + } + node.Value = math.Trunc(nodeSize*100) / 100 + totalStoreSize += nodeSize + } + metricData.Value = math.Trunc(totalStoreSize*100) / 100 + h.WriteJSON(w, metricData, http.StatusOK) +} + +func getDateHistogramIntervalField(clusterID string, bucketSize string) (string, error) { + esClient := elastic.GetClient(clusterID) + ver := esClient.GetVersion() + return elastic.GetDateHistogramIntervalField(ver.Distribution, ver.Number, bucketSize) +} diff --git a/modules/elastic/api/v1/metrics_util.go b/modules/elastic/api/v1/metrics_util.go new file mode 100644 index 00000000..dee6d300 --- /dev/null +++ b/modules/elastic/api/v1/metrics_util.go @@ -0,0 +1,911 @@ +package v1 + +import ( + "fmt" + "infini.sh/framework/core/env" + "net/http" + "strings" + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" +) + +func newMetricItem(metricKey string, order int, group string) *common.MetricItem { + metricItem := common.MetricItem{ + Order: order, + Key: metricKey, + Group: group, + } + + //axis + metricItem.Axis = []*common.MetricAxis{} + + //lines + metricItem.Lines = []*common.MetricLine{} + + return &metricItem +} + +type GroupMetricItem struct { + Key string + Field string + ID string + IsDerivative bool + Units string + FormatType string + MetricItem *common.MetricItem + Field2 string + Calc func(value, value2 float64) float64 +} + +type TreeMapNode struct { + Name string `json:"name"` + Value float64 `json:"value,omitempty"` + Children []*TreeMapNode `json:"children,omitempty"` + SubKeys map[string]int `json:"-"` +} + +type MetricData map[string][][]interface{} + +func generateGroupAggs(nodeMetricItems []GroupMetricItem) map[string]interface{} { + aggs := map[string]interface{}{} + + for _, metricItem := range nodeMetricItems { + aggs[metricItem.ID] = util.MapStr{ + "max": util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID+"_field2"] = util.MapStr{ + "max": util.MapStr{ + "field": metricItem.Field2, + }, + } + } + + if metricItem.IsDerivative { + aggs[metricItem.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + return aggs +} + +func (h *APIHandler) getMetrics(query map[string]interface{}, grpMetricItems []GroupMetricItem, bucketSize int) map[string]*common.MetricItem { + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + grpMetricItemsIndex := map[string]int{} + for i, item := range grpMetricItems { + grpMetricItemsIndex[item.ID] = i + } + grpMetricData := map[string]MetricData{} + + var minDate, maxDate int64 + if response.StatusCode == 200 { + if nodeAgg, ok := response.Aggregations["group_by_level"]; ok { + for _, bucket := range nodeAgg.Buckets { + grpKey := bucket["key"].(string) + for _, metricItem := range grpMetricItems { + metricItem.MetricItem.AddLine(metricItem.Key, grpKey, "", "group1", metricItem.Field, "max", bucketSizeStr, metricItem.Units, metricItem.FormatType, "0.[00]", "0.[00]", false, false) + dataKey := metricItem.ID + if metricItem.IsDerivative { + dataKey = dataKey + "_deriv" + } + if _, ok := grpMetricData[dataKey]; !ok { + grpMetricData[dataKey] = map[string][][]interface{}{} + } + grpMetricData[dataKey][grpKey] = [][]interface{}{} + } + if datesAgg, ok := bucket["dates"].(map[string]interface{}); ok { + if datesBuckets, ok := datesAgg["buckets"].([]interface{}); ok { + for _, dateBucket := range datesBuckets { + if bucketMap, ok := dateBucket.(map[string]interface{}); ok { + v, ok := bucketMap["key"].(float64) + if !ok { + panic("invalid bucket key") + } + dateTime := (int64(v)) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + + for mk1, mv1 := range grpMetricData { + v1, ok := bucketMap[mk1] + if ok { + v2, ok := v1.(map[string]interface{}) + if ok { + v3, ok := v2["value"].(float64) + if ok { + metricID := mk1 + if strings.HasSuffix(mk1, "_deriv") { + metricID = strings.TrimSuffix(mk1, "_deriv") + if _, ok := bucketMap[mk1+"_field2"]; !ok { + v3 = v3 / float64(bucketSize) + } + } + if field2, ok := bucketMap[mk1+"_field2"]; ok { + if idx, ok := grpMetricItemsIndex[metricID]; ok { + if field2Map, ok := field2.(map[string]interface{}); ok { + v4 := field2Map["value"].(float64) + if v4 == 0 { + v3 = 0 + } else { + v3 = grpMetricItems[idx].Calc(v3, v4) + } + } + } + } + if v3 < 0 { + continue + } + points := []interface{}{dateTime, v3} + mv1[grpKey] = append(mv1[grpKey], points) + } + } + } + } + } + } + } + + } + } + } + } + + result := map[string]*common.MetricItem{} + + for _, metricItem := range grpMetricItems { + for _, line := range metricItem.MetricItem.Lines { + line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + dataKey := metricItem.ID + if metricItem.IsDerivative { + dataKey = dataKey + "_deriv" + } + line.Data = grpMetricData[dataKey][line.Metric.Label] + } + result[metricItem.Key] = metricItem.MetricItem + } + return result +} + +func GetMinBucketSize() int { + metricsCfg := struct { + MinBucketSizeInSeconds int `config:"min_bucket_size_in_seconds"` + }{ + MinBucketSizeInSeconds: 20, + } + _, _ = env.ParseConfig("insight", &metricsCfg) + if metricsCfg.MinBucketSizeInSeconds < 20 { + metricsCfg.MinBucketSizeInSeconds = 20 + } + return metricsCfg.MinBucketSizeInSeconds +} + +// defaultBucketSize 也就是每次聚合的时间间隔 +func (h *APIHandler) getMetricRangeAndBucketSize(req *http.Request, defaultBucketSize, defaultMetricCount int) (int, int64, int64, error) { + minBucketSizeInSeconds := GetMinBucketSize() + if defaultBucketSize <= 0 { + defaultBucketSize = minBucketSizeInSeconds + } + if defaultMetricCount <= 0 { + defaultMetricCount = 15 * 60 + } + bucketSize := defaultBucketSize + + bucketSizeStr := h.GetParameterOrDefault(req, "bucket_size", "") //默认 10，每个 bucket 的时间范围，单位秒 + if bucketSizeStr != "" { + du, err := util.ParseDuration(bucketSizeStr) + if err != nil { + return 0, 0, 0, err + } + bucketSize = int(du.Seconds()) + }else { + bucketSize = 0 + } + metricCount := h.GetIntOrDefault(req, "metric_count", defaultMetricCount) //默认 15分钟的区间，每分钟15个指标，也就是 15*6 个 bucket //90 + //min,max are unix nanoseconds + + minStr := h.Get(req, "min", "") + maxStr := h.Get(req, "max", "") + + return GetMetricRangeAndBucketSize(minStr, maxStr, bucketSize, metricCount) +} + +func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, metricCount int) (int, int64, int64, error) { + var min, max int64 + var rangeFrom, rangeTo time.Time + var err error + var useMinMax = bucketSize == 0 + now := time.Now() + if minStr == "" { + rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + } else { + //try 2021-08-21T14:06:04.818Z + rangeFrom, err = util.ParseStandardTime(minStr) + if err != nil { + //try 1629637500000 + v, err := util.ToInt64(minStr) + if err != nil { + log.Error("invalid timestamp:", minStr, err) + rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + } else { + rangeFrom = util.FromUnixTimestamp(v / 1000) + } + } + } + + if maxStr == "" { + rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + } else { + rangeTo, err = util.ParseStandardTime(maxStr) + if err != nil { + v, err := util.ToInt64(maxStr) + if err != nil { + log.Error("invalid timestamp:", maxStr, err) + rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + } else { + rangeTo = util.FromUnixTimestamp(int64(v) / 1000) + } + } + } + + min = rangeFrom.UnixNano() / 1e6 + max = rangeTo.UnixNano() / 1e6 + hours := rangeTo.Sub(rangeFrom).Hours() + + if useMinMax { + + if hours <= 0.25 { + bucketSize = GetMinBucketSize() + } else if hours <= 0.5 { + bucketSize = 30 + } else if hours <= 2 { + bucketSize = 60 + } else if hours < 3 { + bucketSize = 90 + } else if hours < 6 { + bucketSize = 120 + } else if hours < 12 { + bucketSize = 60 * 3 + } else if hours < 25 { //1day + bucketSize = 60 * 5 * 2 + } else if hours <= 7*24+1 { //7days + bucketSize = 60 * 15 * 2 + } else if hours <= 15*24+1 { //15days + bucketSize = 60 * 30 * 2 + } else if hours < 30*24+1 { //<30 days + bucketSize = 60 * 60 //hourly + } else if hours <= 30*24+1 { //<30days + bucketSize = 12 * 60 * 60 //half daily + } else if hours >= 30*24+1 { //>30days + bucketSize = 60 * 60 * 24 //daily bucket + } + } + + return bucketSize, min, max, nil +} + +// 获取单个指标，可以包含多条曲线 +func (h *APIHandler) getSingleMetrics(metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) map[string]*common.MetricItem { + metricData := map[string][][]interface{}{} + + aggs := map[string]interface{}{} + metricItemsMap := map[string]*common.MetricLine{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + metricItemsMap[line.Metric.GetDataKey()] = line + metricData[line.Metric.GetDataKey()] = [][]interface{}{} + + aggs[line.Metric.ID] = util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field, + }, + } + if line.Metric.Field2 != "" { + aggs[line.Metric.ID+"_field2"] = util.MapStr{ + line.Metric.MetricAgg: util.MapStr{ + "field": line.Metric.Field2, + }, + } + } + + if line.Metric.IsDerivative { + //add which metric keys to extract + aggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + if line.Metric.Field2 != "" { + aggs[line.Metric.ID+"_deriv_field2"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID + "_field2", + }, + } + } + } + } + } + bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + + clusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + intervalField, err := getDateHistogramIntervalField(clusterID, bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + query["size"] = 0 + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": aggs, + }, + } + response, err := elastic.GetClient(clusterID).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + + var minDate, maxDate int64 + if response.StatusCode == 200 { + for _, v := range response.Aggregations { + for _, bucket := range v.Buckets { + v, ok := bucket["key"].(float64) + if !ok { + panic("invalid bucket key") + } + dateTime := (int64(v)) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + for mk1, mv1 := range metricData { + v1, ok := bucket[mk1] + if ok { + v2, ok := v1.(map[string]interface{}) + if ok { + v3, ok := v2["value"].(float64) + if ok { + if strings.HasSuffix(mk1, "_deriv") { + if _, ok := bucket[mk1+"_field2"]; !ok { + v3 = v3 / float64(bucketSize) + } + } + if field2, ok := bucket[mk1+"_field2"]; ok { + if line, ok := metricItemsMap[mk1]; ok { + if field2Map, ok := field2.(map[string]interface{}); ok { + v4 := field2Map["value"].(float64) + if v4 == 0 { + v3 = 0 + } else { + v3 = line.Metric.Calc(v3, v4) + } + } + } + } + if v3 < 0 { + continue + } + points := []interface{}{dateTime, v3} + metricData[mk1] = append(mv1, points) + } + + } + } + } + } + } + } + + result := map[string]*common.MetricItem{} + + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + line.Data = metricData[line.Metric.GetDataKey()] + } + result[metricItem.Key] = metricItem + } + + return result +} + +//func (h *APIHandler) executeQuery(query map[string]interface{}, bucketItems *[]common.BucketItem, bucketSize int) map[string]*common.MetricItem { +// response, err := elastic.GetClient(h.Config.Elasticsearch).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) +// +//} + +func (h *APIHandler) getBucketMetrics(query map[string]interface{}, bucketItems *[]common.BucketItem, bucketSize int) map[string]*common.MetricItem { + //bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + panic(err) + } + //grpMetricItemsIndex := map[string]int{} + for _, item := range *bucketItems { + //grpMetricItemsIndex[item.Key] = i + + agg, ok := response.Aggregations[item.Key] + if ok { + fmt.Println(len(agg.Buckets)) + } + + } + //grpMetricData := map[string]MetricData{} + + //var minDate, maxDate int64 + //if response.StatusCode == 200 { + // if nodeAgg, ok := response.Aggregations["group_by_level"]; ok { + // for _, bucket := range nodeAgg.Buckets { + // grpKey := bucket["key"].(string) + // for _, metricItem := range *bucketItems { + // metricItem.MetricItem.AddLine(metricItem.Key, grpKey, "", "group1", metricItem.Field, "max", bucketSizeStr, metricItem.Units, metricItem.FormatType, "0.[00]", "0.[00]", false, false) + // dataKey := metricItem.Key + // if metricItem.IsDerivative { + // dataKey = dataKey + "_deriv" + // } + // if _, ok := grpMetricData[dataKey]; !ok { + // grpMetricData[dataKey] = map[string][][]interface{}{} + // } + // grpMetricData[dataKey][grpKey] = [][]interface{}{} + // } + // if datesAgg, ok := bucket["dates"].(map[string]interface{}); ok { + // if datesBuckets, ok := datesAgg["buckets"].([]interface{}); ok { + // for _, dateBucket := range datesBuckets { + // if bucketMap, ok := dateBucket.(map[string]interface{}); ok { + // v, ok := bucketMap["key"].(float64) + // if !ok { + // panic("invalid bucket key") + // } + // dateTime := (int64(v)) + // minDate = util.MinInt64(minDate, dateTime) + // maxDate = util.MaxInt64(maxDate, dateTime) + // + // for mk1, mv1 := range grpMetricData { + // v1, ok := bucketMap[mk1] + // if ok { + // v2, ok := v1.(map[string]interface{}) + // if ok { + // v3, ok := v2["value"].(float64) + // if ok { + // if strings.HasSuffix(mk1, "_deriv") { + // v3 = v3 / float64(bucketSize) + // } + // if field2, ok := bucketMap[mk1+"_field2"]; ok { + // if idx, ok := grpMetricItemsIndex[mk1]; ok { + // if field2Map, ok := field2.(map[string]interface{}); ok { + // v3 = grpMetricItems[idx].Calc(v3, field2Map["value"].(float64)) + // } + // } + // } + // if v3 < 0 { + // continue + // } + // points := []interface{}{dateTime, v3} + // mv1[grpKey] = append(mv1[grpKey], points) + // } + // } + // } + // } + // } + // } + // } + // + // } + // } + // } + //} + // + //result := map[string]*common.MetricItem{} + // + //for _, metricItem := range grpMetricItems { + // for _, line := range metricItem.MetricItem.Lines { + // line.TimeRange = common.TimeRange{Min: minDate, Max: maxDate} + // dataKey := metricItem.ID + // if metricItem.IsDerivative { + // dataKey = dataKey + "_deriv" + // } + // line.Data = grpMetricData[dataKey][line.ElasticsearchMetric.Label] + // } + // result[metricItem.Key] = metricItem.MetricItem + //} + return nil +} + +func ConvertMetricItemsToAggQuery(metricItems []*common.MetricItem) map[string]interface{} { + aggs := map[string]interface{}{} + for _, metricItem := range metricItems { + for _, line := range metricItem.Lines { + aggs[line.Metric.ID] = util.MapStr{ + "max": util.MapStr{ + "field": line.Metric.Field, + }, + } + if line.Metric.IsDerivative { + //add which metric keys to extract + aggs[line.Metric.ID+"_deriv"] = util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": line.Metric.ID, + }, + } + } + } + } + return aggs +} + +func ConvertBucketItemsToAggQuery(bucketItems []*common.BucketItem, metricItems []*common.MetricItem) util.MapStr { + aggs := util.MapStr{} + + var currentAgg = util.MapStr{} + for _, bucketItem := range bucketItems { + + bucketAgg := util.MapStr{} + + switch bucketItem.Type { + case "terms": + bucketAgg = util.MapStr{ + "terms": bucketItem.Parameters, + } + break + case "date_histogram": + bucketAgg = util.MapStr{ + "date_histogram": bucketItem.Parameters, + } + break + case "date_range": + bucketAgg = util.MapStr{ + "date_range": bucketItem.Parameters, + } + break + } + + //if bucketItem.Buckets!=nil&&len(bucketItem.Buckets)>0{ + nestedAggs := ConvertBucketItemsToAggQuery(bucketItem.Buckets, bucketItem.Metrics) + if len(nestedAggs) > 0 { + util.MergeFields(bucketAgg, nestedAggs, true) + } + //} + currentAgg[bucketItem.Key] = bucketAgg + } + + if metricItems != nil && len(metricItems) > 0 { + metricAggs := ConvertMetricItemsToAggQuery(metricItems) + util.MergeFields(currentAgg, metricAggs, true) + } + + aggs = util.MapStr{ + "aggs": currentAgg, + } + + return aggs +} + +type BucketBase map[string]interface{} + +func (receiver BucketBase) GetChildBucket(name string) (map[string]interface{}, bool) { + bks, ok := receiver[name] + if ok { + bks2, ok := bks.(map[string]interface{}) + return bks2, ok + } + return nil, false +} + +type Bucket struct { + BucketBase //子 buckets + + KeyAsString string `json:"key_as_string,omitempty"` + Key interface{} `json:"key,omitempty"` + DocCount int64 `json:"doc_count,omitempty"` + DocCountErrorUpperBound int64 `json:"doc_count_error_upper_bound,omitempty"` + SumOtherDocCount int64 `json:"sum_other_doc_count,omitempty"` + + Buckets []Bucket `json:"buckets,omitempty"` //本 buckets +} + +type SearchResponse struct { + Took int `json:"took"` + TimedOut bool `json:"timed_out"` + Hits struct { + Total interface{} `json:"total"` + MaxScore float32 `json:"max_score"` + } `json:"hits"` + Aggregations util.MapStr `json:"aggregations,omitempty"` +} + +func ParseAggregationBucketResult(bucketSize int, aggsData util.MapStr, groupKey, resultLabelKey, resultValueKey string, resultItemHandle func()) MetricData { + + metricData := MetricData{} + for k, v := range aggsData { + if k == groupKey { + //start to collect metric for each bucket + objcs, ok := v.(map[string]interface{}) + if ok { + + bks, ok := objcs["buckets"].([]interface{}) + if ok { + for _, bk := range bks { + //check each bucket, collecting metrics + bkMap, ok := bk.(map[string]interface{}) + if ok { + + groupKeyValue, ok := bkMap["key"] + if ok { + } + bkHitMap, ok := bkMap[resultLabelKey] + if ok { + //hit label, 说明匹配到时间范围了 + labelMap, ok := bkHitMap.(map[string]interface{}) + if ok { + labelBks, ok := labelMap["buckets"] + if ok { + labelBksMap, ok := labelBks.([]interface{}) + if ok { + for _, labelItem := range labelBksMap { + metrics, ok := labelItem.(map[string]interface{}) + + labelKeyValue, ok := metrics["to"] //TODO config + if !ok { + labelKeyValue, ok = metrics["from"] //TODO config + } + if !ok { + labelKeyValue, ok = metrics["key"] //TODO config + } + + metric, ok := metrics[resultValueKey] + if ok { + metricMap, ok := metric.(map[string]interface{}) + if ok { + t := "bucket" //metric, bucket + if t == "metric" { + metricValue, ok := metricMap["value"] + if ok { + saveMetric(&metricData, groupKeyValue.(string), labelKeyValue, metricValue, bucketSize) + continue + } + } else { + metricValue, ok := metricMap["buckets"] + if ok { + buckets, ok := metricValue.([]interface{}) + if ok { + var result string = "unavailable" + for _, v := range buckets { + x, ok := v.(map[string]interface{}) + if ok { + if x["key"] == "red" { + result = "red" + break + } + if x["key"] == "yellow" { + result = "yellow" + } else { + if result != "yellow" { + result = x["key"].(string) + } + } + } + } + + v, ok := (metricData)[groupKeyValue.(string)] + if !ok { + v = [][]interface{}{} + } + v2 := []interface{}{} + v2 = append(v2, labelKeyValue) + v2 = append(v2, result) + v = append(v, v2) + + (metricData)[groupKeyValue.(string)] = v + } + + continue + } + } + } + } + } + } + + } + } + + } + } + } + } + } + + } + + } + + return metricData +} + +func ParseAggregationResult(bucketSize int, aggsData util.MapStr, groupKey, metricLabelKey, metricValueKey string) MetricData { + + metricData := MetricData{} + //group bucket key: key1, 获取 key 的 buckets 作为分组的内容 map[group][]{Label，MetricValue} + //metric Label Key: key2, 获取其 key 作为 时间指标 + //metric Value Key: c7qgjrqi4h92sqdaa9b0, 获取其 value 作为 point 内容 + + //groupKey:="key1" + //metricLabelKey:="key2" + //metricValueKey:="c7qi5hii4h935v9bs920" + + //fmt.Println(groupKey," => ",metricLabelKey," => ",metricValueKey) + + for k, v := range aggsData { + //fmt.Println("k:",k) + //fmt.Println("v:",v) + + if k == groupKey { + //fmt.Println("hit group key") + //start to collect metric for each bucket + objcs, ok := v.(map[string]interface{}) + if ok { + + bks, ok := objcs["buckets"].([]interface{}) + if ok { + for _, bk := range bks { + //check each bucket, collecting metrics + //fmt.Println("check bucket:",bk) + + bkMap, ok := bk.(map[string]interface{}) + if ok { + + groupKeyValue, ok := bkMap["key"] + if ok { + //fmt.Println("collecting bucket::",groupKeyValue) + } + bkHitMap, ok := bkMap[metricLabelKey] + if ok { + //hit label, 说明匹配到时间范围了 + labelMap, ok := bkHitMap.(map[string]interface{}) + if ok { + //fmt.Println("bkHitMap",bkHitMap) + + labelBks, ok := labelMap["buckets"] + if ok { + + labelBksMap, ok := labelBks.([]interface{}) + //fmt.Println("get label buckets",ok) + if ok { + //fmt.Println("get label buckets",ok) + + for _, labelItem := range labelBksMap { + metrics, ok := labelItem.(map[string]interface{}) + + //fmt.Println(labelItem) + labelKeyValue, ok := metrics["key"] + if ok { + //fmt.Println("collecting metric label::",int64(labelKeyValue.(float64))) + } + + metric, ok := metrics[metricValueKey] + if ok { + metricMap, ok := metric.(map[string]interface{}) + if ok { + metricValue, ok := metricMap["value"] + if ok { + //fmt.Println("collecting metric value::",metricValue.(float64)) + + saveMetric(&metricData, groupKeyValue.(string), labelKeyValue, metricValue, bucketSize) + continue + } + } + } + } + } + + } + } + + } + } + } + } + } + + } + + } + + //for k,v:=range bucketItems{ + // fmt.Println("k:",k) + // fmt.Println("v:",v) + // aggObect:=aggsData[v.Key] + // fmt.Println("",aggObect) + // //fmt.Println(len(aggObect.Buckets)) + // //for _,bucket:=range aggObect.Buckets{ + // // fmt.Println(bucket.Key) + // // fmt.Println(bucket.GetChildBucket("key2")) + // // //children,ok:=bucket.GetChildBucket() + // // //if ok{ + // // // + // // //} + // //} + //} + + return metricData +} + +func saveMetric(metricData *MetricData, group string, label, value interface{}, bucketSize int) { + + if value == nil { + return + } + + v3, ok := value.(float64) + if ok { + value = v3 / float64(bucketSize) + } + + v, ok := (*metricData)[group] + if !ok { + v = [][]interface{}{} + } + v2 := []interface{}{} + v2 = append(v2, label) + v2 = append(v2, value) + v = append(v, v2) + + (*metricData)[group] = v + //fmt.Printf("save:%v, %v=%v\n",group,label,value) +} + +func parseHealthMetricData(buckets []elastic.BucketBase) ([]interface{}, error) { + metricData := []interface{}{} + var minDate, maxDate int64 + for _, bucket := range buckets { + v, ok := bucket["key"].(float64) + if !ok { + log.Error("invalid bucket key") + return nil, fmt.Errorf("invalid bucket key") + } + dateTime := int64(v) + minDate = util.MinInt64(minDate, dateTime) + maxDate = util.MaxInt64(maxDate, dateTime) + totalCount := bucket["doc_count"].(float64) + if grpStatus, ok := bucket["group_status"].(map[string]interface{}); ok { + if statusBks, ok := grpStatus["buckets"].([]interface{}); ok { + for _, statusBk := range statusBks { + if bkMap, ok := statusBk.(map[string]interface{}); ok { + statusKey := bkMap["key"].(string) + count := bkMap["doc_count"].(float64) + metricData = append(metricData, map[string]interface{}{ + "x": dateTime, + "y": count / totalCount * 100, + "g": statusKey, + }) + } + } + } + } + } + return metricData, nil +} diff --git a/modules/elastic/api/v1/metrics_util_test.go b/modules/elastic/api/v1/metrics_util_test.go new file mode 100644 index 00000000..772bf1e1 --- /dev/null +++ b/modules/elastic/api/v1/metrics_util_test.go @@ -0,0 +1,92 @@ +package v1 + +import ( + "fmt" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "net/http" + "testing" + "time" +) + +func TestGetMetricParams(t *testing.T) { + handler:=APIHandler{} + req:=http.Request{} + bucketSize, min, max, err:=handler.getMetricRangeAndBucketSize(&req,60,15) + + fmt.Println(bucketSize) + fmt.Println(util.FormatUnixTimestamp(min/1000))//2022-01-27 15:28:57 + fmt.Println(util.FormatUnixTimestamp(max/1000))//2022-01-27 15:28:57 + fmt.Println(time.Now())//2022-01-27 15:28:57 + + fmt.Println(bucketSize, min, max, err) +} + +func TestConvertBucketItemsToAggQueryParams(t *testing.T) { + bucketItem:=common.BucketItem{} + bucketItem.Key="key1" + bucketItem.Type=common.TermsBucket + bucketItem.Parameters=map[string]interface{}{} + bucketItem.Parameters["field"]="metadata.labels.cluster_id" + bucketItem.Parameters["size"]=2 + + + nestBucket:=common.BucketItem{} + nestBucket.Key="key2" + nestBucket.Type=common.DateHistogramBucket + nestBucket.Parameters=map[string]interface{}{} + nestBucket.Parameters["field"]="timestamp" + nestBucket.Parameters["calendar_interval"]="1d" + nestBucket.Parameters["time_zone"]="+08:00" + + leafBucket:=common.NewBucketItem(common.TermsBucket,util.MapStr{ + "size":5, + "field":"payload.elasticsearch.cluster_health.status", + }) + + leafBucket.Key="key3" + + metricItems:=[]*common.MetricItem{} + var bucketSizeStr ="10s" + metricItem:=newMetricItem("cluster_summary", 2, "cluster") + metricItem.Key="key4" + metricItem.AddLine("Indexing","Total Indexing","Number of documents being indexed for primary and replica shards.","group1", + "payload.elasticsearch.index_stats.total.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItem.AddLine("Search","Total Search","Number of search requests being executed across primary and replica shards. A single search can run against multiple shards!","group1", + "payload.elasticsearch.index_stats.total.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + nestBucket.AddNestBucket(leafBucket) + nestBucket.Metrics=metricItems + + bucketItem.Buckets=[]*common.BucketItem{} + bucketItem.Buckets=append(bucketItem.Buckets,&nestBucket) + + + aggs:=ConvertBucketItemsToAggQuery([]*common.BucketItem{&bucketItem},nil) + fmt.Println(util.MustToJSON(aggs)) + + response:="{ \"took\": 37, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 10000, \"relation\": \"gte\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"key1\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7pqhptj69a0sg3rn05g\", \"doc_count\": 80482, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-28T00:00:00.000+08:00\", \"key\": 1643299200000, \"doc_count\": 14310, \"c7qi5hii4h935v9bs91g\": { \"value\": 15680 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 2985 } }, { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 66172, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 }, \"c7qi5hii4h935v9bs91g_deriv\": { \"value\": 90526 }, \"c7qi5hii4h935v9bs920_deriv\": { \"value\": 17219 } } ] } }, { \"key\": \"c7qi42ai4h92sksk979g\", \"doc_count\": 660, \"key2\": { \"buckets\": [ { \"key_as_string\": \"2022-01-29T00:00:00.000+08:00\", \"key\": 1643385600000, \"doc_count\": 660, \"c7qi5hii4h935v9bs91g\": { \"value\": 106206 }, \"key3\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [] }, \"c7qi5hii4h935v9bs920\": { \"value\": 20204 } } ] } } ] } } }" + res:=SearchResponse{} + util.FromJSONBytes([]byte(response),&res) + fmt.Println(response) + groupKey:="key1" + metricLabelKey:="key2" + metricValueKey:="c7qi5hii4h935v9bs920" + data:=ParseAggregationResult(int(10),res.Aggregations,groupKey,metricLabelKey,metricValueKey) + fmt.Println(data) + +} + +func TestConvertBucketItems(t *testing.T) { + response:="{ \"took\": 8, \"timed_out\": false, \"_shards\": { \"total\": 1, \"successful\": 1, \"skipped\": 0, \"failed\": 0 }, \"hits\": { \"total\": { \"value\": 81, \"relation\": \"eq\" }, \"max_score\": null, \"hits\": [] }, \"aggregations\": { \"c7v2gm3i7638vvo4pv80\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"c7uv7p3i76360kgdmpb0\", \"doc_count\": 81, \"c7v2gm3i7638vvo4pv8g\": { \"buckets\": [ { \"key_as_string\": \"2022-02-05T00:00:00.000+08:00\", \"key\": 1643990400000, \"doc_count\": 81, \"c7v2gm3i7638vvo4pv90\": { \"doc_count_error_upper_bound\": 0, \"sum_other_doc_count\": 0, \"buckets\": [ { \"key\": \"yellow\", \"doc_count\": 81 } ] } } ] } } ] } } }" + res:=SearchResponse{} + util.FromJSONBytes([]byte(response),&res) + + data:=ParseAggregationBucketResult(int(10),res.Aggregations,"c7v2gm3i7638vvo4pv80","c7v2gm3i7638vvo4pv8g","c7v2gm3i7638vvo4pv90", func() { + + }) + + fmt.Println(data) + +} diff --git a/modules/elastic/api/v1/node_metrics.go b/modules/elastic/api/v1/node_metrics.go new file mode 100644 index 00000000..60cd640c --- /dev/null +++ b/modules/elastic/api/v1/node_metrics.go @@ -0,0 +1,1185 @@ +package v1 + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" + "infini.sh/framework/modules/elastic/common" + "sort" + "strings" + "time" +) + +func (h *APIHandler) getNodeMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) (map[string]*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + return nil, err + } + + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + } + var ( + nodeNames []string + ) + if nodeName != "" { + nodeNames = strings.Split(nodeName, ",") + top = len(nodeNames) + }else{ + nodeNames, err = h.getTopNodeName(clusterID, top, 15) + if err != nil { + log.Error(err) + } + } + if len(nodeNames) > 0 { + must = append(must, util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + { + "terms": util.MapStr{ + "metadata.labels.transport_address": nodeNames, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeNames, + }, + }, + }, + }, + + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + cpuMetric := newMetricItem("cpu", 1, SystemGroupKey) + cpuMetric.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + + nodeMetricItems := []GroupMetricItem{ + { + Key: "cpu", + Field: "payload.elasticsearch.node_stats.process.cpu.percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: cpuMetric, + FormatType: "ratio", + Units: "%", + }, + } + + osCpuMetric := newMetricItem("os_cpu", 2, SystemGroupKey) + osCpuMetric.AddAxi("OS CPU Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_cpu", + Field: "payload.elasticsearch.node_stats.os.cpu.percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osCpuMetric, + FormatType: "ratio", + Units: "%", + }) + + osMemMetric := newMetricItem("os_used_mem", 2, SystemGroupKey) + osMemMetric.AddAxi("OS Mem Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_used_mem", + Field: "payload.elasticsearch.node_stats.os.mem.used_percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osMemMetric, + FormatType: "ratio", + Units: "%", + }) + osLoadMetric := newMetricItem("os_load_average_1m", 2, SystemGroupKey) + osLoadMetric.AddAxi("OS Load 1m Average","group1",common.PositionLeft,"","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_load_average_1m", + Field: "payload.elasticsearch.node_stats.os.cpu.load_average.1m", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: osLoadMetric, + FormatType: "num", + Units: "", + }) + //swap usage + osSwapMetric := newMetricItem("os_used_swap", 3, SystemGroupKey) + osSwapMetric.AddAxi("OS Swap Used Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "os_used_swap", + Field: "payload.elasticsearch.node_stats.os.swap.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + Field2: "payload.elasticsearch.node_stats.os.swap.total_in_bytes", + Calc: func(value, value2 float64) float64 { + return util.ToFixed((value / value2)*100, 2) + }, + MetricItem: osSwapMetric, + FormatType: "ratio", + Units: "%", + }) + openFileMetric := newMetricItem("open_file", 2, SystemGroupKey) + openFileMetric.AddAxi("Open File Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "open_file", + Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: openFileMetric, + FormatType: "num", + Units: "", + }) + openFilePercentMetric := newMetricItem("open_file_percent", 2, SystemGroupKey) + openFilePercentMetric.AddAxi("Open File Percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "open_file_percent", + Field: "payload.elasticsearch.node_stats.process.open_file_descriptors", + ID: util.GetUUID(), + IsDerivative: false, + Field2: "payload.elasticsearch.node_stats.process.max_file_descriptors", + Calc: func(value, value2 float64) float64 { + if value < 0 { + return value + } + return util.ToFixed((value / value2)*100, 2) + }, + MetricItem: openFilePercentMetric, + FormatType: "ratio", + Units: "%", + }) + + diskMetric := newMetricItem("disk", 2, SystemGroupKey) + diskMetric.AddAxi("disk available percent","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "disk", + Field: "payload.elasticsearch.node_stats.fs.total.total_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: diskMetric, + FormatType: "ratio", + Units: "%", + Field2: "payload.elasticsearch.node_stats.fs.total.available_in_bytes", + Calc: func(value, value2 float64) float64 { + return util.ToFixed((value2 / value)*100, 2) + }, + }) + // 索引速率 + indexMetric:=newMetricItem("indexing_rate", 1, OperationGroupKey) + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_rate", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "doc/s", + }) + + indexingBytesMetric := newMetricItem("indexing_bytes", 2, OperationGroupKey) + indexingBytesMetric.AddAxi("Indexing bytes","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_bytes", + Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexingBytesMetric, + FormatType: "bytes", + Units: "bytes/s", + }) + + // 索引延时 + indexLatencyMetric:=newMetricItem("indexing_latency", 1, LatencyGroupKey) + indexLatencyMetric.AddAxi("indexing latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing_latency", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.indexing.index_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + queryMetric:=newMetricItem("query_rate", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_rate", + Field: "payload.elasticsearch.node_stats.indices.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "requests/s", + }) + + // 查询延时 + queryLatencyMetric:=newMetricItem("query_latency", 2, LatencyGroupKey) + queryLatencyMetric.AddAxi("query latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_latency", + Field: "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.query_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + fetchMetric:=newMetricItem("fetch_rate", 3, OperationGroupKey) + fetchMetric.AddAxi("fetch rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fetch_rate", + Field: "payload.elasticsearch.node_stats.indices.search.fetch_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchMetric, + FormatType: "num", + Units: "requests/s", + }) + scrollMetric:=newMetricItem("scroll_rate", 4, OperationGroupKey) + scrollMetric.AddAxi("scroll rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_rate", + Field: "payload.elasticsearch.node_stats.indices.search.scroll_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollMetric, + FormatType: "num", + Units: "requests/s", + }) + + refreshMetric:=newMetricItem("refresh_rate", 5, OperationGroupKey) + refreshMetric.AddAxi("refresh rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "refresh_rate", + Field: "payload.elasticsearch.node_stats.indices.refresh.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshMetric, + FormatType: "num", + Units: "requests/s", + }) + flushMetric:=newMetricItem("flush_rate", 6, OperationGroupKey) + flushMetric.AddAxi("flush rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "flush_rate", + Field: "payload.elasticsearch.node_stats.indices.flush.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushMetric, + FormatType: "num", + Units: "requests/s", + }) + mergeMetric:=newMetricItem("merges_rate", 7, OperationGroupKey) + mergeMetric.AddAxi("merges rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "merges_rate", + Field: "payload.elasticsearch.node_stats.indices.merges.total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeMetric, + FormatType: "num", + Units: "requests/s", + }) + + // fetch延时 + fetchLatencyMetric:=newMetricItem("fetch_latency", 3, LatencyGroupKey) + fetchLatencyMetric.AddAxi("fetch latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fetch_latency", + Field: "payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.fetch_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fetchLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // scroll 延时 + scrollLatencyMetric:=newMetricItem("scroll_latency", 4, LatencyGroupKey) + scrollLatencyMetric.AddAxi("scroll latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_latency", + Field: "payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.search.scroll_total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: scrollLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + // merge 延时 + mergeLatencyMetric:=newMetricItem("merge_latency", 7, LatencyGroupKey) + mergeLatencyMetric.AddAxi("merge latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "merge_latency", + Field: "payload.elasticsearch.node_stats.indices.merges.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.merges.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: mergeLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + // refresh 延时 + refreshLatencyMetric:=newMetricItem("refresh_latency", 5, LatencyGroupKey) + refreshLatencyMetric.AddAxi("refresh latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "refresh_latency", + Field: "payload.elasticsearch.node_stats.indices.refresh.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.refresh.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // flush 时延 + flushLatencyMetric:=newMetricItem("flush_latency", 6, LatencyGroupKey) + flushLatencyMetric.AddAxi("flush latency","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "flush_latency", + Field: "payload.elasticsearch.node_stats.indices.flush.total_time_in_millis", + Field2: "payload.elasticsearch.node_stats.indices.flush.total", + Calc: func(value, value2 float64) float64 { + return value/value2 + }, + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushLatencyMetric, + FormatType: "num", + Units: "ms", + }) + // Query Cache 内存占用大小 + queryCacheMetric:=newMetricItem("query_cache", 1, CacheGroupKey) + queryCacheMetric.AddAxi("query cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache", + Field: "payload.elasticsearch.node_stats.indices.query_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: queryCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache 内存占用大小 + requestCacheMetric:=newMetricItem("request_cache", 2, CacheGroupKey) + requestCacheMetric.AddAxi("request cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache", + Field: "payload.elasticsearch.node_stats.indices.request_cache.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: requestCacheMetric, + FormatType: "bytes", + Units: "", + }) + // Request Cache Hit + requestCacheHitMetric:=newMetricItem("request_cache_hit", 6, CacheGroupKey) + requestCacheHitMetric.AddAxi("request cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache_hit", + Field: "payload.elasticsearch.node_stats.indices.request_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + // Request Cache Miss + requestCacheMissMetric:=newMetricItem("request_cache_miss", 8, CacheGroupKey) + requestCacheMissMetric.AddAxi("request cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "request_cache_miss", + Field: "payload.elasticsearch.node_stats.indices.request_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + // Query Cache Count + queryCacheCountMetric:=newMetricItem("query_cache_count", 4, CacheGroupKey) + queryCacheCountMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_count", + Field: "payload.elasticsearch.node_stats.indices.query_cache.cache_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheCountMetric, + FormatType: "num", + Units: "", + }) + // Query Cache Miss + queryCacheHitMetric:=newMetricItem("query_cache_hit", 5, CacheGroupKey) + queryCacheHitMetric.AddAxi("query cache hit","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_hit", + Field: "payload.elasticsearch.node_stats.indices.query_cache.hit_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheHitMetric, + FormatType: "num", + Units: "hits", + }) + + //// Query Cache evictions + //queryCacheEvictionsMetric:=newMetricItem("query_cache_evictions", 5, CacheGroupKey) + //queryCacheEvictionsMetric.AddAxi("query cache evictions","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + //nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + // Key: "query_cache_evictions", + // Field: "payload.elasticsearch.node_stats.indices.query_cache.evictions", + // ID: util.GetUUID(), + // IsDerivative: true, + // MetricItem: queryCacheEvictionsMetric, + // FormatType: "num", + // Units: "evictions", + //}) + + // Query Cache Miss + queryCacheMissMetric:=newMetricItem("query_cache_miss", 7, CacheGroupKey) + queryCacheMissMetric.AddAxi("query cache miss","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "query_cache_miss", + Field: "payload.elasticsearch.node_stats.indices.query_cache.miss_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryCacheMissMetric, + FormatType: "num", + Units: "misses", + }) + + // Fielddata内存占用大小 + fieldDataCacheMetric:=newMetricItem("fielddata_cache", 3, CacheGroupKey) + fieldDataCacheMetric.AddAxi("FieldData Cache","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "fielddata_cache", + Field: "payload.elasticsearch.node_stats.indices.fielddata.memory_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: fieldDataCacheMetric, + FormatType: "bytes", + Units: "", + }) + + // http 活跃连接数 + httpActiveMetric:=newMetricItem("http_connect_num", 12, HttpGroupKey) + httpActiveMetric.AddAxi("http connect number","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "http_connect_num", + Field: "payload.elasticsearch.node_stats.http.current_open", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: httpActiveMetric, + FormatType: "num", + Units: "conns", + }) + // http 活跃连接数速率 + httpRateMetric:=newMetricItem("http_rate", 12, HttpGroupKey) + httpRateMetric.AddAxi("http rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "http_rate", + Field: "payload.elasticsearch.node_stats.http.total_opened", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: httpRateMetric, + FormatType: "num", + Units: "conns/s", + }) + + // segment 数量 + segmentCountMetric:=newMetricItem("segment_count", 15, StorageGroupKey) + segmentCountMetric.AddAxi("segment count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_count", + Field: "payload.elasticsearch.node_stats.indices.segments.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentCountMetric, + FormatType: "num", + Units: "", + }) + + // segment memory + segmentMemoryMetric:=newMetricItem("segment_memory", 16, MemoryGroupKey) + segmentMemoryMetric.AddAxi("segment memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment stored fields memory + segmentStoredFieldsMemoryMetric:=newMetricItem("segment_stored_fields_memory", 16, MemoryGroupKey) + segmentStoredFieldsMemoryMetric.AddAxi("segment stored fields memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_stored_fields_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.stored_fields_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentStoredFieldsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment terms fields memory + segmentTermsMemoryMetric:=newMetricItem("segment_terms_memory", 16, MemoryGroupKey) + segmentTermsMemoryMetric.AddAxi("segment terms memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_terms_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.terms_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment doc values memory + segmentDocValuesMemoryMetric:=newMetricItem("segment_doc_values_memory", 16, MemoryGroupKey) + segmentDocValuesMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_doc_values_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.doc_values_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentDocValuesMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment index writer memory + segmentIndexWriterMemoryMetric:=newMetricItem("segment_index_writer_memory", 16, MemoryGroupKey) + segmentIndexWriterMemoryMetric.AddAxi("segment doc values memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_index_writer_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.index_writer_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentIndexWriterMemoryMetric, + FormatType: "bytes", + Units: "", + }) + // segment term vectors memory + segmentTermVectorsMemoryMetric:=newMetricItem("segment_term_vectors_memory", 16, MemoryGroupKey) + segmentTermVectorsMemoryMetric.AddAxi("segment term vectors memory","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "segment_term_vectors_memory", + Field: "payload.elasticsearch.node_stats.indices.segments.term_vectors_memory_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: segmentTermVectorsMemoryMetric, + FormatType: "bytes", + Units: "", + }) + + // docs 数量 + docsCountMetric:=newMetricItem("docs_count", 17, DocumentGroupKey) + docsCountMetric.AddAxi("docs count","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "docs_count", + Field: "payload.elasticsearch.node_stats.indices.docs.count", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsCountMetric, + FormatType: "num", + Units: "", + }) + // docs 删除数量 + docsDeletedMetric:=newMetricItem("docs_deleted", 17, DocumentGroupKey) + docsDeletedMetric.AddAxi("docs deleted","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "docs_deleted", + Field: "payload.elasticsearch.node_stats.indices.docs.deleted", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: docsDeletedMetric, + FormatType: "num", + Units: "", + }) + + // index store size + indexStoreMetric:=newMetricItem("index_storage", 18, StorageGroupKey) + indexStoreMetric.AddAxi("indices storage","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "index_storage", + Field: "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexStoreMetric, + FormatType: "bytes", + Units: "", + }) + + // jvm used heap + jvmUsedPercentMetric:=newMetricItem("jvm_heap_used_percent", 1, JVMGroupKey) + jvmUsedPercentMetric.AddAxi("JVM heap used percent","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_heap_used_percent", + Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: jvmUsedPercentMetric, + FormatType: "num", + Units: "%", + }) + //JVM mem Young pools used + youngPoolsUsedMetric:=newMetricItem("jvm_mem_young_used", 2, JVMGroupKey) + youngPoolsUsedMetric.AddAxi("Mem Pools Young Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_young_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: youngPoolsUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM mem Young pools peak used + youngPoolsUsedPeakMetric:=newMetricItem("jvm_mem_young_peak_used", 2, JVMGroupKey) + youngPoolsUsedPeakMetric.AddAxi("Mem Pools Young Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_young_peak_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.young.peak_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: youngPoolsUsedPeakMetric, + FormatType: "bytes", + Units: "", + }) + + //JVM mem old pools used + oldPoolsUsedMetric:=newMetricItem("jvm_mem_old_used", 3, JVMGroupKey) + oldPoolsUsedMetric.AddAxi("Mem Pools Old Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_old_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: oldPoolsUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM mem old pools peak used + oldPoolsUsedPeakMetric:=newMetricItem("jvm_mem_old_peak_used", 3, JVMGroupKey) + oldPoolsUsedPeakMetric.AddAxi("Mem Pools Old Peak Used","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_mem_old_peak_used", + Field: "payload.elasticsearch.node_stats.jvm.mem.pools.old.peak_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: oldPoolsUsedPeakMetric, + FormatType: "bytes", + Units: "", + }) + + //JVM used heap + heapUsedMetric:=newMetricItem("jvm_used_heap", 1, JVMGroupKey) + heapUsedMetric.AddAxi("JVM Used Heap","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_used_heap", + Field: "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: heapUsedMetric, + FormatType: "bytes", + Units: "", + }) + //JVM Young GC Rate + gcYoungRateMetric:=newMetricItem("jvm_young_gc_rate", 2, JVMGroupKey) + gcYoungRateMetric.AddAxi("JVM Young GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_young_gc_rate", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcYoungRateMetric, + FormatType: "num", + Units: "times/s", + }) + //JVM Young GC Latency + gcYoungLatencyMetric:=newMetricItem("jvm_young_gc_latency", 2, JVMGroupKey) + gcYoungLatencyMetric.AddAxi("JVM Young GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_young_gc_latency", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.young.collection_time_in_millis", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcYoungLatencyMetric, + FormatType: "num", + Units: "ms", + }) + + //JVM old GC Rate + gcOldRateMetric:=newMetricItem("jvm_old_gc_rate", 3, JVMGroupKey) + gcOldRateMetric.AddAxi("JVM Old GC Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_old_gc_rate", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcOldRateMetric, + FormatType: "num", + Units: "times/s", + }) + //JVM old GC Latency + gcOldLatencyMetric:=newMetricItem("jvm_old_gc_latency", 3, JVMGroupKey) + gcOldLatencyMetric.AddAxi("JVM Old GC Time","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "jvm_old_gc_latency", + Field: "payload.elasticsearch.node_stats.jvm.gc.collectors.old.collection_time_in_millis", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: gcOldLatencyMetric, + FormatType: "num", + Units: "ms", + }) + //Transport 发送速率 + transTxRateMetric:=newMetricItem("transport_tx_rate", 19, TransportGroupKey) + transTxRateMetric.AddAxi("Transport Send Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_tx_rate", + Field: "payload.elasticsearch.node_stats.transport.tx_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transTxRateMetric, + FormatType: "num", + Units: "times/s", + }) + //Transport 接收速率 + transRxRateMetric:=newMetricItem("transport_rx_rate", 19, TransportGroupKey) + transRxRateMetric.AddAxi("Transport Receive Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_rx_rate", + Field: "payload.elasticsearch.node_stats.transport.rx_count", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transRxRateMetric, + FormatType: "num", + Units: "times/s", + }) + + //Transport 发送流量 + transTxBytesMetric:=newMetricItem("transport_tx_bytes", 19, TransportGroupKey) + transTxBytesMetric.AddAxi("Transport Send Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_tx_bytes", + Field: "payload.elasticsearch.node_stats.transport.tx_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transTxBytesMetric, + FormatType: "bytes", + Units: "s", + }) + //Transport 接收流量 + transRxBytesMetric:=newMetricItem("transport_rx_bytes", 19, TransportGroupKey) + transRxBytesMetric.AddAxi("Transport Receive Bytes","group1",common.PositionLeft,"bytes","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_rx_bytes", + Field: "payload.elasticsearch.node_stats.transport.rx_size_in_bytes", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: transRxBytesMetric, + FormatType: "bytes", + Units: "s", + }) + + //Transport tcp 连接数 + tcpNumMetric:=newMetricItem("transport_outbound_connections", 20, TransportGroupKey) + tcpNumMetric.AddAxi("Transport Outbound Connections","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "transport_outbound_connections", + Field: "payload.elasticsearch.node_stats.transport.total_outbound_connections", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: tcpNumMetric, + FormatType: "num", + Units: "", + }) + + //IO total + totalOperationsMetric:=newMetricItem("total_io_operations", 1, IOGroupKey) + totalOperationsMetric.AddAxi("Total I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: totalOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //IO total + readOperationsMetric:=newMetricItem("total_read_io_operations", 2, IOGroupKey) + readOperationsMetric.AddAxi("Total Read I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_read_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.read_operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: readOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //IO total + writeOperationsMetric:=newMetricItem("total_write_io_operations", 3, IOGroupKey) + writeOperationsMetric.AddAxi("Total Write I/O Operations Rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "total_write_io_operations", + Field: "payload.elasticsearch.node_stats.fs.io_stats.total.write_operations", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: writeOperationsMetric, + FormatType: "num", + Units: "times/s", + }) + + //scroll context + openContextMetric:=newMetricItem("scroll_open_contexts", 7, OperationGroupKey) + writeOperationsMetric.AddAxi("Scroll Open Contexts","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "scroll_open_contexts", + Field: "payload.elasticsearch.node_stats.indices.search.open_contexts", + ID: util.GetUUID(), + MetricItem: openContextMetric, + FormatType: "num", + Units: "", + }) + + // Circuit Breaker + parentBreakerMetric := newMetricItem("parent_breaker", 1, CircuitBreakerGroupKey) + parentBreakerMetric.AddAxi("Parent Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "parent_breaker", + Field: "payload.elasticsearch.node_stats.breakers.parent.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: parentBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + accountingBreakerMetric := newMetricItem("accounting_breaker", 2, CircuitBreakerGroupKey) + accountingBreakerMetric.AddAxi("Accounting Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "accounting_breaker", + Field: "payload.elasticsearch.node_stats.breakers.accounting.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: accountingBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + fielddataBreakerMetric := newMetricItem("fielddata_breaker", 3, CircuitBreakerGroupKey) + fielddataBreakerMetric.AddAxi("Fielddata Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "fielddata_breaker", + Field: "payload.elasticsearch.node_stats.breakers.fielddata.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: fielddataBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + requestBreakerMetric := newMetricItem("request_breaker", 4, CircuitBreakerGroupKey) + requestBreakerMetric.AddAxi("Request Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "request_breaker", + Field: "payload.elasticsearch.node_stats.breakers.request.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: requestBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + inFlightRequestBreakerMetric := newMetricItem("in_flight_requests_breaker", 5, CircuitBreakerGroupKey) + inFlightRequestBreakerMetric.AddAxi("In Flight Requests Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "in_flight_requests_breaker", + Field: "payload.elasticsearch.node_stats.breakers.in_flight_requests.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: inFlightRequestBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + modelInferenceBreakerMetric := newMetricItem("model_inference_breaker", 6, CircuitBreakerGroupKey) + modelInferenceBreakerMetric.AddAxi("Model Inference Breaker","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + nodeMetricItems = append(nodeMetricItems, GroupMetricItem{ + Key: "model_inference_breaker", + Field: "payload.elasticsearch.node_stats.breakers.model_inference.tripped", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: modelInferenceBreakerMetric, + FormatType: "num", + Units: "times/s", + }) + + aggs := generateGroupAggs(nodeMetricItems) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"] = 0 + query["aggs"] = util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": top, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": aggs, + }, + }, + }, + } + return h.getMetrics(query, nodeMetricItems, bucketSize), nil + +} + +func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int) ([]string, error){ + ver := h.Client().GetVersion() + cr, _ := util.VersionCompare(ver.Number, "6.1") + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { + return nil, nil + } + var ( + now = time.Now() + max = now.UnixNano()/1e6 + min = now.Add(-time.Duration(lastMinutes) * time.Minute).UnixNano()/1e6 + bucketSizeStr = "60s" + ) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + + query := util.MapStr{ + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + }, + "aggs": util.MapStr{ + "group_by_index": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>search_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}}, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "search_query_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + }, + }, + "search_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "search_query_total", + }, + }, + }, + }, + }, + }, + "group_by_index1": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": 10000, + }, + "aggs": util.MapStr{ + "max_qps": util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": "dates>index_qps", + }, + }, + "max_qps_bucket_sort": util.MapStr{ + "bucket_sort": util.MapStr{ + "sort": []util.MapStr{ + {"max_qps": util.MapStr{"order": "desc"}}, + }, + "size": top, + }, + }, + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "index_total": util.MapStr{ + "max": util.MapStr{ + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + }, + }, + "index_qps": util.MapStr{ + "derivative": util.MapStr{ + "buckets_path": "index_total", + }, + }, + }, + }, + }, + }, + }, + } + response,err:=elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(),util.MustToJSONBytes(query)) + if err!=nil{ + log.Error(err) + return nil, err + } + var maxQpsKVS = map[string] float64{} + for _, agg := range response.Aggregations { + for _, bk := range agg.Buckets { + key := bk["key"].(string) + if maxQps, ok := bk["max_qps"].(map[string]interface{}); ok { + val := maxQps["value"].(float64) + if _, ok = maxQpsKVS[key] ; ok { + maxQpsKVS[key] = maxQpsKVS[key] + val + }else{ + maxQpsKVS[key] = val + } + } + } + } + var ( + qpsValues TopTermOrder + ) + for k, v := range maxQpsKVS { + qpsValues = append(qpsValues, TopTerm{ + Key: k, + Value: v, + }) + } + sort.Sort(qpsValues) + var length = top + if top > len(qpsValues) { + length = len(qpsValues) + } + nodeNames := []string{} + for i := 0; i 1 { + query["sort"] = []util.MapStr{ + { + reqBody.Sort[0]: util.MapStr{ + "order": reqBody.Sort[1], + }, + }, + } + } + dsl := util.MustToJSONBytes(query) + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(orm.GetIndexName(elastic.NodeConfig{}), dsl) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + return + } + w.Write(util.MustToJSONBytes(response)) +} + +func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var nodeIDs = []string{} + h.DecodeJSON(req, &nodeIDs) + + if len(nodeIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + q1 := orm.Query{WildcardIndex: true} + query := util.MapStr{ + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "collapse": util.MapStr{ + "field": "metadata.labels.node_id", + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + }, + }, + } + q1.RawQuery = util.MustToJSONBytes(query) + + err, results := orm.Search(&event.Event{}, &q1) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + statusMap := map[string]interface{}{} + for _, v := range results.Result { + result, ok := v.(map[string]interface{}) + if ok { + nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) + if ok { + source := map[string]interface{}{} + //timestamp, ok := result["timestamp"].(string) + uptime, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, result) + if ok { + source["uptime"] = uptime + } + + fsTotal, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total"}, result) + if ok { + source["fs"] = util.MapStr{ + "total": fsTotal, + } + } + + jvmMem, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem"}, result) + if ok { + source["jvm"] = util.MapStr{ + "mem": jvmMem, + } + } + indices := util.MapStr{} + docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "docs"}, result) + if ok { + indices["docs"] = docs + } + source["indices"] = indices + shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, result) + if ok { + source["shard_info"] = shardInfo + } + if tempClusterID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "cluster_id"}, result); ok { + if clusterID, ok := tempClusterID.(string); ok { + if meta := elastic.GetMetadata(clusterID); meta != nil && meta.ClusterState != nil { + source["is_master_node"] = meta.ClusterState.MasterNode == nodeID + } + } + } + + statusMap[util.ToString(nodeID)] = source + } + } + } + statusMetric, err := getNodeOnlineStatusOfRecentDay(nodeIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req, 60, (15)) + if err != nil { + panic(err) + return + } + // 索引速率 + indexMetric:=newMetricItem("indexing", 1, OperationGroupKey) + indexMetric.AddAxi("indexing rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems := []GroupMetricItem{} + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "indexing", + Field: "payload.elasticsearch.node_stats.indices.indexing.index_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexMetric, + FormatType: "num", + Units: "Indexing/s", + }) + queryMetric:=newMetricItem("search", 2, OperationGroupKey) + queryMetric.AddAxi("query rate","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + nodeMetricItems=append(nodeMetricItems, GroupMetricItem{ + Key: "search", + Field: "payload.elasticsearch.node_stats.indices.search.query_total", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: queryMetric, + FormatType: "num", + Units: "Search/s", + }) + + aggs:=map[string]interface{}{} + query=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + for _,metricItem:=range nodeMetricItems{ + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + } + } + + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + panic(err) + } + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 100, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + }, + }, + } + metrics := h.getMetrics(query, nodeMetricItems, bucketSize) + indexMetrics := map[string]util.MapStr{} + for key, item := range metrics { + for _, line := range item.Lines { + if _, ok := indexMetrics[line.Metric.Label]; !ok{ + indexMetrics[line.Metric.Label] = util.MapStr{ + } + } + indexMetrics[line.Metric.Label][key] = line.Data + } + } + result := util.MapStr{} + for _, nodeID := range nodeIDs { + source := util.MapStr{} + + source["summary"] = statusMap[nodeID] + source["metrics"] = util.MapStr{ + "status": util.MapStr{ + "metric": util.MapStr{ + "label": "Recent Node Status", + "units": "day", + }, + "data": statusMetric[nodeID], + }, + "indexing": util.MapStr{ + "metric": util.MapStr{ + "label": "Indexing", + "units": "s", + }, + "data": indexMetrics[nodeID]["indexing"], + }, + "search": util.MapStr{ + "metric": util.MapStr{ + "label": "Search", + "units": "s", + }, + "data": indexMetrics[nodeID]["search"], + }, + } + result[nodeID] = source + } + h.WriteJSON(w, result, http.StatusOK) +} + +func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + nodeID := ps.MustGetParameter("node_id") + + q := orm.Query{ + Size: 1, + } + q.Conds = orm.And(orm.Eq("metadata.node_id", nodeID), orm.Eq("metadata.cluster_id", clusterID)) + + err, res := orm.Search(&elastic.NodeConfig{}, &q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + //if len(response.Hits.Hits) == 0 { + // h.WriteError(w, "", http.StatusNotFound) + // return + //} + q1 := orm.Query{ + Size: 1, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "node_stats"), + orm.Eq("metadata.labels.node_id", nodeID), + ) + q1.Collapse("metadata.labels.node_id") + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + kvs := util.MapStr{} + if len(result.Result) > 0 { + vresult, ok := result.Result[0].(map[string]interface{}) + if ok { + transportAddress, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "transport_address"}, vresult) + if ok { + kvs["transport_address"] = transportAddress + } + kvs["timestamp"] = vresult["timestamp"] + if vresult["timestamp"] != nil { + if ts, ok := vresult["timestamp"].(string); ok { + tt, _ := time.Parse(time.RFC3339, ts) + if time.Now().Sub(tt).Seconds() > 30 { + kvs["status"] = "unavailable" + }else{ + kvs["status"] = "available" + } + } + } + roles, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "roles"}, vresult) + if ok { + kvs["roles"] = roles + } + fsTotal, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total"}, vresult) + if ok { + kvs["fs"] = util.MapStr{ + "total": fsTotal, + } + } + + jvm, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm"}, vresult) + if ok { + if jvmVal, ok := jvm.(map[string]interface{});ok { + kvs["jvm"] = util.MapStr{ + "mem": jvmVal["mem"], + "uptime": jvmVal["uptime_in_millis"], + } + } + } + indices := util.MapStr{} + docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "docs"}, vresult) + if ok { + indices["docs"] = docs + } + store, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "indices", "store"}, vresult) + if ok { + indices["store"] = store + } + kvs["indices"] = indices + shardInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, vresult) + if ok { + kvs["shard_info"] = shardInfo + } + } + } + if len( response.Hits.Hits) > 0 { + hit := response.Hits.Hits[0] + innerMetaData, _ := util.GetMapValueByKeys([]string{"metadata", "labels"}, hit.Source) + if mp, ok := innerMetaData.(map[string]interface{}); ok { + kvs["transport_address"] = mp["transport_address"] + kvs["roles"] = mp["roles"] + if kvs["status"] != "available" { + kvs["status"] = mp["status"] + kvs["timestamp"] = hit.Source["timestamp"] + } + } + } + + if meta := elastic.GetMetadata(clusterID); meta != nil && meta.ClusterState != nil { + kvs["is_master_node"] = meta.ClusterState.MasterNode == nodeID + } + h.WriteJSON(w, kvs, http.StatusOK) +} + +func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + clusterUUID, err := adapter.GetClusterUUID(clusterID) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + nodeID := ps.MustGetParameter("node_id") + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_uuid":util.MapStr{ + "value": clusterUUID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeID, + }, + }, + }, + } + resBody := map[string]interface{}{} + bucketSize, min, max, err := h.getMetricRangeAndBucketSize(req,10,60) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + meta := elastic.GetMetadata(clusterID) + if meta != nil && meta.Config.MonitorConfigs != nil && meta.Config.MonitorConfigs.NodeStats.Interval != "" { + du, _ := time.ParseDuration(meta.Config.MonitorConfigs.NodeStats.Interval) + if bucketSize < int(du.Seconds()) { + bucketSize = int(du.Seconds()) + } + } + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + metricItems:=[]*common.MetricItem{} + metricItem:=newMetricItem("cpu", 1, SystemGroupKey) + metricItem.AddAxi("cpu","group1",common.PositionLeft,"ratio","0.[0]","0.[0]",5,true) + metricItem.AddLine("Process CPU","Process CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.process.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) + metricItem.AddLine("OS CPU","OS CPU","process cpu used percent of node.","group1","payload.elasticsearch.node_stats.os.cpu.percent","max",bucketSizeStr,"%","num","0,0.[00]","0,0.[00]",false,false) + metricItems=append(metricItems,metricItem) + metricItem =newMetricItem("jvm", 2, SystemGroupKey) + metricItem.AddAxi("JVM Heap","group1",common.PositionLeft,"bytes","0.[0]","0.[0]",5,true) + metricItem.AddLine("Max Heap","Max Heap","JVM max Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_max_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) + metricItem.AddLine("Used Heap","Used Heap","JVM used Heap of node.","group1","payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes","max",bucketSizeStr,"","bytes","0,0.[00]","0,0.[00]",false,false) + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("index_throughput", 3, OperationGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + metricItem.AddLine("Indexing Rate","Total Shards","Number of documents being indexed for node.","group1","payload.elasticsearch.node_stats.indices.indexing.index_total","max",bucketSizeStr,"doc/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + metricItem=newMetricItem("search_throughput", 4, OperationGroupKey) + metricItem.AddAxi("searching","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + metricItem.AddLine("Search Rate","Total Shards", + "Number of search requests being executed.", + "group1","payload.elasticsearch.node_stats.indices.search.query_total","max",bucketSizeStr,"query/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("index_latency", 5, LatencyGroupKey) + metricItem.AddAxi("indexing","group1",common.PositionLeft,"num","0,0","0,0.[00]",5,true) + + metricItem.AddLine("Indexing","Indexing Latency","Average latency for indexing documents.","group1","payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.index_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Indexing","Delete Latency","Average latency for delete documents.","group1","payload.elasticsearch.node_stats.indices.indexing.delete_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.indexing.delete_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + + metricItem=newMetricItem("search_latency", 6, LatencyGroupKey) + metricItem.AddAxi("searching","group2",common.PositionLeft,"num","0,0","0,0.[00]",5,false) + + metricItem.AddLine("Searching","Query Latency","Average latency for searching query.","group2","payload.elasticsearch.node_stats.indices.search.query_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[0].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.query_total" + metricItem.Lines[0].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Fetch Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.fetch_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[1].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.fetch_total" + metricItem.Lines[1].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItem.AddLine("Searching","Scroll Latency","Average latency for searching fetch.","group2","payload.elasticsearch.node_stats.indices.search.scroll_time_in_millis","max",bucketSizeStr,"ms","num","0,0.[00]","0,0.[00]",false,true) + metricItem.Lines[2].Metric.Field2 = "payload.elasticsearch.node_stats.indices.search.scroll_total" + metricItem.Lines[2].Metric.Calc = func(value, value2 float64) float64 { + return value/value2 + } + metricItems=append(metricItems,metricItem) + metricItem =newMetricItem("parent_breaker", 8, SystemGroupKey) + metricItem.AddLine("Parent Breaker Tripped","Parent Breaker Tripped","Rate of the circuit breaker has been triggered and prevented an out of memory error.","group1","payload.elasticsearch.node_stats.breakers.parent.tripped","max",bucketSizeStr,"times/s","num","0,0.[00]","0,0.[00]",false,true) + metricItems=append(metricItems,metricItem) + metrics := h.getSingleMetrics(metricItems,query, bucketSize) + healthMetric, err := getNodeHealthMetric(query, bucketSize) + if err != nil { + log.Error(err) + } + metrics["node_health"] = healthMetric + resBody["metrics"] = metrics + h.WriteJSON(w, resBody, http.StatusOK) +} + +func getNodeHealthMetric(query util.MapStr, bucketSize int)(*common.MetricItem, error){ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } + query["aggs"] = util.MapStr{ + "dates": util.MapStr{ + "date_histogram": util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs": util.MapStr{ + "min_uptime": util.MapStr{ + "min": util.MapStr{ + "field": "payload.elasticsearch.node_stats.jvm.uptime_in_millis", + }, + }, + }, + }, + } + response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) + if err != nil { + log.Error(err) + return nil, err + } + + metricItem:=newMetricItem("node_health", 0, "") + metricItem.AddLine("Node health","Node Health","","group1","payload.elasticsearch.node_stats.jvm.uptime_in_millis","min",bucketSizeStr,"%","ratio","0.[00]","0.[00]",false,false) + + metricData := []interface{}{} + if response.StatusCode == 200 { + for _, bucket := range response.Aggregations["dates"].Buckets { + v, ok := bucket["key"].(float64) + if !ok { + log.Error("invalid bucket key") + return nil, fmt.Errorf("invalid bucket key") + } + dateTime := int64(v) + statusKey := "available" + if uptimeAgg, ok := bucket["min_uptime"].(map[string]interface{}); ok { + if _, ok = uptimeAgg["value"].(float64); !ok { + statusKey = "unavailable" + } + metricData = append(metricData, map[string]interface{}{ + "x": dateTime, + "y": 100, + "g": statusKey, + }) + } + } + } + metricItem.Lines[0].Data = metricData + metricItem.Lines[0].Type = common.GraphTypeBar + return metricItem, nil +} + +func getNodeOnlineStatusOfRecentDay(nodeIDs []string)(map[string][]interface{}, error){ + q := orm.Query{ + WildcardIndex: true, + } + query := util.MapStr{ + "aggs": util.MapStr{ + "group_by_node_id": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.node_id", + "size": 100, + }, + "aggs": util.MapStr{ + "uptime_histogram": util.MapStr{ + "date_range": util.MapStr{ + "field": "timestamp", + "format": "yyyy-MM-dd", + "time_zone": "+08:00", + "ranges": []util.MapStr{ + { + "from": "now-13d/d", + "to": "now-12d/d", + }, { + "from": "now-12d/d", + "to": "now-11d/d", + }, + { + "from": "now-11d/d", + "to": "now-10d/d", + }, + { + "from": "now-10d/d", + "to": "now-9d/d", + }, { + "from": "now-9d/d", + "to": "now-8d/d", + }, + { + "from": "now-8d/d", + "to": "now-7d/d", + }, + { + "from": "now-7d/d", + "to": "now-6d/d", + }, + { + "from": "now-6d/d", + "to": "now-5d/d", + }, { + "from": "now-5d/d", + "to": "now-4d/d", + }, + { + "from": "now-4d/d", + "to": "now-3d/d", + },{ + "from": "now-3d/d", + "to": "now-2d/d", + }, { + "from": "now-2d/d", + "to": "now-1d/d", + }, { + "from": "now-1d/d", + "to": "now/d", + }, + { + "from": "now/d", + "to": "now", + }, + }, + }, + "aggs": util.MapStr{ + "min_uptime": util.MapStr{ + "min": util.MapStr{ + "field": "payload.elasticsearch.node_stats.jvm.uptime_in_millis", + }, + }, + }, + }, + }, + }, + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "size": 0, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte":"now-15d", + "lte": "now", + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + { + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, + }, + }, + }, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(query) + + err, res := orm.Search(&event.Event{}, &q) + if err != nil { + return nil, err + } + + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + recentStatus := map[string][]interface{}{} + for _, bk := range response.Aggregations["group_by_node_id"].Buckets { + nodeKey := bk["key"].(string) + recentStatus[nodeKey] = []interface{}{} + if histogramAgg, ok := bk["uptime_histogram"].(map[string]interface{}); ok { + if bks, ok := histogramAgg["buckets"].([]interface{}); ok { + for _, bkItem := range bks { + if bkVal, ok := bkItem.(map[string]interface{}); ok { + if minUptime, ok := util.GetMapValueByKeys([]string{"min_uptime", "value"}, bkVal); ok { + //mark node status as offline when uptime less than 10m + if v, ok := minUptime.(float64); ok && v >= 600000 { + recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "online"}) + }else{ + recentStatus[nodeKey] = append(recentStatus[nodeKey], []interface{}{bkVal["key"], "offline"}) + } + } + } + } + } + } + } + return recentStatus, nil +} + +func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + min = h.GetParameterOrDefault(req, "min", "now-15m") + max = h.GetParameterOrDefault(req, "max", "now") + ) + + resBody := map[string] interface{}{} + id := ps.ByName("id") + nodeUUID := ps.ByName("node_id") + q := &orm.Query{ Size: 1} + q.AddSort("timestamp", orm.DESC) + q.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.labels.cluster_id", id), + orm.Eq("metadata.labels.node_id", nodeUUID), + orm.Eq("metadata.name", "node_routing_table"), + ) + + err, result := orm.Search(event.Event{}, q) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + namesM := util.MapStr{} + if len(result.Result) > 0 { + if data, ok := result.Result[0].(map[string]interface{}); ok { + if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { + if rows, ok := routingTable.([]interface{}); ok{ + for _, row := range rows { + if v, ok := row.(map[string]interface{}); ok { + if indexName, ok := v["index"].(string); ok{ + namesM[indexName] = true + } + } + } + } + } + } + } + + indexNames := make([]interface{}, 0, len(namesM) ) + for name, _ := range namesM { + indexNames = append(indexNames, name) + } + + q1 := &orm.Query{ Size: 100} + q1.AddSort("timestamp", orm.DESC) + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.cluster_id", id), + orm.In("metadata.index_name", indexNames), + orm.NotEq("metadata.labels.index_status", "deleted"), + ) + err, result = orm.Search(elastic.IndexConfig{}, q1) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + + indices, err := h.getLatestIndices(req, min, max, id, &result) + if err != nil { + resBody["error"] = err.Error() + h.WriteJSON(w,resBody, http.StatusInternalServerError ) + } + + h.WriteJSON(w, indices, http.StatusOK) +} + +func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) { + //filter indices + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return []interface{}{}, nil + } + + query := util.MapStr{ + "size": 2000, + "_source": []string{"metadata", "payload.elasticsearch.index_stats.index_info", "timestamp"}, + "collapse": util.MapStr{ + "field": "metadata.labels.index_name", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + }, + }, + }, + } + q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + err, searchResult := orm.Search(event.Event{}, q) + if err != nil { + return nil, err + } + indexInfos := map[string]util.MapStr{} + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + indexInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + if v, ok := indexName.(string); ok { + if infoM, ok := indexInfo.(map[string]interface{}); ok { + if _, ok = infoM["index"].(string); ok { + infoM["timestamp"] = hitM["timestamp"] + indexInfos[v] = infoM + } + } + } + } + } + indices := []interface{}{} + var indexPattern *radix.Pattern + if !hasAllPrivilege{ + indexPattern = radix.Compile(allowedIndices...) + } + + for _, hit := range result.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "index_name"}, hitM) + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, hitM) + if v, ok := indexName.(string); ok { + if indexPattern != nil { + if !indexPattern.Match(v) { + continue + } + } + if indexInfos[v] != nil { + if state == "delete" { + indexInfos[v]["status"] = "delete" + indexInfos[v]["health"] = "N/A" + } + indices = append(indices, indexInfos[v]) + } else { + indices = append(indices, util.MapStr{ + "index": v, + "status": state, + "timestamp": hitM["timestamp"], + }) + } + } + } + } + return indices, nil +} + + +func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + clusterID := ps.MustGetParameter("id") + nodeID := ps.MustGetParameter("node_id") + q1 := orm.Query{ + Size: 1, + WildcardIndex: true, + } + q1.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "node_stats"), + orm.Eq("metadata.labels.node_id", nodeID), + orm.Eq("metadata.labels.cluster_id", clusterID), + ) + q1.AddSort("timestamp", orm.DESC) + err, result := orm.Search(&event.Event{}, &q1) + if err != nil { + h.WriteJSON(w,util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError ) + return + } + var shardInfo interface{} = []interface{}{} + if len(result.Result) > 0 { + row, ok := result.Result[0].(map[string]interface{}) + if ok { + shardInfo, ok = util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info", "shards"}, row) + qps, err := h.getIndexQPS(clusterID, 20) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if shards, ok := shardInfo.([]interface{}); ok { + for _, shard := range shards { + if shardM, ok := shard.(map[string]interface{}); ok { + if indexName, ok := shardM["index"].(string); ok { + if _, ok := qps[indexName]; ok { + shardM["index_qps"] = qps[indexName]["index"] + shardM["query_qps"] = qps[indexName]["query"] + shardM["index_bytes_qps"] = qps[indexName]["index_bytes"] + } + } + } + } + } + } + } + if shardInfo == nil { + shardInfo = []interface{}{} + } + + h.WriteJSON(w, shardInfo, http.StatusOK) +} \ No newline at end of file diff --git a/modules/elastic/api/v1/threadpool_metrics.go b/modules/elastic/api/v1/threadpool_metrics.go new file mode 100644 index 00000000..204ece84 --- /dev/null +++ b/modules/elastic/api/v1/threadpool_metrics.go @@ -0,0 +1,539 @@ +package v1 + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" + "strings" +) + +const ( + ThreadPoolGetGroupKey = "thread_pool_get" + ThreadPoolSearchGroupKey = "thread_pool_search" + ThreadPoolFlushGroupKey = "thread_pool_flush" + ThreadPoolRefreshGroupKey = "thread_pool_refresh" + ThreadPoolWriteGroupKey = "thread_pool_write" + ThreadPoolForceMergeGroupKey = "thread_pool_force_merge" + ThreadPoolIndexGroupKey = "thread_pool_index" + ThreadPoolBulkGroupKey = "thread_pool_bulk" +) + +func (h *APIHandler) getThreadPoolMetrics(clusterID string, bucketSize int, min, max int64, nodeName string, top int) map[string]*common.MetricItem{ + bucketSizeStr:=fmt.Sprintf("%vs",bucketSize) + var must = []util.MapStr{ + { + "term":util.MapStr{ + "metadata.labels.cluster_id":util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "node_stats", + }, + }, + }, + } + var ( + nodeNames []string + err error + ) + if nodeName != "" { + nodeNames = strings.Split(nodeName, ",") + top = len(nodeNames) + }else{ + nodeNames, err = h.getTopNodeName(clusterID, top, 15) + if err != nil { + log.Error(err) + } + } + if len(nodeNames) > 0 { + must = append(must, util.MapStr{ + "terms": util.MapStr{ + "metadata.labels.transport_address": nodeNames, + }, + }) + } + + query:=map[string]interface{}{} + query["query"]=util.MapStr{ + "bool": util.MapStr{ + "must": must, + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + }, + } + searchThreadsMetric := newMetricItem("search_threads", 1, ThreadPoolSearchGroupKey) + searchThreadsMetric.AddAxi("Search Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems := []GroupMetricItem{ + { + Key: "search_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.search.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchThreadsMetric, + FormatType: "num", + Units: "", + }, + } + searchQueueMetric := newMetricItem("search_queue", 1, ThreadPoolSearchGroupKey) + searchQueueMetric.AddAxi("Search Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.search.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchQueueMetric, + FormatType: "num", + Units: "", + }) + searchActiveMetric := newMetricItem("search_active", 1, ThreadPoolSearchGroupKey) + searchActiveMetric.AddAxi("Search Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_active", + Field: "payload.elasticsearch.node_stats.thread_pool.search.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: searchActiveMetric, + FormatType: "num", + Units: "", + }) + searchRejectedMetric := newMetricItem("search_rejected", 1, ThreadPoolSearchGroupKey) + searchRejectedMetric.AddAxi("Search Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "search_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.search.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: searchRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + getThreadsMetric := newMetricItem("get_threads", 1, ThreadPoolGetGroupKey) + getThreadsMetric.AddAxi("Get Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.get.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getThreadsMetric, + FormatType: "num", + Units: "", + }) + getQueueMetric := newMetricItem("get_queue", 1, ThreadPoolGetGroupKey) + getQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.get.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getQueueMetric, + FormatType: "num", + Units: "", + }) + getActiveMetric := newMetricItem("get_active", 1, ThreadPoolGetGroupKey) + getActiveMetric.AddAxi("Get Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_active", + Field: "payload.elasticsearch.node_stats.thread_pool.get.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: getActiveMetric, + FormatType: "num", + Units: "", + }) + getRejectedMetric := newMetricItem("get_rejected", 1, ThreadPoolGetGroupKey) + getRejectedMetric.AddAxi("Get Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "get_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.get.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: getRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + flushThreadsMetric := newMetricItem("flush_threads", 1, ThreadPoolFlushGroupKey) + flushThreadsMetric.AddAxi("Flush Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushThreadsMetric, + FormatType: "num", + Units: "", + }) + flushQueueMetric := newMetricItem("flush_queue", 1, ThreadPoolFlushGroupKey) + flushQueueMetric.AddAxi("Get Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushQueueMetric, + FormatType: "num", + Units: "", + }) + flushActiveMetric := newMetricItem("flush_active", 1, ThreadPoolFlushGroupKey) + flushActiveMetric.AddAxi("Flush Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_active", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: flushActiveMetric, + FormatType: "num", + Units: "", + }) + flushRejectedMetric := newMetricItem("flush_rejected", 1, ThreadPoolFlushGroupKey) + flushRejectedMetric.AddAxi("Flush Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "flush_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.flush.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: flushRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + majorVersion := elastic.GetMetadata(clusterID).GetMajorVersion() + ver := elastic.GetClient(clusterID).GetVersion() + + if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && majorVersion < 6{ + indexThreadsMetric := newMetricItem("index_threads", 1, ThreadPoolIndexGroupKey) + indexThreadsMetric.AddAxi("Index Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.index.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexThreadsMetric, + FormatType: "num", + Units: "", + }) + indexQueueMetric := newMetricItem("index_queue", 1, ThreadPoolIndexGroupKey) + indexQueueMetric.AddAxi("Index Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.index.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexQueueMetric, + FormatType: "num", + Units: "", + }) + indexActiveMetric := newMetricItem("index_active", 1, ThreadPoolIndexGroupKey) + indexActiveMetric.AddAxi("Index Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_active", + Field: "payload.elasticsearch.node_stats.thread_pool.index.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: indexActiveMetric, + FormatType: "num", + Units: "", + }) + indexRejectedMetric := newMetricItem("index_rejected", 1, ThreadPoolIndexGroupKey) + indexRejectedMetric.AddAxi("Index Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "index_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.index.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: indexRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + + bulkThreadsMetric := newMetricItem("bulk_threads", 1, ThreadPoolBulkGroupKey) + bulkThreadsMetric.AddAxi("Bulk Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkThreadsMetric, + FormatType: "num", + Units: "", + }) + bulkQueueMetric := newMetricItem("bulk_queue", 1, ThreadPoolBulkGroupKey) + bulkQueueMetric.AddAxi("Bulk Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkQueueMetric, + FormatType: "num", + Units: "", + }) + bulkActiveMetric := newMetricItem("bulk_active", 1, ThreadPoolBulkGroupKey) + bulkActiveMetric.AddAxi("Bulk Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_active", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: bulkActiveMetric, + FormatType: "num", + Units: "", + }) + bulkRejectedMetric := newMetricItem("bulk_rejected", 1, ThreadPoolBulkGroupKey) + bulkRejectedMetric.AddAxi("Bulk Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "bulk_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.bulk.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: bulkRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + }else { + writeThreadsMetric := newMetricItem("write_threads", 1, ThreadPoolWriteGroupKey) + writeThreadsMetric.AddAxi("Write Threads Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.write.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeThreadsMetric, + FormatType: "num", + Units: "", + }) + writeQueueMetric := newMetricItem("write_queue", 1, ThreadPoolWriteGroupKey) + writeQueueMetric.AddAxi("Write Queue Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.write.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeQueueMetric, + FormatType: "num", + Units: "", + }) + writeActiveMetric := newMetricItem("write_active", 1, ThreadPoolWriteGroupKey) + writeActiveMetric.AddAxi("Write Active Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_active", + Field: "payload.elasticsearch.node_stats.thread_pool.write.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: writeActiveMetric, + FormatType: "num", + Units: "", + }) + writeRejectedMetric := newMetricItem("write_rejected", 1, ThreadPoolWriteGroupKey) + writeRejectedMetric.AddAxi("Write Rejected Count", "group1", common.PositionLeft, "num", "0.[0]", "0.[0]", 5, true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "write_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.write.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: writeRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + } + refreshThreadsMetric := newMetricItem("refresh_threads", 1, ThreadPoolRefreshGroupKey) + refreshThreadsMetric.AddAxi("Refresh Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshThreadsMetric, + FormatType: "num", + Units: "", + }) + refreshQueueMetric := newMetricItem("refresh_queue", 1, ThreadPoolRefreshGroupKey) + refreshQueueMetric.AddAxi("Refresh Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshQueueMetric, + FormatType: "num", + Units: "", + }) + refreshActiveMetric := newMetricItem("refresh_active", 1, ThreadPoolRefreshGroupKey) + refreshActiveMetric.AddAxi("Refresh Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_active", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: refreshActiveMetric, + FormatType: "num", + Units: "", + }) + refreshRejectedMetric := newMetricItem("refresh_rejected", 1, ThreadPoolRefreshGroupKey) + refreshRejectedMetric.AddAxi("Refresh Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "refresh_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.refresh.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: refreshRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + forceMergeThreadsMetric := newMetricItem("force_merge_threads", 1, ThreadPoolForceMergeGroupKey) + forceMergeThreadsMetric.AddAxi("Force Merge Threads Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_threads", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.threads", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeThreadsMetric, + FormatType: "num", + Units: "", + }) + forceMergeQueueMetric := newMetricItem("force_merge_queue", 1, ThreadPoolForceMergeGroupKey) + forceMergeQueueMetric.AddAxi("Force Merge Queue Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_queue", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.queue", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeQueueMetric, + FormatType: "num", + Units: "", + }) + forceMergeActiveMetric := newMetricItem("force_merge_active", 1, ThreadPoolForceMergeGroupKey) + forceMergeActiveMetric.AddAxi("Force Merge Active Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_active", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.active", + ID: util.GetUUID(), + IsDerivative: false, + MetricItem: forceMergeActiveMetric, + FormatType: "num", + Units: "", + }) + forceMergeRejectedMetric := newMetricItem("force_merge_rejected", 1, ThreadPoolForceMergeGroupKey) + forceMergeRejectedMetric.AddAxi("Force Merge Rejected Count","group1",common.PositionLeft,"num","0.[0]","0.[0]",5,true) + + queueMetricItems = append(queueMetricItems, GroupMetricItem{ + Key: "force_merge_rejected", + Field: "payload.elasticsearch.node_stats.thread_pool.force_merge.rejected", + ID: util.GetUUID(), + IsDerivative: true, + MetricItem: forceMergeRejectedMetric, + FormatType: "num", + Units: "rejected/s", + }) + //Get Thread Pool queue + aggs:=map[string]interface{}{} + + for _,metricItem:=range queueMetricItems{ + aggs[metricItem.ID]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID + "_field2"]=util.MapStr{ + "max":util.MapStr{ + "field": metricItem.Field2, + }, + } + } + + if metricItem.IsDerivative{ + aggs[metricItem.ID+"_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID, + }, + } + if metricItem.Field2 != "" { + aggs[metricItem.ID + "_field2_deriv"]=util.MapStr{ + "derivative":util.MapStr{ + "buckets_path": metricItem.ID + "_field2", + }, + } + } + } + } + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + log.Error(err) + panic(err) + } + + query["size"]=0 + query["aggs"]= util.MapStr{ + "group_by_level": util.MapStr{ + "terms": util.MapStr{ + "field": "metadata.labels.transport_address", + "size": top, + }, + "aggs": util.MapStr{ + "dates": util.MapStr{ + "date_histogram":util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + }, + "aggs":aggs, + }, + }, + }, + } + return h.getMetrics(query, queueMetricItems, bucketSize) +} diff --git a/modules/elastic/api/view.go b/modules/elastic/api/view.go new file mode 100644 index 00000000..3348feb1 --- /dev/null +++ b/modules/elastic/api/view.go @@ -0,0 +1,500 @@ +package api + +import ( + "fmt" + log "github.com/cihub/seelog" + "github.com/segmentio/encoding/json" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/radix" + "infini.sh/framework/core/util" + "net/http" + "strconv" + "strings" + "time" +) + +func (h *APIHandler) HandleCreateViewAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + + targetClusterID := ps.ByName("id") + exists, _, err := h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + var viewReq = &elastic.ViewRequest{} + + err = h.DecodeJSON(req, viewReq) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + id := util.GetUUID() + viewReq.Attributes.UpdatedAt = time.Now() + viewReq.Attributes.ClusterID = targetClusterID + _, err = esClient.Index(orm.GetIndexName(viewReq.Attributes), "", id, viewReq.Attributes, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + resBody = map[string]interface{}{ + "id": id, + "type": "index-pattern", + "updated_at": viewReq.Attributes.UpdatedAt, + "attributes": viewReq.Attributes, + "namespaces": []string{"default"}, + } + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) HandleGetViewListAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + + targetClusterID := ps.ByName("id") + strSize := h.GetParameterOrDefault(req, "per_page", "10000") + size, _ := strconv.Atoi(strSize) + search := h.GetParameterOrDefault(req, "search", "") + if search != "" { + search = fmt.Sprintf(`,{"match":{"title":%s}}`, search) + } + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + + queryDSL := []byte(fmt.Sprintf(`{"_source":["title","viewName", "updated_at"],"size": %d, "query":{"bool":{"must":[{"match":{"cluster_id":"%s"}}%s]}}}`, size, targetClusterID, search)) + + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.View{}), queryDSL) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var total = len(searchRes.Hits.Hits) + if totalVal, ok := searchRes.Hits.Total.(map[string]interface{}); ok { + total = int(totalVal["value"].(float64)) + } + resBody = map[string]interface{}{ + "per_page": size, + "total": total, + } + var savedObjects = make([]map[string]interface{}, 0, len(searchRes.Hits.Hits)) + for _, hit := range searchRes.Hits.Hits { + var savedObject = map[string]interface{}{ + "id": hit.ID, + "attributes": map[string]interface{}{ + "title": hit.Source["title"], + "viewName": hit.Source["viewName"], + }, + "score": 0, + "type": "index-pattern", + "namespaces": []string{"default"}, + "updated_at": hit.Source["updated_at"], + } + savedObjects = append(savedObjects, savedObject) + } + resBody["saved_objects"] = savedObjects + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) HandleDeleteViewAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + + viewID := ps.ByName("view_id") + + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + + _, err := esClient.Delete(orm.GetIndexName(elastic.View{}), "", viewID, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) HandleResolveIndexAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + + targetClusterID := ps.ByName("id") + wild := ps.ByName("wild") + //wild = strings.ReplaceAll(wild, "*", "") + + exists, client, err := h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, targetClusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + h.WriteJSON(w, elastic.AliasAndIndicesResponse{ + Aliases: []elastic.AAIR_Alias{}, + Indices: []elastic.AAIR_Indices{}, + }, http.StatusOK) + return + } + //ccs + if strings.Contains(wild, ":") { + q := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "indices": util.MapStr{ + "terms": util.MapStr{ + "field": "_index", + "size": 200, + }, + }, + }, + } + searchRes, err := client.SearchWithRawQueryDSL(wild, util.MustToJSONBytes(q)) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + indices := []elastic.AAIR_Indices{} + parts := strings.SplitN(wild, ":", 2) + if parts[1] == "" { + wild = "*" + } + var filterPattern *radix.Pattern + if !hasAllPrivilege { + filterPattern = radix.Compile(allowedIndices...) + } + inputPattern := radix.Compile(wild) + if agg, ok := searchRes.Aggregations["indices"]; ok { + for _, bk := range agg.Buckets { + if k, ok := bk["key"].(string); ok { + if !hasAllPrivilege && !filterPattern.Match(k) { + continue + } + if inputPattern.Match(k) { + indices = append(indices, elastic.AAIR_Indices{ + Name: k, + Attributes: []string{"open"}, + }) + } + } + } + } + h.WriteJSON(w, elastic.AliasAndIndicesResponse{ + Aliases: []elastic.AAIR_Alias{}, + Indices: indices, + }, http.StatusOK) + return + } + + res, err := client.GetAliasesAndIndices() + if err != nil || res == nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if wild == "" { + wild = "*" + } + var filterPattern *radix.Pattern + if !hasAllPrivilege { + filterPattern = radix.Compile(allowedIndices...) + } + inputPattern := radix.Compile(wild) + var ( + aliases = []elastic.AAIR_Alias{} + indices = []elastic.AAIR_Indices{} + ) + for _, alias := range res.Aliases { + if !hasAllPrivilege && !filterPattern.Match(alias.Name) { + continue + } + if inputPattern.Match(alias.Name) { + aliases = append(aliases, alias) + } + } + for _, index := range res.Indices { + if !hasAllPrivilege && !filterPattern.Match(index.Name) { + continue + } + if inputPattern.Match(index.Name) { + indices = append(indices, index) + } + } + res.Indices = indices + res.Aliases = aliases + + h.WriteJSON(w, res, http.StatusOK) +} + +func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + targetClusterID := ps.ByName("id") + var reqIDs = []struct { + ID string `json:"id"` + Type string `json:"type"` + }{} + + err := h.DecodeJSON(req, &reqIDs) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var strIDs []string + var indexNames []string + for _, reqID := range reqIDs { + if reqID.Type == "view" { + strIDs = append(strIDs, fmt.Sprintf(`"%s"`, reqID.ID)) + } else if reqID.Type == "index" { + indexNames = append(indexNames, reqID.ID) + } + } + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + esTragertClient := elastic.GetClient(targetClusterID) + queryDSL := []byte(fmt.Sprintf(`{"query": {"bool": {"must": [{"terms": {"_id": [%s]}}, + {"match": {"cluster_id": "%s"}}]}}}`, strings.Join(strIDs, ","), targetClusterID)) + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.View{}), queryDSL) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + var savedObjects = make([]map[string]interface{}, 0, len(searchRes.Hits.Hits)) + for _, hit := range searchRes.Hits.Hits { + var savedObject = map[string]interface{}{ + "id": hit.ID, + "attributes": map[string]interface{}{ + "title": hit.Source["title"], + "fields": hit.Source["fields"], + "viewName": hit.Source["viewName"], + "timeFieldName": hit.Source["timeFieldName"], + "fieldFormatMap": hit.Source["fieldFormatMap"], + }, + "score": 0, + "type": "view", + "namespaces": []string{"default"}, + "migrationVersion": map[string]interface{}{"index-pattern": "7.6.0"}, + "updated_at": hit.Source["updated_at"], + } + savedObjects = append(savedObjects, savedObject) + } + //index mock + for _, indexName := range indexNames { + fields, err := elastic.GetFieldCaps(esTragertClient, indexName, []string{"_source", "_id", "_type", "_index"}) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + bufFields, _ := json.Marshal(fields) + var savedObject = map[string]interface{}{ + "id": indexName, //fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s-%s", targetClusterID,indexName)))), + "attributes": map[string]interface{}{ + "title": indexName, + "fields": string(bufFields), + "viewName": indexName, + "timeFieldName": "", + "fieldFormatMap": "", + }, + "score": 0, + "type": "index", + "namespaces": []string{"default"}, + "migrationVersion": map[string]interface{}{"index-pattern": "7.6.0"}, + "updated_at": time.Now(), + } + savedObjects = append(savedObjects, savedObject) + } + resBody["saved_objects"] = savedObjects + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + + targetClusterID := ps.ByName("id") + exists, _, err := h.GetClusterClient(targetClusterID) + + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + + if !exists { + resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) + log.Error(resBody["error"]) + h.WriteJSON(w, resBody, http.StatusNotFound) + return + } + + var viewReq = &elastic.ViewRequest{} + + err = h.DecodeJSON(req, viewReq) + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if viewReq.Attributes.Title == "" { + resBody["error"] = "miss title" + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + id := ps.ByName("view_id") + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + viewReq.Attributes.UpdatedAt = time.Now() + viewReq.Attributes.ClusterID = targetClusterID + _, err = esClient.Index(orm.GetIndexName(viewReq.Attributes), "", id, viewReq.Attributes, "wait_for") + if err != nil { + log.Error(err) + resBody["error"] = err + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + h.WriteJSON(w, viewReq.Attributes, http.StatusOK) +} + +func (h *APIHandler) HandleGetFieldCapsAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + resBody := map[string]interface{}{} + targetClusterID := ps.ByName("id") + + pattern := h.GetParameterOrDefault(req, "pattern", "*") + keyword := h.GetParameterOrDefault(req, "keyword", "") + aggregatable := h.GetParameterOrDefault(req, "aggregatable", "") + size := h.GetIntOrDefault(req, "size", 0) + typ := h.GetParameterOrDefault(req, "type", "") + esType := h.GetParameterOrDefault(req, "es_type", "") + + metaFields := req.URL.Query()["meta_fields"] + esClient := elastic.GetClient(targetClusterID) + kbnFields, err := elastic.GetFieldCaps(esClient, pattern, metaFields) + if err != nil { + log.Error(err) + resBody["error"] = err.Error() + h.WriteJSON(w, resBody, http.StatusInternalServerError) + return + } + if keyword != "" || aggregatable != "" || typ != "" || esType != "" || size > 0 { + var filteredFields []elastic.ElasticField + var count = 0 + for _, field := range kbnFields { + if keyword != "" && !strings.Contains(field.Name, keyword) { + continue + } + if aggregatable == "true" && !field.Aggregatable { + continue + } + if typ != "" && field.Type != typ { + continue + } + if esType != "" && field.ESTypes[0] != esType { + continue + } + count++ + if size > 0 && count > size { + break + } + filteredFields = append(filteredFields, field) + } + kbnFields = filteredFields + } + + resBody["fields"] = kbnFields + h.WriteJSON(w, resBody, http.StatusOK) +} + +func (h *APIHandler) HandleGetViewAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("view_id") + + obj := elastic.View{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteGetOKJSON(w, id, obj) +} + +func (h *APIHandler) SetDefaultLayout(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var viewReq = &elastic.View{} + + err := h.DecodeJSON(req, viewReq) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + id := ps.MustGetParameter("view_id") + viewObj := elastic.View{} + viewObj.ID = id + exists, err := orm.Get(&viewObj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + viewObj.DefaultLayoutID = viewReq.DefaultLayoutID + ctx := &orm.Context{ + Refresh: "wait_for", + } + err = orm.Update(ctx, &viewObj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "success": true, + }, 200) + +} diff --git a/modules/security/api/account.go b/modules/security/api/account.go new file mode 100644 index 00000000..4c6d3aaf --- /dev/null +++ b/modules/security/api/account.go @@ -0,0 +1,206 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + "fmt" + "golang.org/x/crypto/bcrypt" + rbac "infini.sh/console/core/security" + "infini.sh/console/modules/security/realm" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" + "net/http" +) + +const userInSession = "user_session:" + +// const SSOProvider = "sso" +const NativeProvider = "native" + +//const LDAPProvider = "ldap" + +func (h APIHandler) Logout(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + rbac.DeleteUserToken(reqUser.UserId) + h.WriteOKJSON(w, util.MapStr{ + "status": "ok", + }) +} + +func (h APIHandler) Profile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + if reqUser.Provider == NativeProvider { + user, err := h.User.Get(reqUser.UserId) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + if user.Nickname == "" { + user.Nickname = user.Username + } + + u := util.MapStr{ + "user_id": user.ID, + "name": user.Username, + "email": user.Email, + "nick_name": user.Nickname, + "phone": user.Phone, + } + + h.WriteOKJSON(w, api.FoundResponse(reqUser.UserId, u)) + } else { + + //TODO fetch external profile + + u := util.MapStr{ + "user_id": reqUser.UserId, + "name": reqUser.Username, + "email": "", //TOOD, save user profile come from SSO + "nick_name": reqUser.Username, //TODO + "phone": "", //TODO + } + h.WriteOKJSON(w, api.FoundResponse(reqUser.UserId, u)) + } + +} + +func (h APIHandler) UpdatePassword(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + var req struct { + OldPassword string `json:"old_password"` + NewPassword string `json:"new_password"` + } + err = h.DecodeJSON(r, &req) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + user, err := h.User.Get(reqUser.UserId) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(req.OldPassword)) + if err == bcrypt.ErrMismatchedHashAndPassword { + h.ErrorInternalServer(w, "old password is not correct") + return + } + hash, err := bcrypt.GenerateFromPassword([]byte(req.NewPassword), bcrypt.DefaultCost) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + user.Password = string(hash) + err = h.User.Update(&user) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + h.WriteOKJSON(w, api.UpdateResponse(reqUser.UserId)) + return +} + +func (h APIHandler) UpdateProfile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + var req struct { + Name string `json:"name"` + Phone string `json:"phone"` + Email string `json:"email"` + } + err = h.DecodeJSON(r, &req) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + user, err := h.User.Get(reqUser.UserId) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + user.Username = req.Name + user.Email = req.Email + user.Phone = req.Phone + err = h.User.Update(&user) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + h.WriteOKJSON(w, api.UpdateResponse(reqUser.UserId)) + return +} + +func (h APIHandler) Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var req struct { + Username string `json:"username"` + Password string `json:"password"` + } + err := h.DecodeJSON(r, &req) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + var user *rbac.User + + //check user validation + ok, user, err := realm.Authenticate(req.Username, req.Password) + if err != nil { + h.WriteError(w, err.Error(), 500) + return + } + + if !ok { + h.WriteError(w, "invalid username or password", 403) + return + } + + if user == nil { + h.ErrorInternalServer(w, fmt.Sprintf("failed to authenticate user: %v", req.Username)) + return + } + + //check permissions + ok, err = realm.Authorize(user) + if err != nil || !ok { + h.ErrorInternalServer(w, fmt.Sprintf("failed to authorize user: %v", req.Username)) + return + } + + //fetch user profile + //TODO + if user.Nickname == "" { + user.Nickname = user.Username + } + + //generate access token + token, err := rbac.GenerateAccessToken(user) + if err != nil { + h.ErrorInternalServer(w, fmt.Sprintf("failed to authorize user: %v", req.Username)) + return + } + + //api.SetSession(w, r, userInSession+req.Username, req.Username) + h.WriteOKJSON(w, token) +} diff --git a/modules/security/api/init.go b/modules/security/api/init.go new file mode 100644 index 00000000..a36a0f9c --- /dev/null +++ b/modules/security/api/init.go @@ -0,0 +1,47 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + "infini.sh/console/core" + rbac "infini.sh/console/core/security" + "infini.sh/console/core/security/enum" + "infini.sh/framework/core/api" +) + +type APIHandler struct { + core.Handler + rbac.Adapter +} + +const adapterType = "native" + +var apiHandler = APIHandler{Adapter: rbac.GetAdapter(adapterType)} //TODO handle hard coded + +func Init() { + + api.HandleAPIMethod(api.GET, "/permission/:type", apiHandler.RequireLogin(apiHandler.ListPermission)) + + api.HandleAPIMethod(api.POST, "/role/:type", apiHandler.RequirePermission(apiHandler.CreateRole, enum.RoleAllPermission...)) + api.HandleAPIMethod(api.GET, "/role/:id", apiHandler.RequirePermission(apiHandler.GetRole, enum.RoleReadPermission...)) + api.HandleAPIMethod(api.DELETE, "/role/:id", apiHandler.RequirePermission(apiHandler.DeleteRole, enum.RoleAllPermission...)) + api.HandleAPIMethod(api.PUT, "/role/:id", apiHandler.RequirePermission(apiHandler.UpdateRole, enum.RoleAllPermission...)) + api.HandleAPIMethod(api.GET, "/role/_search", apiHandler.RequirePermission(apiHandler.SearchRole, enum.RoleReadPermission...)) + + api.HandleAPIMethod(api.POST, "/user", apiHandler.RequirePermission(apiHandler.CreateUser, enum.UserAllPermission...)) + api.HandleAPIMethod(api.GET, "/user/:id", apiHandler.RequirePermission(apiHandler.GetUser, enum.UserReadPermission...)) + api.HandleAPIMethod(api.DELETE, "/user/:id", apiHandler.RequirePermission(apiHandler.DeleteUser, enum.UserAllPermission...)) + api.HandleAPIMethod(api.PUT, "/user/:id", apiHandler.RequirePermission(apiHandler.UpdateUser, enum.UserAllPermission...)) + api.HandleAPIMethod(api.GET, "/user/_search", apiHandler.RequirePermission(apiHandler.SearchUser, enum.UserReadPermission...)) + api.HandleAPIMethod(api.PUT, "/user/:id/password", apiHandler.RequirePermission(apiHandler.UpdateUserPassword, enum.UserAllPermission...)) + + api.HandleAPIMethod(api.POST, "/account/login", apiHandler.Login) + api.HandleAPIMethod(api.POST, "/account/logout", apiHandler.Logout) + api.HandleAPIMethod(api.DELETE, "/account/logout", apiHandler.Logout) + + api.HandleAPIMethod(api.GET, "/account/profile", apiHandler.RequireLogin(apiHandler.Profile)) + api.HandleAPIMethod(api.PUT, "/account/password", apiHandler.RequireLogin(apiHandler.UpdatePassword)) + +} diff --git a/modules/security/api/permission.go b/modules/security/api/permission.go new file mode 100644 index 00000000..c706df23 --- /dev/null +++ b/modules/security/api/permission.go @@ -0,0 +1,28 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + log "github.com/cihub/seelog" + rbac "infini.sh/console/core/security" + httprouter "infini.sh/framework/core/api/router" + "net/http" +) + +func (h APIHandler) ListPermission(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + typ := ps.MustGetParameter("type") + err := rbac.IsAllowRoleType(typ) + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + var permissions interface{} + if typ == rbac.Elasticsearch { + permissions = rbac.GetPermissions(typ) + } + h.WriteOKJSON(w, permissions) + return +} diff --git a/modules/security/api/role.go b/modules/security/api/role.go new file mode 100644 index 00000000..86350fb9 --- /dev/null +++ b/modules/security/api/role.go @@ -0,0 +1,186 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + log "github.com/cihub/seelog" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" + "net/http" + "time" +) + +func (h APIHandler) CreateRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + roleType := ps.MustGetParameter("type") + + //localUser, err := rbac.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + err := rbac.IsAllowRoleType(roleType) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + role := &rbac.Role{ + Type: roleType, + } + err = h.DecodeJSON(r, role) + if err != nil { + h.Error400(w, err.Error()) + return + } + if _, ok := rbac.RoleMap[role.Name]; ok { + h.ErrorInternalServer(w, "role name already exists") + return + } + now := time.Now() + role.Created = &now + role.Updated = role.Created + role.Type = roleType + var id string + id, err = h.Adapter.Role.Create(role) + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + rbac.RoleMap[role.Name] = *role + _ = h.WriteOKJSON(w, api.CreateResponse(id)) + return + +} + +func (h APIHandler) SearchRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + var ( + keyword = h.GetParameterOrDefault(r, "keyword", "") + from = h.GetIntOrDefault(r, "from", 0) + size = h.GetIntOrDefault(r, "size", 20) + ) + + res, err := h.Adapter.Role.Search(keyword, from, size) + if err != nil { + log.Error(err) + h.ErrorInternalServer(w, err.Error()) + return + } + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + + hits := response.Hits.Hits + list := make([]elastic.IndexDocument, 0) + total := response.GetTotal() + var index string + for _, v := range hits { + index = v.Index + } + for k, v := range rbac.BuiltinRoles { + mval := map[string]interface{}{} + vbytes := util.MustToJSONBytes(v) + util.MustFromJSONBytes(vbytes, &mval) + list = append(list, elastic.IndexDocument{ + ID: k, + Index: index, + Type: "_doc", + Source: mval, + }) + total++ + } + list = append(list, hits...) + response.Hits.Hits = list + response.Hits.Total = total + + h.WriteOKJSON(w, response) + return + +} + +func (h APIHandler) GetRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + role, err := h.Adapter.Role.Get(id) + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + h.WriteOKJSON(w, api.Response{Hit: role}) + return +} + +func (h APIHandler) DeleteRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + + //localUser, err := biz.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + oldRole, err := h.Role.Get(id) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + } + err = h.Adapter.Role.Delete(id) + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + delete(rbac.RoleMap, oldRole.Name) + _ = h.WriteOKJSON(w, api.DeleteResponse(id)) + return +} + +func (h APIHandler) UpdateRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + //localUser, err := biz.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + role := &rbac.Role{} + err := h.DecodeJSON(r, role) + if err != nil { + h.Error400(w, err.Error()) + return + } + role.ID = id + + oldRole, err := h.Role.Get(id) + if err != nil { + log.Error(err) + h.ErrorInternalServer(w, err.Error()) + return + } + if role.Name != oldRole.Name { + h.ErrorInternalServer(w, "Changing role name is not allowed") + return + } + now := time.Now() + role.Type = oldRole.Type + role.Updated = &now + role.Created = oldRole.Created + err = h.Role.Update(role) + delete(rbac.RoleMap, oldRole.Name) + rbac.RoleMap[role.Name] = *role + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + _ = h.WriteOKJSON(w, api.UpdateResponse(id)) + return +} diff --git a/modules/security/api/user.go b/modules/security/api/user.go new file mode 100644 index 00000000..e636bb5f --- /dev/null +++ b/modules/security/api/user.go @@ -0,0 +1,260 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package api + +import ( + "bytes" + "errors" + "github.com/buger/jsonparser" + log "github.com/cihub/seelog" + "golang.org/x/crypto/bcrypt" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic" + "net/http" + "sort" + "time" +) + +func (h APIHandler) CreateUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var user rbac.User + err := h.DecodeJSON(r, &user) + if err != nil { + h.Error400(w, err.Error()) + return + } + if user.Username == "" { + h.Error400(w, "username is required") + return + } + //localUser, err := biz.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + if h.userNameExists(w, user.Username) { + return + } + randStr := util.GenerateRandomString(8) + hash, err := bcrypt.GenerateFromPassword([]byte(randStr), bcrypt.DefaultCost) + if err != nil { + return + } + user.Password = string(hash) + + now := time.Now() + user.Created = &now + user.Updated = &now + + id, err := h.User.Create(&user) + user.ID = id + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + _ = h.WriteOKJSON(w, util.MapStr{ + "_id": id, + "password": randStr, + "result": "created", + }) + return + +} + +func (h APIHandler) userNameExists(w http.ResponseWriter, name string) bool { + u, err := h.User.GetBy("name", name) + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return true + } + if u != nil { + h.ErrorInternalServer(w, "user name already exists") + return true + } + return false +} + +func (h APIHandler) GetUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + user, err := h.User.Get(id) + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) + return + } + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + h.WriteOKJSON(w, api.FoundResponse(id, user)) + return +} + +func (h APIHandler) UpdateUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + var user rbac.User + err := h.DecodeJSON(r, &user) + if err != nil { + _ = log.Error(err.Error()) + h.Error400(w, err.Error()) + return + } + //localUser, err := biz.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + oldUser, err := h.User.Get(id) + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + if user.Username != oldUser.Username && h.userNameExists(w, user.Username) { + return + } + + now := time.Now() + user.Updated = &now + user.Created = oldUser.Created + user.ID = id + err = h.User.Update(&user) + + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + //let user relogin after roles changed + sort.Slice(user.Roles, func(i, j int) bool { + return user.Roles[i].ID < user.Roles[j].ID + }) + sort.Slice(oldUser.Roles, func(i, j int) bool { + return oldUser.Roles[i].ID < oldUser.Roles[j].ID + }) + changeLog, _ := util.DiffTwoObject(user.Roles, oldUser.Roles) + if len(changeLog) > 0 { + rbac.DeleteUserToken(id) + } + _ = h.WriteOKJSON(w, api.UpdateResponse(id)) + return +} + +func (h APIHandler) DeleteUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + user, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error("failed to get user from context, err: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if user != nil && user.UserId == id { + h.WriteError(w, "can not delete yourself", http.StatusInternalServerError) + return + } + err = h.User.Delete(id) + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) + return + } + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + rbac.DeleteUserToken(id) + _ = h.WriteOKJSON(w, api.DeleteResponse(id)) + return +} + +func (h APIHandler) SearchUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var ( + keyword = h.GetParameterOrDefault(r, "keyword", "") + from = h.GetIntOrDefault(r, "from", 0) + size = h.GetIntOrDefault(r, "size", 20) + ) + + res, err := h.User.Search(keyword, from, size) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + //remove password field + hitsBuf := bytes.Buffer{} + hitsBuf.Write([]byte("[")) + jsonparser.ArrayEach(res.Raw, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + value = jsonparser.Delete(value, "_source", "password") + hitsBuf.Write(value) + hitsBuf.Write([]byte(",")) + }, "hits", "hits") + buf := hitsBuf.Bytes() + if buf[len(buf)-1] == ',' { + buf[len(buf)-1] = ']' + } else { + hitsBuf.Write([]byte("]")) + } + res.Raw, err = jsonparser.Set(res.Raw, hitsBuf.Bytes(), "hits", "hits") + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + + h.Write(w, res.Raw) + return + +} + +func (h APIHandler) UpdateUserPassword(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + var req = struct { + Password string `json:"password"` + }{} + err := h.DecodeJSON(r, &req) + if err != nil { + _ = log.Error(err.Error()) + h.Error400(w, err.Error()) + return + } + //localUser, err := biz.FromUserContext(r.Context()) + //if err != nil { + // log.Error(err.Error()) + // h.ErrorInternalServer(w, err.Error()) + // return + //} + user, err := h.User.Get(id) + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + hash, err := bcrypt.GenerateFromPassword([]byte(req.Password), bcrypt.DefaultCost) + if err != nil { + return + } + user.Password = string(hash) + //t:=time.Now() + //user.Updated =&t + err = h.User.Update(&user) + if err != nil { + _ = log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + //disable old token to let user login + rbac.DeleteUserToken(id) + + _ = h.WriteOKJSON(w, api.UpdateResponse(id)) + return + +} diff --git a/modules/security/config/config.go b/modules/security/config/config.go new file mode 100644 index 00000000..bb34b922 --- /dev/null +++ b/modules/security/config/config.go @@ -0,0 +1,32 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package config + +import ( + ldap2 "infini.sh/console/modules/security/realm/authc/ldap" +) + +type Config struct { + Enabled bool `config:"enabled"` + Authentication AuthenticationConfig `config:"authc"` + OAuthConfig OAuthConfig `config:"oauth"` +} + +type RealmConfig struct { + Enabled bool `config:"enabled"` + Order int `config:"order"` +} + +type RealmsConfig struct { + Native RealmConfig `config:"native"` + + //ldap,oauth, active_directory, pki, file, saml, kerberos, oidc, jwt + OAuth map[string]OAuthConfig `config:"oauth"` + LDAP map[string]ldap2.LDAPConfig `config:"ldap"` +} + +type AuthenticationConfig struct { + Realms RealmsConfig `config:"realms"` +} diff --git a/modules/security/config/oauth.go b/modules/security/config/oauth.go new file mode 100644 index 00000000..0bf39206 --- /dev/null +++ b/modules/security/config/oauth.go @@ -0,0 +1,20 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package config + +type OAuthConfig struct { + Enabled bool `config:"enabled"` + ClientID string `config:"client_id"` + ClientSecret string `config:"client_secret"` + DefaultRoles []string `config:"default_roles"` + RoleMapping map[string][]string `config:"role_mapping"` + AuthorizeUrl string `config:"authorize_url"` + TokenUrl string `config:"token_url"` + RedirectUrl string `config:"redirect_url"` + Scopes []string `config:"scopes"` + + SuccessPage string `config:"success_page"` + FailedPage string `config:"failed_page"` +} diff --git a/modules/security/credential/api/credential.go b/modules/security/credential/api/credential.go new file mode 100644 index 00000000..6a413ff8 --- /dev/null +++ b/modules/security/credential/api/credential.go @@ -0,0 +1,277 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + "context" + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/core" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/task" + "infini.sh/framework/core/util" + "net/http" + "strconv" +) + +type APIHandler struct { + core.Handler +} + +func (h *APIHandler) createCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + cred := credential.Credential{} + err := h.DecodeJSON(req, &cred) + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + err = cred.Validate() + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + err = cred.Encode() + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + ctx := orm.Context{ + Refresh: "wait_for", + } + cred.ID = util.GetUUID() + err = orm.Create(&ctx, &cred) + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteCreatedOKJSON(w, cred.ID) +} + +func (h *APIHandler) updateCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + obj := credential.Credential{} + + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + newObj := credential.Credential{} + err = h.DecodeJSON(req, &newObj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + err = newObj.Validate() + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + encodeChanged := false + if obj.Type != newObj.Type { + encodeChanged = true + } else { + switch newObj.Type { + case credential.BasicAuth: + var oldPwd string + if oldParams, ok := obj.Payload[newObj.Type].(map[string]interface{}); ok { + if pwd, ok := oldParams["password"].(string); ok { + oldPwd = pwd + } else { + http.Error(w, fmt.Sprintf("invalid password of credential [%s]", obj.ID), http.StatusInternalServerError) + return + } + } + if params, ok := newObj.Payload[newObj.Type].(map[string]interface{}); ok { + if pwd, ok := params["password"].(string); ok && pwd != oldPwd { + obj.Payload = newObj.Payload + encodeChanged = true + } else { + if oldParams, ok := obj.Payload[obj.Type].(map[string]interface{}); ok { + oldParams["username"] = params["username"] + } + } + } + default: + h.WriteError(w, fmt.Sprintf("unsupport credential type [%s]", newObj.Type), http.StatusInternalServerError) + return + } + } + obj.Name = newObj.Name + obj.Type = newObj.Type + obj.Tags = newObj.Tags + if encodeChanged { + err = obj.Encode() + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + } + ctx := &orm.Context{ + Refresh: "wait_for", + } + obj.Invalid = false + err = orm.Update(ctx, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + task.RunWithinGroup("credential_callback", func(ctx context.Context) error { + credential.TriggerChangeEvent(&obj) + return nil + }) + + h.WriteUpdatedOKJSON(w, id) +} + +func (h *APIHandler) deleteCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + + obj := credential.Credential{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + //check dependency + toDelete, err := canDelete(&obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if !toDelete { + h.WriteError(w, "This credential is in use and cannot be deleted", http.StatusInternalServerError) + return + } + ctx := &orm.Context{ + Refresh: "wait_for", + } + err = orm.Delete(ctx, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteDeletedOKJSON(w, id) +} + +func canDelete(cred *credential.Credential) (bool, error) { + if cred == nil { + return false, fmt.Errorf("parameter cred can not be nil") + } + q := orm.Query{ + Conds: orm.And(orm.Eq("credential_id", cred.ID)), + } + err, result := orm.Search(elastic.ElasticsearchConfig{}, &q) + if err != nil { + return false, fmt.Errorf("query elasticsearch config error: %w", err) + } + return result.Total == 0, nil +} + +func (h *APIHandler) searchCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + keyword = h.GetParameterOrDefault(req, "keyword", "") + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustQ []interface{} + ) + if keyword != "" { + mustQ = append(mustQ, util.MapStr{ + "query_string": util.MapStr{ + "default_field": "*", + "query": keyword, + }, + }) + } + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + + queryDSL := util.MapStr{ + "size": size, + "from": from, + "sort": []util.MapStr{ + { + "created": util.MapStr{ + "order": "desc", + }, + }, + }, + } + if len(mustQ) > 0 { + queryDSL["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": mustQ, + }, + } + } + + q := orm.Query{} + q.RawQuery = util.MustToJSONBytes(queryDSL) + + err, res := orm.Search(&credential.Credential{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + searchRes := elastic.SearchResponse{} + util.MustFromJSONBytes(res.Raw, &searchRes) + if len(searchRes.Hits.Hits) > 0 { + for _, hit := range searchRes.Hits.Hits { + delete(hit.Source, "encrypt") + util.MapStr(hit.Source).Delete("payload.basic_auth.password") + } + } + + h.WriteJSON(w, searchRes, http.StatusOK) +} + +func (h *APIHandler) getCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + obj := credential.Credential{} + + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + util.MapStr(obj.Payload).Delete("basic_auth.password") + h.WriteGetOKJSON(w, id, obj) +} diff --git a/modules/security/credential/api/init.go b/modules/security/credential/api/init.go new file mode 100644 index 00000000..49515044 --- /dev/null +++ b/modules/security/credential/api/init.go @@ -0,0 +1,58 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package api + +import ( + log "github.com/cihub/seelog" + "infini.sh/console/core/security/enum" + "infini.sh/framework/core/api" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/common" +) + +func Init() { + handler := APIHandler{} + api.HandleAPIMethod(api.POST, "/credential", handler.RequirePermission(handler.createCredential, enum.PermissionCredentialWrite)) + api.HandleAPIMethod(api.PUT, "/credential/:id", handler.RequirePermission(handler.updateCredential, enum.PermissionCredentialWrite)) + api.HandleAPIMethod(api.DELETE, "/credential/:id", handler.RequirePermission(handler.deleteCredential, enum.PermissionCredentialWrite)) + api.HandleAPIMethod(api.GET, "/credential/_search", handler.RequirePermission(handler.searchCredential, enum.PermissionCredentialRead)) + api.HandleAPIMethod(api.GET, "/credential/:id", handler.RequirePermission(handler.getCredential, enum.PermissionCredentialRead)) + + credential.RegisterChangeEvent(func(cred *credential.Credential) { + var keys []string + elastic.WalkConfigs(func(key, value interface{}) bool { + if v, ok := value.(*elastic.ElasticsearchConfig); ok { + if v.CredentialID == cred.ID { + if k, ok := key.(string); ok { + keys = append(keys, k) + } + } + } + return true + }) + for _, key := range keys { + conf := elastic.GetConfig(key) + if conf.CredentialID == cred.ID { + obj, err := cred.Decode() + if err != nil { + log.Error(err) + continue + } + if v, ok := obj.(model.BasicAuth); ok { + newConf := *conf + newConf.BasicAuth = &v + _, err = common.InitElasticInstance(newConf) + if err != nil { + log.Error(err) + } + log.Tracef("updated cluster config: %s", util.MustToJSON(newConf)) + } + } + } + }) +} diff --git a/modules/security/module.go b/modules/security/module.go new file mode 100644 index 00000000..2b71f581 --- /dev/null +++ b/modules/security/module.go @@ -0,0 +1,83 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package security + +import ( + rbac "infini.sh/console/core/security" + authapi "infini.sh/console/modules/security/api" + "infini.sh/console/modules/security/config" + credapi "infini.sh/console/modules/security/credential/api" + "infini.sh/console/modules/security/realm" + "infini.sh/console/modules/security/realm/authc/oauth" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" +) + +type Module struct { + cfg *config.Config +} + +func (module *Module) Name() string { + return "security" +} + +func (module *Module) Setup() { + module.cfg = &config.Config{ + Enabled: false, + Authentication: config.AuthenticationConfig{ + Realms: config.RealmsConfig{ + Native: config.RealmConfig{ + Enabled: true, + }, + }, + }, + OAuthConfig: config.OAuthConfig{ + SuccessPage: "/#/user/sso/success", + FailedPage: "/#/user/sso/failed", + }, + } + + ok, err := env.ParseConfig("security", &module.cfg) + if ok && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { + panic(err) + } + + if !module.cfg.Enabled { + return + } + + credapi.Init() + + if module.cfg.OAuthConfig.Enabled { + oauth.Init(module.cfg.OAuthConfig) + } + + authapi.Init() +} + +func InitSchema() { + orm.RegisterSchemaWithIndexName(rbac.Role{}, "rbac-role") + orm.RegisterSchemaWithIndexName(rbac.User{}, "rbac-user") + orm.RegisterSchemaWithIndexName(credential.Credential{}, "credential") +} + +func (module *Module) Start() error { + if !module.cfg.Enabled { + return nil + } + + InitSchema() + + realm.Init(module.cfg) + + return nil +} + +func (module *Module) Stop() error { + + return nil +} diff --git a/modules/security/realm/authc/ldap/init.go b/modules/security/realm/authc/ldap/init.go new file mode 100644 index 00000000..d708c713 --- /dev/null +++ b/modules/security/realm/authc/ldap/init.go @@ -0,0 +1,5 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package ldap diff --git a/modules/security/realm/authc/ldap/ldap.go b/modules/security/realm/authc/ldap/ldap.go new file mode 100644 index 00000000..8b7a54f4 --- /dev/null +++ b/modules/security/realm/authc/ldap/ldap.go @@ -0,0 +1,151 @@ +package ldap + +import ( + "context" + "errors" + log "github.com/cihub/seelog" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/guardian/auth" + "infini.sh/framework/lib/guardian/auth/strategies/basic" + "infini.sh/framework/lib/guardian/auth/strategies/ldap" + "time" +) + +type LDAPConfig struct { + Enabled bool `config:"enabled"` + Tls bool `config:"tls"` + Host string `config:"host"` + Port int `config:"port"` + BindDn string `config:"bind_dn"` + BindPassword string `config:"bind_password"` + BaseDn string `config:"base_dn"` + UserFilter string `config:"user_filter"` + UidAttribute string `config:"uid_attribute"` + GroupAttribute string `config:"group_attribute"` + + RoleMapping struct { + Group map[string][]string `config:"group"` + Uid map[string][]string `config:"uid"` + } `config:"role_mapping"` +} + +func (r *LDAPRealm) mapLDAPRoles(authInfo auth.Info) []string { + var ret []string + + if global.Env().IsDebug { + log.Tracef("mapping LDAP authInfo: %v", authInfo) + } + + //check uid + uid := authInfo.GetID() + if uid == "" { + uid = authInfo.GetUserName() + } + + if global.Env().IsDebug { + log.Tracef("ldap config: %v", util.MustToJSON(r.config)) + } + + if roles, ok := r.config.RoleMapping.Uid[uid]; ok { + ret = append(ret, roles...) + } else { + if global.Env().IsDebug { + log.Tracef("ldap uid mapping config: %v", r.config.RoleMapping.Uid) + } + log.Debugf("LDAP uid: %v, user: %v", uid, authInfo) + } + + //map group + for _, roleName := range authInfo.GetGroups() { + newRoles, ok := r.config.RoleMapping.Group[roleName] + if ok { + ret = append(ret, newRoles...) + } else { + if global.Env().IsDebug { + log.Tracef("ldap group mapping config: %v", r.config.RoleMapping.Group) + } + log.Debugf("LDAP group: %v, roleName: %v, match: %v", uid, roleName, newRoles) + } + } + + return ret +} + +func New(cfg2 LDAPConfig) *LDAPRealm { + + var realm = &LDAPRealm{ + config: cfg2, + ldapCfg: ldap.Config{ + Port: cfg2.Port, + Host: cfg2.Host, + TLS: nil, + BindDN: cfg2.BindDn, + BindPassword: cfg2.BindPassword, + Attributes: nil, + BaseDN: cfg2.BaseDn, + UserFilter: cfg2.UserFilter, + GroupFilter: "", + UIDAttribute: cfg2.UidAttribute, + GroupAttribute: cfg2.GroupAttribute, + }, + } + realm.ldapFunc = ldap.GetAuthenticateFunc(&realm.ldapCfg) + return realm +} + +const providerName = "ldap" + +type LDAPRealm struct { + config LDAPConfig + ldapCfg ldap.Config + ldapFunc basic.AuthenticateFunc +} + +func (r *LDAPRealm) GetType() string { + return providerName +} + +func (r *LDAPRealm) Authenticate(username, password string) (bool, *rbac.User, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Second*10)) + defer cancel() + + authInfo, err := r.ldapFunc(ctx, nil, []byte(username), []byte(password)) + if err != nil { + return false, nil, err + } + u := &rbac.User{ + AuthProvider: providerName, + Username: authInfo.GetUserName(), + Nickname: authInfo.GetUserName(), + Email: "", + } + u.Payload = &authInfo + u.ID = authInfo.GetUserName() + return true, u, err +} + +func (r *LDAPRealm) Authorize(user *rbac.User) (bool, error) { + authInfo := user.Payload.(*auth.Info) + if authInfo != nil { + roles := r.mapLDAPRoles(*authInfo) + for _, roleName := range roles { + user.Roles = append(user.Roles, rbac.UserRole{ + ID: roleName, + Name: roleName, + }) + } + } else { + log.Warnf("LDAP %v auth Info is nil", user.Username) + } + + var _, privilege = user.GetPermissions() + + if len(privilege) == 0 { + log.Debug("no privilege assigned to user:", user) + return false, errors.New("no privilege assigned to this user:" + user.Username) + } + + return true, nil +} diff --git a/modules/security/realm/authc/native/init.go b/modules/security/realm/authc/native/init.go new file mode 100644 index 00000000..6d78bf08 --- /dev/null +++ b/modules/security/realm/authc/native/init.go @@ -0,0 +1,71 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package native + +import ( + "errors" + "fmt" + log "github.com/cihub/seelog" + "golang.org/x/crypto/bcrypt" + rbac "infini.sh/console/core/security" +) + +var handler rbac.Adapter + +func init() { + handler = rbac.Adapter{ + User: &User{}, + Role: &Role{}, + } + rbac.RegisterAdapter(providerName, handler) +} + +const providerName = "native" + +type NativeRealm struct { + // Implement any required fields +} + +func (r *NativeRealm) GetType() string { + return providerName +} + +func (r *NativeRealm) Authenticate(username, password string) (bool, *rbac.User, error) { + // Implement the authentication logic + // Retrieve the user profile upon successful authentication + // Return the authentication result, user profile, and any potential error + + user, err := handler.User.GetBy("name", username) + if err != nil { + return false, user, err + } + if user == nil { + return false, nil, fmt.Errorf("user account [%s] not found", username) + } + + err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(password)) + if err == bcrypt.ErrMismatchedHashAndPassword { + err = errors.New("incorrect password") + } + + if err == nil { + user.AuthProvider = providerName + return true, user, nil + } + + return false, nil, err +} + +func (r *NativeRealm) Authorize(user *rbac.User) (bool, error) { + + var _, privilege = user.GetPermissions() + + if len(privilege) == 0 { + log.Error("no privilege assigned to user:", user) + return false, errors.New("no privilege assigned to this user:" + user.Username) + } + + return true, nil +} diff --git a/modules/security/realm/authc/native/load.go b/modules/security/realm/authc/native/load.go new file mode 100644 index 00000000..2bbd37a4 --- /dev/null +++ b/modules/security/realm/authc/native/load.go @@ -0,0 +1,129 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package native + +import ( + _ "embed" + "github.com/mitchellh/mapstructure" + "infini.sh/framework/core/elastic" + "path" + "strings" + + log "github.com/cihub/seelog" + "github.com/segmentio/encoding/json" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/api/routetree" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +type ElasticsearchAPIMetadata struct { + Name string `json:"name"` + Methods []string `json:"methods"` + Path string `json:"path"` +} + +type ElasticsearchAPIMetadataList []ElasticsearchAPIMetadata + +func (list ElasticsearchAPIMetadataList) GetNames() []string { + var names []string + for _, md := range list { + if !util.StringInArray(names, md.Name) { + names = append(names, md.Name) + } + } + return names +} + +//go:embed permission.json +var permissionFile []byte + +func loadJsonConfig() map[string]ElasticsearchAPIMetadataList { + externalConfig := path.Join(global.Env().GetConfigDir(), "permission.json") + if util.FileExists(externalConfig) { + log.Infof("loading permission file from %s", externalConfig) + bytes, err := util.FileGetContent(externalConfig) + if err != nil { + log.Errorf("load permission file failed, use embedded config, err: %v", err) + } else { + permissionFile = bytes + } + } + apis := map[string]ElasticsearchAPIMetadataList{} + err := json.Unmarshal(permissionFile, &apis) + if err != nil { + log.Error("json config unmarshal err " + err.Error()) + return nil + } + + return apis +} + +func Init() { + + //load local files + apis := loadJsonConfig() + if apis != nil { + var esAPIRouter = routetree.New() + for _, list := range apis { + for _, md := range list { + //skip wildcard * + if strings.HasSuffix(md.Path, "*") { + continue + } + for _, method := range md.Methods { + esAPIRouter.Handle(method, md.Path, md.Name) + } + } + } + rbac.RegisterAPIPermissionRouter("elasticsearch", esAPIRouter) + } + + permissions := map[string]interface{}{ + "index_privileges": apis["indices"].GetNames(), + } + delete(apis, "indices") + otherApis := map[string][]string{} + for key, list := range apis { + otherApis[key] = list.GetNames() + + } + permissions["cluster_privileges"] = otherApis + rbac.RegisterPermission(rbac.Elasticsearch, permissions) + + //load role from store + loadRemoteRolePermission() +} + +func loadRemoteRolePermission() { + log.Trace("start loading roles from adapter") + rbac.RoleMap = make(map[string]rbac.Role) + for k, role := range rbac.BuiltinRoles { + rbac.RoleMap[k] = role + } + + log.Debug("load security permissions,", rbac.RoleMap, rbac.BuiltinRoles) + + res, err := handler.Role.Search("", 0, 1000) + if err != nil { + log.Error(err) + return + } + response := elastic.SearchResponse{} + util.FromJSONBytes(res.Raw, &response) + + for _, v := range response.Hits.Hits { + var role rbac.Role + delete(v.Source, "created") + delete(v.Source, "updated") + err = mapstructure.Decode(v.Source, &role) + if err != nil { + log.Error(err) + return + } + rbac.RoleMap[role.Name] = role + } + +} diff --git a/modules/security/realm/authc/native/permission.json b/modules/security/realm/authc/native/permission.json new file mode 100644 index 00000000..37882a7c --- /dev/null +++ b/modules/security/realm/authc/native/permission.json @@ -0,0 +1,487 @@ +{ + + "cat": [ + {"name": "cat.*", "methods": ["get"], + "path": "_cat/*" + }, + {"name": "cat.indices", "methods": ["get"], + "path": "_cat/indices" + }, + {"name": "cat.indices", "methods": ["get"], + "path": "_cat/indices/:target" + }, + {"name": "cat.help", "methods": ["get"], + "path": "_cat/help" + }, + {"name": "cat.repositories", "methods": ["get"], + "path": "_cat/repositories" + }, + {"name": "cat.pending_tasks", "methods": ["get"], + "path": "_cat/pending_tasks" + }, + {"name": "cat.tasks", "methods": ["get"], + "path": "_cat/tasks" + }, + {"name": "cat.allocation", "methods": ["get"], + "path": "_cat/allocation" + }, + {"name": "cat.count", "methods": ["get"], + "path": "_cat/count" + }, + {"name": "cat.shards", "methods": ["get"], + "path": "_cat/shards" + }, + {"name": "cat.shards", "methods": ["get"], + "path": "_cat/shards/:target" + }, + {"name": "cat.aliases", "methods": ["get"], + "path": "_cat/aliases" + }, + {"name": "cat.aliases", "methods": ["get"], + "path": "_cat/aliases/:name" + }, + {"name": "cat.nodeattrs", "methods": ["get"], + "path": "_cat/nodeattrs" + }, + {"name": "cat.templates", "methods": ["get"], + "path": "_cat/templates" + }, + {"name": "cat.thread_pool", "methods": ["get"], + "path": "_cat/thread_pool" + }, + {"name": "cat.health", "methods": ["get"], + "path": "_cat/health" + }, + {"name": "cat.recovery", "methods": ["get"], + "path": "_cat/recovery" + }, + {"name": "cat.fielddata", "methods": ["get"], + "path": "_cat/fielddata" + }, + {"name": "cat.nodes", "methods": ["get"], + "path": "_cat/nodes" + }, + {"name": "cat.plugins", "methods": ["get"], + "path": "_cat/plugins" + }, + {"name": "cat.segments", "methods": ["get"], + "path": "_cat/segments" + }, + {"name": "cat.snapshots", "methods": ["get"], + "path": "_cat/snapshots" + }, + {"name": "cat.master", "methods": ["get"], + "path": "_cat/master" + } + ], + "cluster": [ + {"name": "cluster.*", "methods": ["*"], + "path": "_cluster/*" + }, + {"name": "cluster.info", "methods": ["get"], + "path": "/" + }, + {"name": "cluster.health", "methods": ["get"], + "path": "_cluster/health" + }, + {"name": "cluster.get_settings", "methods":["get"], + "path": "_cluster/settings" + }, + {"name": "cluster.pending_tasks", "methods": ["get"], + "path": "_cluster/pending_tasks" + }, + {"name": "cluster.stats", "methods": ["get"], + "path": "_cluster/stats" + }, + {"name": "cluster.remote_info", "methods": ["get"], + "path": "_remote/info" + }, + {"name": "cluster.allocation_explain", "methods": ["get"], + "path": "_cluster/allocation/explain" + }, + {"name": "cluster.put_settings", "methods": ["put"], + "path": "_cluster/settings" + }, + {"name": "cluster.reroute", "methods": ["post"], + "path": "_cluster/reroute" + }, + {"name": "cluster.count", "methods": ["get"], + "path": "_count" + }, + {"name": "cluster.state", "methods": ["get"], + "path": "_cluster/state" + }, + {"name": "cluster.bulk", "methods": ["put", "post"], + "path": "_bulk" + }, + {"name": "cluster.mget", "methods": ["get", "post"], + "path": "_mget" + }, + {"name": "cluster.ping", "methods": ["head"], + "path": "/" + }, + {"name": "cluster.msearch", "methods": ["get", "post"], + "path": "_msearch" + }, + {"name": "cluster.msearch_template", "methods": ["get", "post"], + "path": "_msearch/template" + }, + {"name": "cluster.mtermvectors", "methods": ["get", "post"], + "path": "_mtermvectors" + }, + {"name": "cluster.rank_eval", "methods": ["get", "post"], + "path": "_rank_eval" + }, + {"name": "cluster.search", "methods": ["get", "post"], + "path": "_search" + }, + {"name": "cluster.search_shards", "methods": ["get", "post"], + "path": "_search_shards" + }, + {"name": "cluster.exists_alias", "methods": ["head"], + "path": "_alias/:alias" + }, + {"name": "cluster.get_alias", "methods": ["get"], + "path": "_alias/:alias" + } + ], + "indices": [ + {"name": "indices.*", "methods": ["*"], + "path": "/*" + }, + {"name": "indices.exists_alias", "methods": ["head"], + "path": "/:index_name/_alias/:alias" + }, + {"name": "indices.get_alias", "methods": ["get"], + "path": "/:index_name/_alias/:alias" + }, + {"name": "indices.recovery", "methods": ["get"], + "path": "/:index_name/_recovery" + }, + {"name": "indices.delete", "methods": ["delete"], + "path": "/:index_name" + }, + {"name": "indices.put", "methods": ["put"], + "path": "/:index_name" + }, + {"name": "indices.clear_cache", "methods": ["post"], + "path": "/:index_name/_cache/clear" + }, + {"name": "indices.update_by_query", "methods": ["post"], + "path": "/:index_name/_update_by_query" + }, + {"name": "indices.shrink", "methods": ["post"], + "path": "/:index_name/_shrink" + }, + {"name": "indices.forcemerge", "methods": ["post"], + "path": "/:index_name/_forcemerge" + }, + {"name": "indices.put_alias", "methods": ["put"], + "path": "/:index_name/_alias/:alias" + }, + {"name": "indices.create", "methods": ["post"], + "path": "/:index_name" + }, + {"name": "indices.split", "methods": ["post"], + "path": "/:index_name/_split" + }, + {"name": "indices.flush", "methods": ["post"], + "path": "/:index_name/_flush" + }, + {"name": "indices.get_mapping", "methods": ["get"], + "path": "/:index_name/_mapping" + }, + {"name": "indices.upgrade", "methods": ["post"], + "path": "/:index_name/_upgrade" + }, + {"name": "indices.validate_query", "methods": ["get", "post"], + "path": "/:index_name/_validate/query" + }, + {"name": "indices.analyze", "methods": ["post"], + "path": "/:index_name/analyze" + }, + {"name": "indices.exists", "methods": ["head"], + "path": "/:index_name" + }, + {"name": "indices.close", "methods": ["post"], + "path": "/:index_name/_close" + }, + {"name": "indices.get_field_mapping", "methods": ["get"], + "path": "/:index_name/_mapping/:fields" + }, + {"name": "indices.delete_alias", "methods": ["delete"], + "path": "/:index_name/_alias/:alias" + }, + {"name": "indices.refresh", "methods": ["get", "post"], + "path": "/:index_name/_refresh" + }, + {"name": "indices.segments", "methods": ["get"], + "path": "/:index_name/_segments" + }, + {"name": "indices.termvectors", "methods": ["get"], + "path": "/:index_name/_termvectors" + }, + {"name": "indices.flush_synced", "methods": ["get", "post"], + "path": "/:index_name/_flush/synced" + }, + {"name": "indices.put_mapping", "methods": ["put"], + "path": "/:index_name/_mapping" + }, + {"name": "indices.get", "methods": ["get"], + "path": "/:index_name" + }, + {"name": "indices.get_settings", "methods": ["get"], + "path": "/:index_name/_settings" + }, + {"name": "indices.open", "methods": ["post"], + "path": "/:index_name/_open" + }, + {"name": "indices.put_settings", "methods": ["put"], + "path": "/:index_name/_settings" + }, + {"name": "indices.stats", "methods": ["get"], + "path": "/:index_name/_stats" + }, + {"name": "indices.delete_by_query", "methods": ["post"], + "path": "/:index_name/_delete_by_query" + }, + {"name": "indices.rollover", "methods": ["post"], + "path": "/:index_name/_rollover" + }, + {"name": "indices.count", "methods": ["get"], + "path": "/:index_name/_count" + }, + {"name": "indices.shard_stores", "methods": ["get"], + "path": "/:index_name/_shard_stores" + }, + {"name": "indices.bulk", "methods": ["post"], + "path": "/:index_name/_bulk" + }, + {"name": "indices.mget", "methods": ["get", "post"], + "path": "/:index_name/_mget" + }, + {"name": "indices.msearch", "methods": ["get", "post"], + "path": "/:index_name/_msearch" + }, + {"name": "indices.msearch_template", "methods": ["get", "post"], + "path": "/:index_name/_msearch/template" + }, + {"name": "indices.mtermvectors", "methods": ["get"], + "path": "/:index_name/_mtermvectors" + }, + {"name": "indices.rank_eval", "methods": ["get"], + "path": "/:index_name/_rank_eval" + }, + {"name": "indices.search", "methods": ["get", "post"], + "path": "/:index_name/_search" + }, + {"name": "indices.search_shards", "methods": ["get", "post"], + "path": "/:index_name/_search_shards" + }, + {"name": "indices.field_caps", "methods":["get", "post"], + "path": "/:index_name/_field_caps" + }, + {"name": "indices.exists_template", "methods":["get"], + "path": "/_template/:name" + }, + {"name": "indices.field_usage_stats", "methods":["get"], + "path": "/:index_name/_field_usage_stats" + }, + {"name": "doc.*", "methods": ["*"], + "path": "/:index_name/:doctype" + }, + {"name": "doc.update", "methods": ["put"], + "path": "/:index_name/:doctype/:doc_id" + }, + {"name": "doc.update", "methods": ["post"], + "path": "/:index_name/_update/:doc_id" + }, + {"name": "doc.create", "methods": ["post"], + "path": "/:index_name/:doctype" + }, + {"name": "doc.create", "methods": ["post", "put"], + "path": "/:index_name/_create/:doc_id" + }, + + {"name": "doc.delete", "methods": ["delete"], + "path": "/:index_name/:doctype/:doc_id" + }, + {"name": "doc.get", "methods": ["get"], + "path": "/:index_name/:doctype/:doc_id" + }, + {"name": "doc.get", "methods": ["get"], + "path": "/:index_name/_source/:doc_id" + }, + {"name": "doc.exists", "methods": ["head"], + "path": "/:index_name/:doctype/:doc_id" + }, + {"name": "doc.exists_source", "methods": ["head"], + "path": "/:index_name/_source/:doc_id" + }, + {"name": "doc.explain", "methods": ["get"], + "path": "/:index_name/_explain/:doc_id" + } + ], + + "ingest": [ + {"name": "ingest.*", "methods": ["*"], + "path": "/_ingest/*" + }, + {"name": "ingest.delete_pipeline", "methods": ["delete"], + "path": "/_ingest/pipeline" + }, + {"name": "ingest.put_pipeline", "methods": ["put"], + "path": "/_ingest/pipeline" + }, + {"name": "ingest.simulate", "methods": ["get", "post"], + "path": "/_ingest/pipeline/_simulate" + }, + {"name": "ingest.put_pipeline", "methods": ["get"], + "path": "/_ingest/pipeline" + }, + {"name": "ingest.processor_grok", "methods": ["get"], + "path": "/_ingest/processor/grok" + } + ], + + "nodes": [ + {"name": "nodes.*", "methods": ["*"], + "path": "/_nodes/*" + }, + {"name": "nodes.info", "methods": ["get"], + "path": "/_nodes" + }, + {"name": "nodes.info", "methods": ["get"], + "path": "/_nodes/:node_id" + }, + {"name": "nodes.stats", "methods": ["get"], + "path": "/_nodes/stats" + }, + {"name": "nodes.reload_secure_settings", "methods": ["post"], + "path": "/_nodes/reload_secure_settings" + }, + {"name": "nodes.usage", "methods": ["get"], + "path": "/_nodes/usage" + }, + {"name": "nodes.hot_threads", "methods": ["get"], + "path": "/_nodes/hot_threads" + } + ], + + "reindex": [ + {"name": "reindex.*", "methods": ["*"], + "path": "/_reindex/*" + }, + {"name": "reindex", "methods": ["post"], + "path": "/_reindex" + }, + {"name": "reindex.rethrottle", "methods": ["post"], + "path": "/_reindex/:rid/_rethrottle" + } + ], + + "render_search_template": [ + {"name": "render_search_template.*", "methods": ["*"], + "path": "/_render/template" + }, + {"name": "render_search_template", "methods": ["post", "get"], + "path": "/_render/template" + }, + {"name": "render_search_template_by_id", "methods": ["post", "get"], + "path": "/_render/template/:tid" + } + ], + "scripts": [ + {"name": "scripts.*", "methods": ["*"], + "path": "/_scripts/:sid" + }, + {"name": "scripts.get", "methods": ["get"], + "path": "/_scripts/:sid" + }, + {"name": "scripts.put", "methods": ["put"], + "path": "/_scripts/:sid" + }, + {"name": "scripts.delete", "methods": ["delete"], + "path": "/_scripts/:sid" + }, + {"name": "scripts.painless_execute", "methods": ["get", "post"], + "path": "_scripts/painless/_execute" + } + ], + + "scroll": [ + {"name": "scroll.*", "methods": ["*"], + "path": "/_search/scroll*" + }, + {"name": "scroll.search", "methods": ["get", "post"], + "path": "/_search/scroll" + }, + {"name": "scroll.delete", "methods": ["delete"], + "path": "/_search/scroll/:scroll_id" + }, + {"name": "scroll.get", "methods": ["get"], + "path": "/_search/scroll/:scroll_id" + }, + {"name": "scroll.create", "methods": ["post"], + "path": "/_search/scroll/:scroll_id" + } + ], + + "snapshot": [ + {"name": "snapshot.*", "methods": ["*"], + "path": "/_snapshot/*" + }, + {"name": "snapshot.get_repository", "methods": ["get"], + "path": "/_snapshot/:repo_name" + }, + {"name": "snapshot.create_repository", "methods": ["post"], + "path": "/_snapshot/:repo_name" + }, + {"name": "snapshot.create", "methods": ["post"], + "path": "/_snapshot/:repo_name/:snapshot_name" + }, + {"name": "snapshot.restore", "methods": ["post"], + "path": "/_snapshot/:repo_name/:snapshot_name/_restore" + }, + {"name": "snapshot.status", "methods": ["get"], + "path": "/_snapshot/_status" + }, + {"name": "snapshot.delete", "methods": ["delete"], + "path": "/_snapshot/:repo_name/:snapshot_name" + }, + {"name": "snapshot.delete_repository", "methods": ["delete"], + "path": "/_snapshot/:repo_name" + }, + {"name": "snapshot.verify_repository", "methods": ["post"], + "path": "/_snapshot/:repo_name/_verify" + }, + {"name": "snapshot.get", "methods": ["get"], + "path": "/_snapshot/:repo_name/:snapshot_name" + } + ], + + "tasks": [ + {"name": "tasks.*", "methods": ["*"], + "path": "/_tasks/*" + }, + {"name": "tasks.list", "methods": ["get"], + "path": "/_tasks" + }, + {"name": "tasks.cancel", "methods": ["post"], + "path": "/_tasks/:task_id/_cancel" + }, + {"name": "tasks.get", "methods": ["get"], + "path": "/_tasks/:task_id" + } + ], + "sql": [ + {"name": "sql.*", "methods": ["*"], "path": "/_sql/*"}, + {"name": "sql.clear", "methods": ["post"], "path": "/_sql/close"}, + {"name": "sql.get_async", "methods": ["get"], "path": "/_sql/async/:search_id"}, + {"name": "sql.delete_async", "methods": ["delete"], "path": "/_sql/async/delete/:search_id"}, + {"name": "sql.get_async_status", "methods": ["get"], "path": "/_sql/async/status/:search_id"}, + {"name": "sql.search", "methods": ["get", "post"], "path": "/_sql"}, + {"name": "sql.search", "methods": ["post"], "path": "/_plugins/_sql"}, + {"name": "sql.translate", "methods": ["get", "post"], "path": "/_sql/translate"} + ] +} \ No newline at end of file diff --git a/modules/security/realm/authc/native/role.go b/modules/security/realm/authc/native/role.go new file mode 100644 index 00000000..b397478f --- /dev/null +++ b/modules/security/realm/authc/native/role.go @@ -0,0 +1,76 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package native + +import ( + "fmt" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "strings" +) + +type Role struct { +} + +func (dal *Role) Get(id string) (rbac.Role, error) { + + r, ok := rbac.BuiltinRoles[id] + if ok { + return r, nil + } + + role := rbac.Role{} + role.ID = id + _, err := orm.Get(&role) + return role, err +} + +func (dal *Role) GetBy(field string, value interface{}) (rbac.Role, error) { + role := rbac.Role{} + err, result := orm.GetBy(field, value, &role) + if result.Total > 0 { + if len(result.Result) > 0 { + bytes := util.MustToJSONBytes(result.Result[0]) + err := util.FromJSONBytes(bytes, &role) + if err != nil { + panic(err) + } + return role, nil + } + } + return role, err +} + +func (dal *Role) Update(role *rbac.Role) error { + return orm.Save(nil, role) +} + +func (dal *Role) Create(role *rbac.Role) (string, error) { + role.ID = util.GetUUID() + return role.ID, orm.Save(nil, role) +} + +func (dal *Role) Delete(id string) error { + role := rbac.Role{} + role.ID = id + return orm.Delete(nil, role) +} + +func (dal *Role) Search(keyword string, from, size int) (orm.Result, error) { + query := orm.Query{} + + queryDSL := `{"query":{"bool":{"must":[%s]}}, "from": %d,"size": %d}` + mustBuilder := &strings.Builder{} + + if keyword != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) + } + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), from, size) + query.RawQuery = []byte(queryDSL) + + err, result := orm.Search(rbac.Role{}, &query) + return result, err +} diff --git a/modules/security/realm/authc/native/user.go b/modules/security/realm/authc/native/user.go new file mode 100644 index 00000000..b05daab9 --- /dev/null +++ b/modules/security/realm/authc/native/user.go @@ -0,0 +1,72 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package native + +import ( + "fmt" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "strings" +) + +type User struct { +} + +func (dal *User) Get(id string) (rbac.User, error) { + user := rbac.User{} + user.ID = id + _, err := orm.Get(&user) + return user, err +} + +func (dal *User) GetBy(field string, value interface{}) (*rbac.User, error) { + user := &rbac.User{} + err, result := orm.GetBy(field, value, rbac.User{}) + if err != nil { + return nil, err + } + if len(result.Result) == 0 { + return nil, nil + } + userBytes, err := util.ToJSONBytes(result.Result[0]) + if err != nil { + return nil, err + } + util.FromJSONBytes(userBytes, &user) + return user, err +} + +func (dal *User) Update(user *rbac.User) error { + + return orm.Update(nil, user) +} + +func (dal *User) Create(user *rbac.User) (string, error) { + user.ID = util.GetUUID() + return user.ID, orm.Save(nil, user) +} + +func (dal *User) Delete(id string) error { + user := rbac.User{} + user.ID = id + return orm.Delete(nil, user) +} + +func (dal *User) Search(keyword string, from, size int) (orm.Result, error) { + query := orm.Query{} + + queryDSL := `{"query":{"bool":{"must":[%s]}}, "from": %d,"size": %d}` + mustBuilder := &strings.Builder{} + + if keyword != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) + } + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), from, size) + query.RawQuery = []byte(queryDSL) + + err, result := orm.Search(rbac.User{}, &query) + return result, err +} diff --git a/modules/security/realm/authc/oauth/api.go b/modules/security/realm/authc/oauth/api.go new file mode 100644 index 00000000..7c145997 --- /dev/null +++ b/modules/security/realm/authc/oauth/api.go @@ -0,0 +1,19 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package oauth + +import ( + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/api" +) + +type APIHandler struct { + api.Handler + rbac.Adapter +} + +const adapterType = "native" + +var apiHandler = APIHandler{Adapter: rbac.GetAdapter(adapterType)} //TODO handle hard coded diff --git a/modules/security/realm/authc/oauth/init.go b/modules/security/realm/authc/oauth/init.go new file mode 100644 index 00000000..5c93f63a --- /dev/null +++ b/modules/security/realm/authc/oauth/init.go @@ -0,0 +1,41 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package oauth + +import ( + "golang.org/x/oauth2" + rbac "infini.sh/console/core/security" + "infini.sh/console/modules/security/config" + "infini.sh/framework/core/api" +) + +var ( + oAuthConfig config.OAuthConfig + defaultOAuthRoles []rbac.UserRole + oauthCfg oauth2.Config +) + +// func New(cfg config.OAuthConfig) *OAuthRealm { +func Init(cfg config.OAuthConfig) { + + //init oauth + if cfg.Enabled { + api.HandleUIMethod(api.GET, "/sso/login/", apiHandler.AuthHandler) + api.HandleUIMethod(api.GET, "/sso/callback/", apiHandler.CallbackHandler) + + oAuthConfig = cfg + oauthCfg = oauth2.Config{ + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + Endpoint: oauth2.Endpoint{ + AuthURL: cfg.AuthorizeUrl, + TokenURL: cfg.TokenUrl, + }, + RedirectURL: cfg.RedirectUrl, + Scopes: cfg.Scopes, + } + } + +} diff --git a/modules/security/realm/authc/oauth/oauth.go b/modules/security/realm/authc/oauth/oauth.go new file mode 100644 index 00000000..b6a35de0 --- /dev/null +++ b/modules/security/realm/authc/oauth/oauth.go @@ -0,0 +1,228 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package oauth + +import ( + "encoding/base64" + log "github.com/cihub/seelog" + "github.com/google/go-github/github" + "golang.org/x/oauth2" + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" + "math/rand" + "net/http" + "strings" + "time" +) + +func (h APIHandler) getDefaultRoles() []rbac.UserRole { + if len(oAuthConfig.DefaultRoles) == 0 { + return nil + } + + if len(defaultOAuthRoles) > 0 { + return defaultOAuthRoles + } + + roles := h.getRolesByRoleIDs(oAuthConfig.DefaultRoles) + if len(roles) > 0 { + defaultOAuthRoles = roles + } + return roles +} + +func (h APIHandler) getRolesByRoleIDs(roles []string) []rbac.UserRole { + out := []rbac.UserRole{} + for _, v := range roles { + role, err := h.Adapter.Role.Get(v) + if err != nil { + if !strings.Contains(err.Error(), "record not found") { + panic(err) + } + + //try name + role, err = h.Adapter.Role.GetBy("name", v) + if err != nil { + continue + } + } + out = append(out, rbac.UserRole{ID: role.ID, Name: role.Name}) + } + return out +} + +const oauthSession string = "oauth-session" + +func (h APIHandler) AuthHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + b := make([]byte, 16) + rand.Read(b) + + state := base64.URLEncoding.EncodeToString(b) + + session, err := api.GetSessionStore(r, oauthSession) + session.Values["state"] = state + session.Values["redirect_url"] = h.Get(r, "redirect_url", "") + err = session.Save(r, w) + if err != nil { + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + url := oauthCfg.AuthCodeURL(state) + http.Redirect(w, r, url, 302) +} + +func joinError(url string, err error) string { + if err != nil { + return url + "?err=" + util.UrlEncode(err.Error()) + } + return url +} + +func (h APIHandler) CallbackHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + + session, err := api.GetSessionStore(r, oauthSession) + if err != nil { + log.Error(w, "failed to sso, aborted") + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + if r.URL.Query().Get("state") != session.Values["state"] { + log.Error("failed to sso, no state match; possible csrf OR cookies not enabled") + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + tkn, err := oauthCfg.Exchange(oauth2.NoContext, r.URL.Query().Get("code")) + if err != nil { + log.Error("failed to sso, there was an issue getting your token") + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + if !tkn.Valid() { + log.Error("failed to sso, retreived invalid token") + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + //only for github, TODO + client := github.NewClient(oauthCfg.Client(oauth2.NoContext, tkn)) + + user, res, err := client.Users.Get(oauth2.NoContext, "") + if err != nil { + if res != nil { + log.Error("failed to sso, error getting name:", err, res.String()) + } + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + if user != nil { + roles := []rbac.UserRole{} + + var id, name, email string + if user.Login != nil && *user.Login != "" { + id = *user.Login + } + if user.Name != nil && *user.Name != "" { + name = *user.Name + } + if user.Email != nil && *user.Email != "" { + email = *user.Email + } + + if id == "" { + log.Error("failed to sso, user id can't be nil") + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + if name == "" { + name = id + } + + //get by roleMapping + roles = h.getRoleMapping(user) + if len(roles) > 0 { + u := &rbac.User{ + AuthProvider: "github", + Username: id, + Nickname: name, + Email: email, + Roles: roles, + } + + u.ID = id + + //generate access token + data, err := rbac.GenerateAccessToken(u) + if err != nil { + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) + return + } + + token := rbac.Token{ExpireIn: time.Now().Unix() + 86400} + rbac.SetUserToken(u.ID, token) + + //data["status"] = "ok" + url := oAuthConfig.SuccessPage + "?payload=" + util.UrlEncode(util.MustToJSON(data)) + http.Redirect(w, r, url, 302) + return + } + } + http.Redirect(w, r, joinError(oAuthConfig.FailedPage, err), 302) +} + +func (h APIHandler) getRoleMapping(user *github.User) []rbac.UserRole { + roles := []rbac.UserRole{} + + if user != nil { + if len(oAuthConfig.RoleMapping) > 0 { + r, ok := oAuthConfig.RoleMapping[*user.Login] + if ok { + roles = h.getRolesByRoleIDs(r) + } + } + } + + if len(roles) == 0 { + return h.getDefaultRoles() + } + return roles +} + +const providerName = "oauth" + +type OAuthRealm struct { + // Implement any required fields +} + +//func (r *OAuthRealm) GetType() string{ +// return providerName +//} + +//func (r *OAuthRealm) Authenticate(username, password string) (bool, *rbac.User, error) { +// +// //if user == nil { +// // return false,nil, fmt.Errorf("user account [%s] not found", username) +// //} +// +// return false,nil, err +//} +// +//func (r *OAuthRealm) Authorize(user *rbac.User) (bool, error) { +// var _, privilege = user.GetPermissions() +// +// if len(privilege) == 0 { +// log.Error("no privilege assigned to user:", user) +// return false, errors.New("no privilege assigned to this user:" + user.Name) +// } +// +// return true,nil +//} diff --git a/modules/security/realm/authc/saml/main.go b/modules/security/realm/authc/saml/main.go new file mode 100644 index 00000000..9b5f5cce --- /dev/null +++ b/modules/security/realm/authc/saml/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "github.com/crewjam/saml" + + "github.com/crewjam/saml/samlsp" +) + +var metdataurl = "https://sso.infini.ltd/metadata" //Metadata of the IDP +var sessioncert = "./sessioncert" //Key pair used for creating a signed session +var sessionkey = "./sessionkey" +var serverkey = "./serverkey" //Server TLS +var servercert = "./servercert" +var serverurl = "https://localhost" // base url of this service +var entityId = serverurl //Entity ID uniquely identifies your service for IDP (does not have to be server url) +var listenAddr = "0.0.0.0:443" + +func hello(w http.ResponseWriter, r *http.Request) { + s := samlsp.SessionFromContext(r.Context()) + if s == nil { + return + } + sa, ok := s.(samlsp.SessionWithAttributes) + if !ok { + return + } + + fmt.Fprintf(w, "Token contents, %+v!", sa.GetAttributes()) +} + +func main() { + keyPair, err := tls.LoadX509KeyPair(sessioncert, sessionkey) + panicIfError(err) + keyPair.Leaf, err = x509.ParseCertificate(keyPair.Certificate[0]) + panicIfError(err) + //idpMetadataURL, err := url.Parse(metdataurl) + + panicIfError(err) + rootURL, err := url.Parse(serverurl) + panicIfError(err) + samlSP, _ := samlsp.New(samlsp.Options{ + URL: *rootURL, + Key: keyPair.PrivateKey.(*rsa.PrivateKey), + Certificate: keyPair.Leaf, + IDPMetadata: &saml.EntityDescriptor{ + //EntityID: + }, // you can also have Metadata XML instead of URL + EntityID: entityId, + }) + app := http.HandlerFunc(hello) + http.Handle("/hello", samlSP.RequireAccount(app)) + http.Handle("/saml/", samlSP) + panicIfError(http.ListenAndServeTLS(listenAddr, servercert, serverkey, nil)) +} +func panicIfError(err error) { + if err != nil { + panic(err) + } +} diff --git a/modules/security/realm/authz/authz.go b/modules/security/realm/authz/authz.go new file mode 100644 index 00000000..5e3cea5f --- /dev/null +++ b/modules/security/realm/authz/authz.go @@ -0,0 +1,10 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package authz + +func Authorize() (map[string]interface{}, error) { + + return nil, nil +} diff --git a/modules/security/realm/realm.go b/modules/security/realm/realm.go new file mode 100644 index 00000000..05e5bdd2 --- /dev/null +++ b/modules/security/realm/realm.go @@ -0,0 +1,96 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package realm + +import ( + log "github.com/cihub/seelog" + rbac "infini.sh/console/core/security" + "infini.sh/console/modules/security/config" + ldap2 "infini.sh/console/modules/security/realm/authc/ldap" + "infini.sh/console/modules/security/realm/authc/native" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +var realms = []rbac.SecurityRealm{} + +func Init(config *config.Config) { + + if !config.Enabled { + return + } + + if config.Authentication.Realms.Native.Enabled { + native.Init() + nativeRealm := native.NativeRealm{} + realms = append(realms, &nativeRealm) //TODO sort by order + } + + //if len(config.Authentication.Realms.OAuth) > 0 { + // for _, v := range config.Authentication.Realms.OAuth { + // { + // realm:=oauth.New(v) + // realms=append(realms,realm) //TODO sort by order + // } + // } + //} + + if global.Env().IsDebug { + log.Tracef("config: %v", util.MustToJSON(config)) + } + + if len(config.Authentication.Realms.LDAP) > 0 { + for _, v := range config.Authentication.Realms.LDAP { + if v.Enabled { + realm := ldap2.New(v) + realms = append(realms, realm) //TODO sort by order + } + } + } +} + +func Authenticate(username, password string) (bool, *rbac.User, error) { + + for i, realm := range realms { + ok, user, err := realm.Authenticate(username, password) + log.Debugf("authenticate result: %v, user: %v, err: %v, realm: %v", ok, user, err, i) + if ok && user != nil && err == nil { + return true, user, nil + } + } + if global.Env().IsDebug { + log.Errorf("failed to authenticate user: %v", username) + } + return false, nil, errors.Errorf("failed to authenticate user: %v", username) +} + +func Authorize(user *rbac.User) (bool, error) { + + for i, realm := range realms { + //skip if not the same auth provider, TODO: support cross-provider authorization + if user.AuthProvider != realm.GetType() { + continue + } + + ok, err := realm.Authorize(user) + log.Debugf("authorize result: %v, user: %v, err: %v, realm: %v", ok, user, err, i) + if ok && err == nil { + //return on any success, TODO, maybe merge all roles and privileges from all realms + return true, nil + } + } + + roles, privilege := user.GetPermissions() + if len(roles) == 0 && len(privilege) == 0 { + if global.Env().IsDebug { + log.Errorf("failed to authorize user: %v", user.Username) + } + return false, errors.New("no roles or privileges") + } + + return false, errors.Errorf("failed to authorize user: %v", user.Username) + +} diff --git a/plugin/api/alerting/api.go b/plugin/api/alerting/api.go index 87d80eda..9eade64a 100644 --- a/plugin/api/alerting/api.go +++ b/plugin/api/alerting/api.go @@ -5,17 +5,17 @@ package alerting import ( + "infini.sh/console/core" + "infini.sh/console/core/security/enum" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" ) - type AlertAPI struct { - api.Handler + core.Handler } func (alert *AlertAPI) Init() { - api.HandleAPIMethod(api.GET, "/alerting/rule/:rule_id", alert.RequirePermission(alert.getRule,enum.PermissionAlertRuleRead)) + api.HandleAPIMethod(api.GET, "/alerting/rule/:rule_id", alert.RequirePermission(alert.getRule, enum.PermissionAlertRuleRead)) api.HandleAPIMethod(api.POST, "/alerting/rule", alert.RequirePermission(alert.createRule, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.POST, "/alerting/rule/test", alert.RequireLogin(alert.sendTestMessage)) api.HandleAPIMethod(api.DELETE, "/alerting/rule/:rule_id", alert.RequirePermission(alert.deleteRule, enum.PermissionAlertRuleWrite)) @@ -47,13 +47,11 @@ func (alert *AlertAPI) Init() { api.HandleAPIMethod(api.GET, "/alerting/message/_search", alert.RequirePermission(alert.searchAlertMessage, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.POST, "/alerting/message/_ignore", alert.RequirePermission(alert.ignoreAlertMessage, enum.PermissionAlertMessageWrite)) - api.HandleAPIMethod(api.GET, "/alerting/message/_stats", alert.RequirePermission(alert.getAlertMessageStats, enum.PermissionAlertMessageRead)) + api.HandleAPIMethod(api.GET, "/alerting/message/_stats", alert.RequirePermission(alert.getAlertMessageStats, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.GET, "/alerting/message/:message_id", alert.RequirePermission(alert.getAlertMessage, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.GET, "/alerting/message/:message_id/notification", alert.getMessageNotificationInfo) - //just for test //api.HandleAPIMethod(api.GET, "/alerting/rule/test", alert.testRule) } - diff --git a/plugin/api/alerting/rule.go b/plugin/api/alerting/rule.go index b2872248..df39a1be 100644 --- a/plugin/api/alerting/rule.go +++ b/plugin/api/alerting/rule.go @@ -9,11 +9,12 @@ import ( "fmt" log "github.com/cihub/seelog" "github.com/r3labs/diff/v2" + "infini.sh/console/core/security" "infini.sh/console/model/alerting" "infini.sh/console/model/insight" + "infini.sh/console/modules/elastic/api" alerting2 "infini.sh/console/service/alerting" _ "infini.sh/console/service/alerting/elasticsearch" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" @@ -24,7 +25,6 @@ import ( "infini.sh/framework/core/task" "infini.sh/framework/core/util" elastic2 "infini.sh/framework/modules/elastic" - "infini.sh/framework/modules/elastic/api" "infini.sh/framework/modules/elastic/common" "net/http" "strings" @@ -32,7 +32,7 @@ import ( ) func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - rules := []alerting.Rule{} + rules := []alerting.Rule{} err := alertAPI.DecodeJSON(req, &rules) if err != nil { log.Error(err) @@ -41,7 +41,7 @@ func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, p }, http.StatusInternalServerError) return } - user, err := rbac.FromUserContext(req.Context()) + user, err := security.FromUserContext(req.Context()) if err != nil { log.Error(err) alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -69,7 +69,7 @@ func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, p ids = append(ids, rule.ID) rule.Created = time.Now() rule.Updated = time.Now() - if rule.Schedule.Interval == ""{ + if rule.Schedule.Interval == "" { rule.Schedule.Interval = "1m" } //filter empty metric group @@ -93,19 +93,19 @@ func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, p }, http.StatusInternalServerError) return } - saveAlertActivity("alerting_rule_change", "create", util.MapStr{ - "cluster_id": rule.Resource.ID, - "rule_id": rule.ID, + saveAlertActivity("alerting_rule_change", "create", util.MapStr{ + "cluster_id": rule.Resource.ID, + "rule_id": rule.ID, "cluster_name": rule.Resource.Name, - "rule_name": rule.Name, - },nil, &rule) + "rule_name": rule.Name, + }, nil, &rule) eng := alerting2.GetEngine(rule.Resource.Type) if rule.Enabled { ruleTask := task.ScheduleTask{ - ID: rule.ID, - Interval: rule.Schedule.Interval, + ID: rule.ID, + Interval: rule.Schedule.Interval, Description: rule.Metrics.Expression, - Task: eng.GenerateTask(rule), + Task: eng.GenerateTask(rule), } task.RegisterScheduleTask(ruleTask) task.StartTask(ruleTask.ID) @@ -115,7 +115,7 @@ func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, p alertAPI.WriteJSON(w, util.MapStr{ "result": "created", - "ids": ids, + "ids": ids, }, http.StatusOK) } func (alertAPI *AlertAPI) getRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -125,7 +125,7 @@ func (alertAPI *AlertAPI) getRule(w http.ResponseWriter, req *http.Request, ps h _, err := orm.Get(&obj) if err != nil { - if errors.Is(err, elastic2.ErrNotFound){ + if errors.Is(err, elastic2.ErrNotFound) { alertAPI.WriteJSON(w, util.MapStr{ "_id": id, "found": false, @@ -166,7 +166,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request exists, err := orm.Get(&obj) if !exists || err != nil { - if errors.Is(err, elastic2.ErrNotFound){ + if errors.Is(err, elastic2.ErrNotFound) { alertAPI.WriteJSON(w, util.MapStr{ "_id": id, "found": false, @@ -182,7 +182,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request expression, _ := cond.GenerateConditionExpression() obj.Conditions.Items[i].Expression = strings.ReplaceAll(expression, "result", metricExpression) } - alertNumbers, err := alertAPI.getRuleAlertMessageNumbers([]string{obj.ID}) + alertNumbers, err := alertAPI.getRuleAlertMessageNumbers([]string{obj.ID}) if err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -222,12 +222,12 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request } q := &orm.Query{ WildcardIndex: true, - RawQuery: util.MustToJSONBytes(queryDSL), + RawQuery: util.MustToJSONBytes(queryDSL), } err, result := orm.Search(alerting.AlertMessage{}, q) if err != nil { log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } var state interface{} = "N/A" @@ -259,7 +259,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request err, result = orm.Search(alerting.Channel{}, q) if err != nil { log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } chm := map[string]alerting.Channel{} @@ -296,25 +296,25 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request } detailObj := util.MapStr{ - "rule_name": obj.Name, - "resource_name": obj.Resource.Name, - "resource_id": obj.Resource.ID, - "resource_objects": obj.Resource.Objects, - "resource_time_field": obj.Resource.TimeField, - "resource_raw_filter": obj.Resource.RawFilter, - "metrics": obj.Metrics, - "bucket_size": obj.Metrics.BucketSize, //统计周期 - "updated": obj.Updated, - "conditions": obj.Conditions, - "message_count": alertNumbers[obj.ID], //所有关联告警消息数（包括已恢复的） - "state": state, - "enabled": obj.Enabled, - "created": obj.Created, - "creator": obj.Creator, - "tags": obj.Tags, - "alerting_message": alertingMessageItem, - "expression": obj.Metrics.Expression, - "notification_config": obj.NotificationConfig, + "rule_name": obj.Name, + "resource_name": obj.Resource.Name, + "resource_id": obj.Resource.ID, + "resource_objects": obj.Resource.Objects, + "resource_time_field": obj.Resource.TimeField, + "resource_raw_filter": obj.Resource.RawFilter, + "metrics": obj.Metrics, + "bucket_size": obj.Metrics.BucketSize, //统计周期 + "updated": obj.Updated, + "conditions": obj.Conditions, + "message_count": alertNumbers[obj.ID], //所有关联告警消息数（包括已恢复的） + "state": state, + "enabled": obj.Enabled, + "created": obj.Created, + "creator": obj.Creator, + "tags": obj.Tags, + "alerting_message": alertingMessageItem, + "expression": obj.Metrics.Expression, + "notification_config": obj.NotificationConfig, "recovery_notification_config": obj.RecoveryNotificationConfig, } @@ -322,7 +322,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request } -func saveActivity(activityInfo *event.Activity){ +func saveActivity(activityInfo *event.Activity) { queueConfig := queue.GetOrInitConfig("platform##activities") if queueConfig.Labels == nil { queueConfig.ReplaceLabels(util.MapStr{ @@ -336,7 +336,7 @@ func saveActivity(activityInfo *event.Activity){ Timestamp: time.Now(), Metadata: event.EventMetadata{ Category: "elasticsearch", - Name: "activity", + Name: "activity", }, Fields: util.MapStr{ "activity": activityInfo, @@ -346,16 +346,16 @@ func saveActivity(activityInfo *event.Activity){ } } -func saveAlertActivity(name, typ string, labels map[string]interface{}, changelog diff.Changelog, oldState interface{}){ +func saveAlertActivity(name, typ string, labels map[string]interface{}, changelog diff.Changelog, oldState interface{}) { activityInfo := &event.Activity{ - ID: util.GetUUID(), + ID: util.GetUUID(), Timestamp: time.Now(), Metadata: event.ActivityMetadata{ Category: "elasticsearch", - Group: "platform", - Name: name, - Type: typ, - Labels: labels, + Group: "platform", + Name: name, + Type: typ, + Labels: labels, }, Changelog: changelog, Fields: util.MapStr{ @@ -382,8 +382,7 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p id = oldRule.ID create := oldRule.Created - rule := &alerting.Rule{ - } + rule := &alerting.Rule{} err = alertAPI.DecodeJSON(req, rule) if err != nil { alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -421,12 +420,12 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p log.Error(err) return } - saveAlertActivity("alerting_rule_change", "update", util.MapStr{ - "cluster_id": rule.Resource.ID, - "rule_id": rule.ID, - "rule_name": rule.Name, + saveAlertActivity("alerting_rule_change", "update", util.MapStr{ + "cluster_id": rule.Resource.ID, + "rule_id": rule.ID, + "rule_name": rule.Name, "cluster_name": rule.Resource.Name, - },changeLog, oldRule) + }, changeLog, oldRule) if rule.Enabled { exists, err = checkResourceExists(rule) @@ -449,7 +448,7 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p } task.RegisterScheduleTask(ruleTask) task.StartTask(ruleTask.ID) - }else{ + } else { task.DeleteTask(id) } @@ -459,10 +458,10 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p }, 200) } -func clearKV(ruleID string){ +func clearKV(ruleID string) { _ = kv.DeleteKey(alerting2.KVLastNotificationTime, []byte(ruleID)) _ = kv.DeleteKey(alerting2.KVLastEscalationTime, []byte(ruleID)) - _ = kv.DeleteKey(alerting2.KVLastMessageState,[]byte(ruleID)) + _ = kv.DeleteKey(alerting2.KVLastMessageState, []byte(ruleID)) } func (alertAPI *AlertAPI) deleteRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -487,12 +486,12 @@ func (alertAPI *AlertAPI) deleteRule(w http.ResponseWriter, req *http.Request, p log.Error(err) return } - saveAlertActivity("alerting_rule_change", "delete", util.MapStr{ - "cluster_id": obj.Resource.ID, - "rule_id": obj.ID, + saveAlertActivity("alerting_rule_change", "delete", util.MapStr{ + "cluster_id": obj.Resource.ID, + "rule_id": obj.ID, "cluster_name": obj.Resource.Name, - "rule_name": obj.Name, - },nil, &obj) + "rule_name": obj.Name, + }, nil, &obj) task.DeleteTask(obj.ID) clearKV(obj.ID) @@ -541,12 +540,12 @@ func (alertAPI *AlertAPI) batchDeleteRule(w http.ResponseWriter, req *http.Reque } var newIDs []string for _, rule := range rules { - saveAlertActivity("alerting_rule_change", "delete", util.MapStr{ - "cluster_id": rule.Resource.ID, - "rule_id": rule.ID, + saveAlertActivity("alerting_rule_change", "delete", util.MapStr{ + "cluster_id": rule.Resource.ID, + "rule_id": rule.ID, "cluster_name": rule.Resource.Name, - "rule_name": rule.Name, - },nil, &rule) + "rule_name": rule.Name, + }, nil, &rule) task.DeleteTask(rule.ID) clearKV(rule.ID) newIDs = append(newIDs, rule.ID) @@ -587,16 +586,14 @@ func (alertAPI *AlertAPI) batchDeleteRule(w http.ResponseWriter, req *http.Reque func (alertAPI *AlertAPI) searchRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var ( keyword = alertAPI.GetParameterOrDefault(req, "keyword", "") - from = alertAPI.GetIntOrDefault(req, "from", 0) - size = alertAPI.GetIntOrDefault(req, "size", 20) + from = alertAPI.GetIntOrDefault(req, "from", 0) + size = alertAPI.GetIntOrDefault(req, "size", 20) ) - mustQuery := []util.MapStr{ - } + mustQuery := []util.MapStr{} clusterFilter, hasAllPrivilege := alertAPI.GetClusterFilter(req, "resource.resource_id") if !hasAllPrivilege && clusterFilter == nil { - alertAPI.WriteJSON(w, elastic.SearchResponse{ - }, http.StatusOK) + alertAPI.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) return } if !hasAllPrivilege { @@ -663,7 +660,7 @@ func (alertAPI *AlertAPI) searchRule(w http.ResponseWriter, req *http.Request, p alertAPI.WriteJSON(w, searchRes, http.StatusOK) } -func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[string]interface{},error) { +func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) (map[string]interface{}, error) { esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ @@ -693,7 +690,7 @@ func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) ( map[str }, } - searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.AlertMessage{}), util.MustToJSONBytes(queryDsl) ) + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.AlertMessage{}), util.MustToJSONBytes(queryDsl)) if err != nil { return nil, err } @@ -736,10 +733,10 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque }, } - searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl) ) + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl)) if err != nil { log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } if len(searchRes.Hits.Hits) == 0 { @@ -751,7 +748,7 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque for _, hit := range searchRes.Hits.Hits { if ruleID, ok := hit.Source["rule_id"].(string); ok { latestAlertInfos[ruleID] = util.MapStr{ - "status": hit.Source["state"], + "status": hit.Source["state"], } } } @@ -786,10 +783,10 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque }, }, } - searchRes, err = esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl) ) + searchRes, err = esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl)) if err != nil { log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } for _, hit := range searchRes.Hits.Hits { @@ -862,7 +859,7 @@ func disableRule(obj *alerting.Rule) { func (alertAPI *AlertAPI) sendTestMessage(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { typ := alertAPI.GetParameterOrDefault(req, "type", "notification") - rule := alerting.Rule{} + rule := alerting.Rule{} err := alertAPI.DecodeJSON(req, &rule) if err != nil { log.Error(err) @@ -875,7 +872,7 @@ func (alertAPI *AlertAPI) sendTestMessage(w http.ResponseWriter, req *http.Reque rule.ID = util.GetUUID() } eng := alerting2.GetEngine(rule.Resource.Type) - actionResults, err := eng.Test(&rule, typ) + actionResults, err := eng.Test(&rule, typ) if err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -942,14 +939,13 @@ func (alertAPI *AlertAPI) getPreviewMetricData(w http.ResponseWriter, req *http. bkSize = duration.Seconds() } - bucketSize, min, max, err := api.GetMetricRangeAndBucketSize(minStr, maxStr, int(bkSize), 15) filterParam := &alerting.FilterParam{ - Start: min, - End: max, + Start: min, + End: max, BucketSize: fmt.Sprintf("%ds", bucketSize), } - metricItem, _, err := getRuleMetricData(rule, filterParam) + metricItem, _, err := getRuleMetricData(rule, filterParam) if err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -963,7 +959,7 @@ func (alertAPI *AlertAPI) getPreviewMetricData(w http.ResponseWriter, req *http. } func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - rule := &alerting.Rule{ + rule := &alerting.Rule{ ID: ps.ByName("rule_id"), } exists, err := orm.Get(rule) @@ -980,11 +976,11 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request ) bucketSize, min, max, err := api.GetMetricRangeAndBucketSize(minStr, maxStr, 60, 15) filterParam := &alerting.FilterParam{ - Start: min, - End: max, + Start: min, + End: max, BucketSize: fmt.Sprintf("%ds", bucketSize), } - metricItem, queryResult, err := getRuleMetricData(rule, filterParam) + metricItem, queryResult, err := getRuleMetricData(rule, filterParam) if err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -992,21 +988,21 @@ func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request }, http.StatusInternalServerError) return } - resBody := util.MapStr{ - "metric": metricItem, + resBody := util.MapStr{ + "metric": metricItem, "bucket_label": rule.Metrics.BucketLabel, } if alertAPI.GetParameter(req, "debug") == "1" { resBody["query"] = queryResult.Query } - alertAPI.WriteJSON(w,resBody, http.StatusOK) + alertAPI.WriteJSON(w, resBody, http.StatusOK) } -func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.AlertMetricItem, *alerting.QueryResult, error) { +func getRuleMetricData(rule *alerting.Rule, filterParam *alerting.FilterParam) (*alerting.AlertMetricItem, *alerting.QueryResult, error) { eng := alerting2.GetEngine(rule.Resource.Type) metricData, queryResult, err := eng.GetTargetMetricData(rule, true, filterParam) if err != nil { - return nil,queryResult, err + return nil, queryResult, err } formatType := "num" @@ -1102,7 +1098,7 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) } metricItem.BucketGroups = append(metricItem.BucketGroups, md.GroupValues) metricItem.Lines = append(metricItem.Lines, &common.MetricLine{ - Data: targetData, + Data: targetData, BucketSize: filterParam.BucketSize, Metric: common.MetricSummary{ Label: label, @@ -1112,7 +1108,7 @@ func getRuleMetricData( rule *alerting.Rule, filterParam *alerting.FilterParam) }, }) } - return &metricItem,queryResult, nil + return &metricItem, queryResult, nil } func (alertAPI *AlertAPI) batchEnableRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -1213,7 +1209,7 @@ func (alertAPI *AlertAPI) batchDisableRule(w http.ResponseWriter, req *http.Requ func (alertAPI *AlertAPI) searchFieldValues(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var keyword = alertAPI.GetParameterOrDefault(req, "keyword", "") var field = alertAPI.GetParameterOrDefault(req, "field", "category") - items , err := searchListItems(field, keyword, 20) + items, err := searchListItems(field, keyword, 20) if err != nil { log.Error(err) alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -1222,7 +1218,7 @@ func (alertAPI *AlertAPI) searchFieldValues(w http.ResponseWriter, req *http.Req alertAPI.WriteJSON(w, items, http.StatusOK) } -func searchListItems(field, keyword string, size int) ([]string, error){ +func searchListItems(field, keyword string, size int) ([]string, error) { query := util.MapStr{ "size": 0, "aggs": util.MapStr{ @@ -1234,8 +1230,8 @@ func searchListItems(field, keyword string, size int) ([]string, error){ }, }, } - if v := strings.TrimSpace(keyword); v != ""{ - query["query"]= util.MapStr{ + if v := strings.TrimSpace(keyword); v != "" { + query["query"] = util.MapStr{ "query_string": util.MapStr{ "default_field": field, "query": fmt.Sprintf("*%s*", v), @@ -1257,7 +1253,7 @@ func searchListItems(field, keyword string, size int) ([]string, error){ items := []string{} for _, bk := range searchRes.Aggregations["items"].Buckets { if v, ok := bk["key"].(string); ok { - if strings.Contains(v, keyword){ + if strings.Contains(v, keyword) { items = append(items, v) } } @@ -1265,7 +1261,7 @@ func searchListItems(field, keyword string, size int) ([]string, error){ return items, nil } -func getRulesByID(ruleIDs []string) ([]alerting.Rule, error){ +func getRulesByID(ruleIDs []string) ([]alerting.Rule, error) { if len(ruleIDs) == 0 { return nil, nil } @@ -1292,4 +1288,4 @@ func getRulesByID(ruleIDs []string) ([]alerting.Rule, error){ rules = append(rules, rule) } return rules, nil -} \ No newline at end of file +} diff --git a/plugin/api/data/api.go b/plugin/api/data/api.go index c8f6d444..57f9a74f 100644 --- a/plugin/api/data/api.go +++ b/plugin/api/data/api.go @@ -5,12 +5,13 @@ package data import ( + "infini.sh/console/core" + "infini.sh/console/core/security/enum" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" ) type DataAPI struct { - api.Handler + core.Handler } func InitAPI() { @@ -18,4 +19,4 @@ func InitAPI() { api.HandleAPIMethod(api.POST, "/data/export", dataApi.RequirePermission(dataApi.exportData, enum.PermissionAlertChannelRead, enum.PermissionAlertRuleRead)) api.HandleAPIMethod(api.POST, "/data/import", dataApi.RequirePermission(dataApi.importData, enum.PermissionAlertChannelWrite, enum.PermissionAlertRuleWrite)) -} \ No newline at end of file +} diff --git a/plugin/api/email/api.go b/plugin/api/email/api.go index db1f8855..e9fcb32f 100644 --- a/plugin/api/email/api.go +++ b/plugin/api/email/api.go @@ -17,6 +17,7 @@ import ( type EmailAPI struct { api.Handler } + func InitAPI() { email := EmailAPI{} api.HandleAPIMethod(api.POST, "/email/server/_test", email.testEmailServer) @@ -72,4 +73,4 @@ func InitEmailServer() error { return common.RefreshEmailServer() } return nil -} \ No newline at end of file +} diff --git a/plugin/api/index_management/elasticsearch.go b/plugin/api/index_management/elasticsearch.go index ab792a9d..e6122b49 100644 --- a/plugin/api/index_management/elasticsearch.go +++ b/plugin/api/index_management/elasticsearch.go @@ -4,9 +4,9 @@ import ( "fmt" log "github.com/cihub/seelog" "infini.sh/console/common" + "infini.sh/console/core/security" "infini.sh/console/model" "infini.sh/console/service" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" @@ -51,7 +51,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req queryDsl["query"] = clusterFilter } - user, auditLogErr := rbac.FromUserContext(req.Context()) + user, auditLogErr := security.FromUserContext(req.Context()) if auditLogErr == nil && handler.GetHeader(req, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(user.Username). WithLogTypeAccess().WithResourceTypeClusterManagement(). diff --git a/plugin/api/index_management/index.go b/plugin/api/index_management/index.go index 6536db42..e3fcd7b1 100644 --- a/plugin/api/index_management/index.go +++ b/plugin/api/index_management/index.go @@ -1,6 +1,7 @@ package index_management import ( + "infini.sh/console/core" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" "net/http" @@ -10,7 +11,6 @@ import ( "infini.sh/console/config" model2 "infini.sh/console/model" - "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -18,7 +18,7 @@ import ( type APIHandler struct { Config *config.AppConfig - api.Handler + core.Handler } func (handler APIHandler) GetDictListAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -117,4 +117,4 @@ func (handler APIHandler) UpdateDictItemAction(w http.ResponseWriter, req *http. resp["payload"] = dict handler.WriteJSON(w, resp, http.StatusOK) -} \ No newline at end of file +} diff --git a/plugin/api/index_management/indices.go b/plugin/api/index_management/indices.go index 54d923eb..2182000a 100644 --- a/plugin/api/index_management/indices.go +++ b/plugin/api/index_management/indices.go @@ -3,9 +3,9 @@ package index_management import ( log "github.com/cihub/seelog" "infini.sh/console/common" + "infini.sh/console/core/security" "infini.sh/console/model" "infini.sh/console/service" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/radix" @@ -43,7 +43,7 @@ func (handler APIHandler) HandleGetMappingsAction(w http.ResponseWriter, req *ht func (handler APIHandler) HandleCatIndicesAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { targetClusterID := ps.ByName("id") - user, auditLogErr := rbac.FromUserContext(req.Context()) + user, auditLogErr := security.FromUserContext(req.Context()) if auditLogErr == nil && handler.GetHeader(req, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(user.Username). WithLogTypeAccess().WithResourceTypeClusterManagement(). @@ -139,7 +139,7 @@ func (handler APIHandler) HandleCreateIndexAction(w http.ResponseWriter, req *ht targetClusterID := ps.ByName("id") client := elastic.GetClient(targetClusterID) indexName := ps.ByName("index") - claims, auditLogErr := rbac.ValidateLogin(req.Header.Get("Authorization")) + claims, auditLogErr := security.ValidateLogin(req.Header.Get("Authorization")) if auditLogErr == nil && handler.GetHeader(req, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(claims.Username). WithLogTypeOperation().WithResourceTypeClusterManagement(). diff --git a/plugin/api/init.go b/plugin/api/init.go index 09fec3e3..af2a9e4b 100644 --- a/plugin/api/init.go +++ b/plugin/api/init.go @@ -2,6 +2,7 @@ package api import ( "infini.sh/console/config" + "infini.sh/console/core/security/enum" "infini.sh/console/plugin/api/alerting" "infini.sh/console/plugin/api/data" "infini.sh/console/plugin/api/email" @@ -12,7 +13,6 @@ import ( "infini.sh/console/plugin/api/notification" "infini.sh/console/plugin/api/platform" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" "path" ) diff --git a/plugin/api/insight/api.go b/plugin/api/insight/api.go index 27b232be..3ea59235 100644 --- a/plugin/api/insight/api.go +++ b/plugin/api/insight/api.go @@ -4,10 +4,13 @@ package insight -import "infini.sh/framework/core/api" +import ( + "infini.sh/console/core" + "infini.sh/framework/core/api" +) type InsightAPI struct { - api.Handler + core.Handler } func InitAPI() { diff --git a/plugin/api/insight/metadata.go b/plugin/api/insight/metadata.go index c36afb07..b26f80f5 100644 --- a/plugin/api/insight/metadata.go +++ b/plugin/api/insight/metadata.go @@ -40,7 +40,7 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, }, http.StatusInternalServerError) return } - if reqBody.IndexPattern != "" && !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern){ + if reqBody.IndexPattern != "" && !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern) { h.WriteError(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } @@ -72,7 +72,7 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, for fieldName := range fieldsMeta.Dates { timeFields = append(timeFields, fieldName) } - }else{ + } else { timeFields = []string{reqBody.TimeField} } @@ -80,10 +80,10 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, for _, tfield := range timeFields { aggs["maxTime_"+tfield] = util.MapStr{ - "max": util.MapStr{ "field": tfield }, + "max": util.MapStr{"field": tfield}, } aggs["minTime_"+tfield] = util.MapStr{ - "min": util.MapStr{ "field": tfield }, + "min": util.MapStr{"field": tfield}, } } query := util.MapStr{ @@ -108,7 +108,7 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, } tfieldsM := map[string]util.MapStr{} for ak, av := range searchRes.Aggregations { - if strings.HasPrefix(ak,"maxTime_") { + if strings.HasPrefix(ak, "maxTime_") { tfield := ak[8:] if _, ok := tfieldsM[tfield]; !ok { tfieldsM[tfield] = util.MapStr{} @@ -116,7 +116,7 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, tfieldsM[tfield]["max"] = av.Value continue } - if strings.HasPrefix(ak,"minTime_") { + if strings.HasPrefix(ak, "minTime_") { tfield := ak[8:] if _, ok := tfieldsM[tfield]; !ok { tfieldsM[tfield] = util.MapStr{} @@ -129,13 +129,13 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, h.WriteJSON(w, result, http.StatusOK) } -func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ +func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") reqBody := struct { - IndexPattern string `json:"index_pattern"` - ViewID string `json:"view_id"` - TimeField string `json:"time_field"` - Filter interface{} `json:"filter"` + IndexPattern string `json:"index_pattern"` + ViewID string `json:"view_id"` + TimeField string `json:"time_field"` + Filter interface{} `json:"filter"` }{} err := h.DecodeJSON(req, &reqBody) if err != nil { @@ -143,7 +143,7 @@ func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - if reqBody.IndexPattern != "" && !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern){ + if reqBody.IndexPattern != "" && !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern) { h.WriteError(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } @@ -154,7 +154,7 @@ func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, } exists, err := orm.Get(&view) if err != nil || !exists { - h.WriteError(w, err.Error(), http.StatusNotFound) + h.WriteError(w, err.Error(), http.StatusNotFound) return } reqBody.IndexPattern = view.Title @@ -170,13 +170,13 @@ func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, fieldsMeta, err := getMetadataByIndexPattern(clusterID, reqBody.IndexPattern, reqBody.TimeField, reqBody.Filter, fieldsFormat) if err != nil { log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) + h.WriteError(w, err.Error(), http.StatusInternalServerError) return } h.WriteJSON(w, fieldsMeta, http.StatusOK) } -func (h *InsightAPI) HandleGetMetricData(w http.ResponseWriter, req *http.Request, ps httprouter.Params){ +func (h *InsightAPI) HandleGetMetricData(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { reqBody := insight.Metric{} err := h.DecodeJSON(req, &reqBody) if err != nil { @@ -185,9 +185,9 @@ func (h *InsightAPI) HandleGetMetricData(w http.ResponseWriter, req *http.Reques return } clusterID := ps.MustGetParameter("id") - if !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern){ + if !h.IsIndexAllowed(req, clusterID, reqBody.IndexPattern) { allowedSystemIndices := getAllowedSystemIndices() - if clusterID != global.MustLookupString(elastic.GlobalSystemElasticsearchID) || !radix.Compile(allowedSystemIndices...).Match(reqBody.IndexPattern){ + if clusterID != global.MustLookupString(elastic.GlobalSystemElasticsearchID) || !radix.Compile(allowedSystemIndices...).Match(reqBody.IndexPattern) { h.WriteError(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } @@ -205,8 +205,9 @@ func (h *InsightAPI) HandleGetMetricData(w http.ResponseWriter, req *http.Reques var ( allowedSystemIndicesOnce sync.Once - allowedSystemIndices []string + allowedSystemIndices []string ) + func getAllowedSystemIndices() []string { allowedSystemIndicesOnce.Do(func() { metricIndexName := orm.GetWildcardIndexName(event.Event{}) @@ -235,20 +236,20 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { return nil, err } - var agg = searchResult["aggregations"] + var agg = searchResult["aggregations"] if metric.Filter != nil { if aggM, ok := agg.(map[string]interface{}); ok { agg = aggM["filter_agg"] } } - timeBeforeGroup := metric.AutoTimeBeforeGroup() + timeBeforeGroup := metric.AutoTimeBeforeGroup() metricData := CollectMetricData(agg, timeBeforeGroup) var targetMetricData []insight.MetricData formula := strings.TrimSpace(metric.Formula) if len(metric.Items) == 1 && formula == "" { - targetMetricData = metricData - }else { + targetMetricData = metricData + } else { tpl, err := template.New("insight_formula").Parse(formula) if err != nil { return nil, err @@ -270,7 +271,7 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { for _, md := range metricData { targetData := insight.MetricData{ Groups: md.Groups, - Data: map[string][]insight.MetricDataItem{}, + Data: map[string][]insight.MetricDataItem{}, } expression, err := govaluate.NewEvaluableExpression(formula) if err != nil { @@ -335,13 +336,13 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { return result, nil } -func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter interface{}, fieldsFormat map[string]string) (interface{}, error){ +func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter interface{}, fieldsFormat map[string]string) (interface{}, error) { fieldsMeta, err := getFieldsMetadata(indexPattern, clusterID) if err != nil { return nil, err } var ( - metas []insight.Visualization + metas []insight.Visualization seriesType string aggTypes []string @@ -352,8 +353,8 @@ func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter } length := len(fieldNames) step := 50 - for i := 0; i length { end = length } @@ -379,8 +380,8 @@ func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter if count <= 10 { if timeField == "" { seriesType = "pie" - }else { - if aggField.Type == "string"{ + } else { + if aggField.Type == "string" { seriesType = "column" options["seriesField"] = "group" } @@ -391,7 +392,7 @@ func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter aggTypes = []string{"count", "terms"} defaultAggType = "count" } else { - aggTypes = []string{"min", "max", "avg", "sum", "medium","count", "rate"} + aggTypes = []string{"min", "max", "avg", "sum", "medium", "count", "rate"} defaultAggType = "avg" if options["seriesField"] == "group" { defaultAggType = "count" @@ -406,20 +407,19 @@ func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter } } seriesItem := insight.SeriesItem{ - Type: seriesType, + Type: seriesType, Options: options, Metric: insight.Metric{ - Items: []insight.MetricItem{ - { - Name: "a", - Field: aggField.Name, - FieldType: aggField.Type, - Statistic: defaultAggType, - + Items: []insight.MetricItem{ + { + Name: "a", + Field: aggField.Name, + FieldType: aggField.Type, + Statistic: defaultAggType, + }, }, - }, - AggTypes: aggTypes, - }} + AggTypes: aggTypes, + }} if seriesType == "column" || seriesType == "pie" { seriesItem.Metric.Groups = []insight.MetricGroupItem{ {aggField.Name, 10}, @@ -437,10 +437,10 @@ func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter return metas, nil } -func countFieldValue(fields []string, clusterID, indexPattern string, filter interface{}) (map[string]float64, error){ +func countFieldValue(fields []string, clusterID, indexPattern string, filter interface{}) (map[string]float64, error) { aggs := util.MapStr{} for _, field := range fields { - aggs[field] = util.MapStr{ + aggs[field] = util.MapStr{ "cardinality": util.MapStr{ "field": field, }, @@ -455,7 +455,7 @@ func countFieldValue(fields []string, clusterID, indexPattern string, filter int }, "aggs": aggs, }, - } , + }, } if filter != nil { queryDsl["query"] = filter @@ -466,22 +466,22 @@ func countFieldValue(fields []string, clusterID, indexPattern string, filter int if err != nil { return nil, err } - fieldsCount := map[string] float64{} + fieldsCount := map[string]float64{} res := map[string]interface{}{} util.MustFromJSONBytes(searchRes.RawResult.Body, &res) if aggsM, ok := res["aggregations"].(map[string]interface{}); ok { if sampleAgg, ok := aggsM["sample"].(map[string]interface{}); ok { for key, agg := range sampleAgg { - if key == "doc_count"{ + if key == "doc_count" { continue } - if mAgg, ok := agg.(map[string]interface{});ok{ + if mAgg, ok := agg.(map[string]interface{}); ok { fieldsCount[key] = mAgg["value"].(float64) } } - }else{ + } else { for key, agg := range aggsM { - if mAgg, ok := agg.(map[string]interface{});ok{ + if mAgg, ok := agg.(map[string]interface{}); ok { fieldsCount[key] = mAgg["value"].(float64) } } @@ -494,17 +494,18 @@ func countFieldValue(fields []string, clusterID, indexPattern string, filter int type FieldsMetadata struct { Aggregatable map[string]elastic.ElasticField - Dates map[string]elastic.ElasticField + Dates map[string]elastic.ElasticField } -func getFieldsMetadata(indexPattern string, clusterID string) (*FieldsMetadata, error){ - fields, err := elastic.GetFieldCaps(clusterID, indexPattern, nil) +func getFieldsMetadata(indexPattern string, clusterID string) (*FieldsMetadata, error) { + esClient := elastic.GetClient(clusterID) + fields, err := elastic.GetFieldCaps(esClient, indexPattern, nil) if err != nil { return nil, err } var fieldsMeta = &FieldsMetadata{ Aggregatable: map[string]elastic.ElasticField{}, - Dates: map[string]elastic.ElasticField{}, + Dates: map[string]elastic.ElasticField{}, } for _, field := range fields { if field.Type == "date" { @@ -531,4 +532,4 @@ func parseFieldsFormat(formatMap string) (map[string]string, error) { } } return fieldsFormat, nil -} \ No newline at end of file +} diff --git a/plugin/api/layout/api.go b/plugin/api/layout/api.go index 1109f782..3f02ac02 100644 --- a/plugin/api/layout/api.go +++ b/plugin/api/layout/api.go @@ -5,12 +5,13 @@ package layout import ( + "infini.sh/console/core" + "infini.sh/console/core/security/enum" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" ) type LayoutAPI struct { - api.Handler + core.Handler } func InitAPI() { diff --git a/plugin/api/layout/layout.go b/plugin/api/layout/layout.go index 4c127102..a426e381 100644 --- a/plugin/api/layout/layout.go +++ b/plugin/api/layout/layout.go @@ -6,8 +6,8 @@ package layout import ( log "github.com/cihub/seelog" + "infini.sh/console/core/security" "infini.sh/console/model" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -25,7 +25,7 @@ func (h *LayoutAPI) createLayout(w http.ResponseWriter, req *http.Request, ps ht return } obj.ID = util.GetUUID() - user, err := rbac.FromUserContext(req.Context()) + user, err := security.FromUserContext(req.Context()) if err != nil { log.Error(err) h.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -147,13 +147,13 @@ func (h *LayoutAPI) deleteLayout(w http.ResponseWriter, req *http.Request, ps ht func (h *LayoutAPI) searchLayout(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var ( - keyword = h.GetParameterOrDefault(req, "keyword", "") - strSize = h.GetParameterOrDefault(req, "size", "20") - strFrom = h.GetParameterOrDefault(req, "from", "0") - viewID = strings.TrimSpace(h.GetParameterOrDefault(req, "view_id", "")) + keyword = h.GetParameterOrDefault(req, "keyword", "") + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + viewID = strings.TrimSpace(h.GetParameterOrDefault(req, "view_id", "")) typ = strings.TrimSpace(h.GetParameterOrDefault(req, "type", "")) - isFixed = strings.TrimSpace(h.GetParameterOrDefault(req, "is_fixed", "")) - mustQ []util.MapStr + isFixed = strings.TrimSpace(h.GetParameterOrDefault(req, "is_fixed", "")) + mustQ []util.MapStr ) if isFixed != "" { fixed, err := strconv.ParseBool(isFixed) @@ -188,8 +188,8 @@ func (h *LayoutAPI) searchLayout(w http.ResponseWriter, req *http.Request, ps ht if keyword != "" { mustQ = append(mustQ, util.MapStr{ "query_string": util.MapStr{ - "default_field":"*", - "query": keyword, + "default_field": "*", + "query": keyword, }, }) } @@ -224,4 +224,4 @@ func (h *LayoutAPI) searchLayout(w http.ResponseWriter, req *http.Request, ps ht return } h.Write(w, res.Raw) -} \ No newline at end of file +} diff --git a/plugin/api/notification/api.go b/plugin/api/notification/api.go index 17695d0b..a060ee43 100644 --- a/plugin/api/notification/api.go +++ b/plugin/api/notification/api.go @@ -1,11 +1,12 @@ package notification import ( + "infini.sh/console/core" "infini.sh/framework/core/api" ) type NotificationAPI struct { - api.Handler + core.Handler } func InitAPI() { diff --git a/plugin/api/notification/notification.go b/plugin/api/notification/notification.go index 551e1159..e4acf03d 100644 --- a/plugin/api/notification/notification.go +++ b/plugin/api/notification/notification.go @@ -3,12 +3,12 @@ package notification import ( "errors" "fmt" + "infini.sh/console/core/security" "net/http" "time" log "github.com/cihub/seelog" "infini.sh/console/model" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -23,7 +23,7 @@ type SearchNotificationsRequest struct { } func (h *NotificationAPI) searchNotifications(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - user, err := rbac.FromUserContext(req.Context()) + user, err := security.FromUserContext(req.Context()) if err != nil { log.Error("failed to get user from context, err: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -93,7 +93,7 @@ type SetNotificationsReadRequest struct { } func (h *NotificationAPI) setNotificationsRead(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - user, err := rbac.FromUserContext(req.Context()) + user, err := security.FromUserContext(req.Context()) if err != nil { log.Error("failed to get user from context, err: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) diff --git a/plugin/api/platform/api.go b/plugin/api/platform/api.go index c91b5187..7ee25438 100644 --- a/plugin/api/platform/api.go +++ b/plugin/api/platform/api.go @@ -8,10 +8,11 @@ import ( "fmt" log "github.com/cihub/seelog" "infini.sh/console/common" + "infini.sh/console/core" + "infini.sh/console/core/security" "infini.sh/console/model" "infini.sh/console/service" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" @@ -22,7 +23,7 @@ import ( ) type PlatformAPI struct { - api.Handler + core.Handler } func InitAPI() { @@ -43,12 +44,12 @@ func (h *PlatformAPI) searchCollection(w http.ResponseWriter, req *http.Request, return } if api.IsAuthEnable() { - claims, err := rbac.ValidateLogin(req.Header.Get("Authorization")) + claims, err := security.ValidateLogin(req.Header.Get("Authorization")) if err != nil { h.WriteError(w, err.Error(), http.StatusUnauthorized) return } - err = rbac.ValidatePermission(claims, meta.RequirePermission["read"]) + err = security.ValidatePermission(claims, meta.RequirePermission["read"]) if err != nil { h.WriteError(w, err.Error(), http.StatusForbidden) return @@ -128,7 +129,7 @@ func (h *PlatformAPI) rewriteQueryWithFilter(queryDsl []byte, filter util.MapStr func (h *PlatformAPI) getCollectionMeta(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { collName := ps.MustGetParameter("collection_name") if collName == "activity" { - user, auditLogErr := rbac.FromUserContext(req.Context()) + user, auditLogErr := security.FromUserContext(req.Context()) if auditLogErr == nil && h.GetHeader(req, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(user.Username). WithLogTypeAccess().WithResourceTypeAccountCenter(). diff --git a/plugin/api/platform/domain.go b/plugin/api/platform/domain.go index 240d0d81..e48c09b1 100644 --- a/plugin/api/platform/domain.go +++ b/plugin/api/platform/domain.go @@ -5,9 +5,9 @@ package platform import ( + "infini.sh/console/core/security/enum" consoleModel "infini.sh/console/model" "infini.sh/console/model/alerting" - "infini.sh/framework/core/api/rbac/enum" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" "infini.sh/framework/core/model" diff --git a/plugin/audit_log/monitoring_interceptor.go b/plugin/audit_log/monitoring_interceptor.go index 566893c8..de0cd5aa 100644 --- a/plugin/audit_log/monitoring_interceptor.go +++ b/plugin/audit_log/monitoring_interceptor.go @@ -7,10 +7,10 @@ package audit_log import ( "context" "infini.sh/console/common" + "infini.sh/console/core/security" "infini.sh/console/model" "infini.sh/console/service" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac" "net/http" "regexp" "strings" @@ -36,7 +36,7 @@ func (m *MonitoringInterceptor) PreHandle(c context.Context, _ http.ResponseWrit targetClusterID = matches[1] eventName = strings.Replace(matches[2], "/", " ", -1) } - claims, auditLogErr := rbac.ValidateLogin(request.Header.Get("Authorization")) + claims, auditLogErr := security.ValidateLogin(request.Header.Get("Authorization")) if auditLogErr == nil && handler.GetHeader(request, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(claims.Username). WithLogTypeAccess().WithResourceTypeClusterManagement(). diff --git a/plugin/managed/client/client.go b/plugin/managed/client/client.go new file mode 100644 index 00000000..761d4659 --- /dev/null +++ b/plugin/managed/client/client.go @@ -0,0 +1,261 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package client + +import ( + "context" + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/plugin/managed/common" + "infini.sh/console/plugin/managed/config" + "infini.sh/framework/core/api" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/keystore" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/model" + "infini.sh/framework/core/task" + "infini.sh/framework/core/util" + "net/http" + "net/url" + "os" + "path/filepath" + "sync" + "time" +) + +const bucketName = "instance_registered" +const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" + +func ConnectToManager() error { + + if !global.Env().SystemConfig.Configs.Managed { + return nil + } + + if exists, err := kv.ExistsKey(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID)); exists && err == nil { + //already registered skip further process + log.Info("already registered to config manager") + global.Register(configRegisterEnvKey, true) + return nil + } + + log.Info("register new instance to config manager") + + //register to config manager + if global.Env().SystemConfig.Configs.Servers == nil || len(global.Env().SystemConfig.Configs.Servers) == 0 { + return errors.Errorf("no config manager was found") + } + + info := model.GetInstanceInfo() + + req := util.Request{Method: util.Verb_POST} + req.ContentType = "application/json" + req.Path = common.REGISTER_API + req.Body = util.MustToJSONBytes(info) + + server, res, err := submitRequestToManager(&req) + if err == nil && server != "" { + if res.StatusCode == 200 || util.ContainStr(string(res.Body), "exists") { + log.Infof("success register to config manager: %v", string(server)) + err := kv.AddValue(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(util.GetLowPrecisionCurrentTime().String())) + if err != nil { + panic(err) + } + global.Register(configRegisterEnvKey, true) + } + } else { + log.Error("failed to register to config manager,", err, ",", server) + } + return err +} + +func submitRequestToManager(req *util.Request) (string, *util.Result, error) { + var err error + var res *util.Result + for _, server := range global.Env().SystemConfig.Configs.Servers { + req.Url, err = url.JoinPath(server, req.Path) + if err != nil { + continue + } + res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + if err != nil { + continue + } + return server, res, nil + } + return "", nil, err +} + +var clientInitLock = sync.Once{} +var mTLSClient *http.Client + +func ListenConfigChanges() error { + clientInitLock.Do(func() { + if global.Env().SystemConfig.Configs.Managed { + //init client + cfg := global.Env().GetClientConfigByEndpoint("configs", "") + if cfg != nil { + hClient, err := api.NewHTTPClient(cfg) + if err != nil { + panic(err) + } + mTLSClient = hClient + } + + //init config sync listening + req := common.ConfigSyncRequest{} + req.Client = model.GetInstanceInfo() + + var syncFunc = func() { + if global.Env().IsDebug { + log.Trace("fetch configs from manger") + } + + cfgs := config.GetConfigs(false, false) + req.Configs = cfgs + req.Hash = util.MD5digestString(util.MustToJSONBytes(cfgs)) + + //fetch configs from manager + request := util.Request{Method: util.Verb_POST} + request.ContentType = "application/json" + request.Path = common.SYNC_API + request.Body = util.MustToJSONBytes(req) + + if global.Env().IsDebug { + log.Debug("config sync request: ", string(util.MustToJSONBytes(req))) + } + + _, res, err := submitRequestToManager(&request) + if err != nil { + log.Error("failed to submit request to config manager,", err) + return + } + + if res != nil { + obj := common.ConfigSyncResponse{} + err := util.FromJSONBytes(res.Body, &obj) + if err != nil { + panic(err) + } + + if global.Env().IsDebug { + log.Debug("config sync response: ", string(res.Body)) + } + + if obj.Changed { + + //update secrets //TODO client send salt to manager first, manager encrypt secrets with salt and send back + if obj.Secrets != nil { + for k, v := range obj.Secrets.Keystore { + if v.Type == "plaintext" { + err := saveKeystore(k, v.Value) + if err != nil { + log.Error("error on save keystore:", k, ",", err) + } + } + } + + //TODO maybe we have other secrets + } + + for _, v := range obj.Configs.DeletedConfigs { + if v.Managed { + err := config.DeleteConfig(v.Name) + if err != nil { + panic(err) + } + } else { + log.Error("config [", v.Name, "] is not managed by config manager, skip deleting") + } + } + + for _, v := range obj.Configs.CreatedConfigs { + err := config.SaveConfig(v.Name, v) + if err != nil { + panic(err) + } + } + + for _, v := range obj.Configs.UpdatedConfigs { + err := config.SaveConfig(v.Name, v) + if err != nil { + panic(err) + } + } + + var keyValuePairs = []util.KeyValue{} + //checking backup files, remove old configs, only keep max num of files + filepath.Walk(global.Env().SystemConfig.PathConfig.Config, func(file string, f os.FileInfo, err error) error { + + //only allow to delete backup files + if !util.SuffixStr(file, ".bak") { + return nil + } + + keyValuePairs = append(keyValuePairs, util.KeyValue{Key: file, Value: f.ModTime().Unix()}) + return nil + }) + if len(keyValuePairs) > 0 { + keyValuePairs = util.SortKeyValueArray(keyValuePairs, true) + + if len(keyValuePairs) > global.Env().SystemConfig.Configs.MaxBackupFiles { + tobeDeleted := keyValuePairs[global.Env().SystemConfig.Configs.MaxBackupFiles:] + for _, v := range tobeDeleted { + log.Debug("delete config file: ", v.Key) + err := util.FileDelete(v.Key) + if err != nil { + panic(err) + } + } + } + } + } + } + + } + + syncFunc() + + if !global.Env().SystemConfig.Configs.ScheduledTask { + log.Debug("register background task for checking configs changes") + global.RegisterBackgroundCallback(&global.BackgroundTask{ + Tag: "checking configs changes", + Interval: util.GetDurationOrDefault(global.Env().SystemConfig.Configs.Interval, time.Duration(30)*time.Second), + Func: syncFunc, + }) + } else { + log.Debug("register schedule task for checking configs changes") + task.RegisterScheduleTask(task.ScheduleTask{ + Description: fmt.Sprintf("sync configs from manager"), + Type: "interval", + Interval: global.Env().SystemConfig.Configs.Interval, + Task: func(ctx context.Context) { + syncFunc() + }, + }) + } + } + + }) + + return nil +} + +func saveKeystore(k string, v string) error { + + log.Debug("save keystore:", k) + + ks, err := keystore.GetWriteableKeystore() + if err != nil { + return err + } + err = ks.Store(k, util.UnsafeStringToBytes(v)) + if err != nil { + return err + } + err = ks.Save() + return err +} diff --git a/plugin/managed/common/certs.go b/plugin/managed/common/certs.go new file mode 100644 index 00000000..749afcc9 --- /dev/null +++ b/plugin/managed/common/certs.go @@ -0,0 +1,43 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +import ( + "crypto/x509" + "encoding/pem" + log "github.com/cihub/seelog" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "os" + "path" +) + +func GetOrInitDefaultCaCerts()(string, string, error){ + dataDir := global.Env().GetDataDir() + caFile := path.Join(dataDir, "certs/ca.crt") + caKey := path.Join(dataDir, "certs/ca.key") + if !(util.FileExists(caFile) && util.FileExists(caKey) ) { + err := os.MkdirAll(path.Join(dataDir, "certs"), 0775) + if err != nil { + return "", "", err + } + log.Info("auto generating cert files") + _, rootKey, rootCertPEM := util.GetRootCert() + + caKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), + }) + _, err = util.FilePutContentWithByte(caKey, caKeyPEM) + if err != nil { + return "", "", err + } + _, err = util.FilePutContentWithByte(caFile, rootCertPEM) + if err != nil { + return "", "", err + } + } + return caFile, caKey, nil +} + diff --git a/plugin/managed/common/domain.go b/plugin/managed/common/domain.go new file mode 100644 index 00000000..d615d4a4 --- /dev/null +++ b/plugin/managed/common/domain.go @@ -0,0 +1,89 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +import "infini.sh/framework/core/model" + +const REGISTER_API = "/instance/_register" +const SYNC_API = "/configs/_sync" +const GET_INSTALL_SCRIPT_API = "/instance/_get_install_script" + +type ConfigFile struct { + Name string `json:"name,omitempty"` + Location string `json:"location,omitempty"` + Content string `json:"content,omitempty"` + Updated int64 `json:"updated,omitempty"` + Version int64 `json:"version,omitempty"` + Size int64 `json:"size,omitempty"` + Readonly bool `json:"readonly,omitempty"` + Managed bool `json:"managed"` + Hash string `json:"hash,omitempty"` +} + +type ConfigList struct { + Main ConfigFile `json:"main,omitempty"` + Configs map[string]ConfigFile `json:"configs,omitempty"` +} + +type ConfigDeleteRequest struct { + Configs []string `json:"configs"` +} + +type ConfigUpdateRequest struct { + Configs map[string]string `json:"configs"` +} + +type ConfigSyncRequest struct { + ForceSync bool `json:"force_sync"` //ignore hash check in server + Hash string `json:"hash"` + Client model.Instance `json:"client"` + Configs ConfigList `json:"configs"` +} + +type ConfigSyncResponse struct { + Changed bool `json:"changed"` + Configs struct { + CreatedConfigs map[string]ConfigFile `json:"created,omitempty"` + DeletedConfigs map[string]ConfigFile `json:"deleted,omitempty"` + UpdatedConfigs map[string]ConfigFile `json:"updated,omitempty"` + } `json:"configs,omitempty"` + + Secrets *Secrets `json:"secrets,omitempty"` +} + +type ResourceGroup struct { + Name string `json:"name"` + List []string `json:"list"` +} + +type ConfigGroup struct { + Files []string `config:"files"` +} + +type InstanceGroup struct { + ConfigGroups []string `config:"configs"` + Instances []string `config:"instances"` + Secrets []string `config:"secrets"` +} + +type Secrets struct { + Keystore map[string]KeystoreValue `json:"keystore,omitempty"` +} + +type KeystoreValue struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type ConfigRepo struct { + ConfigGroups map[string]ConfigGroup `config:"configs"` + InstanceGroups map[string]InstanceGroup `config:"instances"` + SecretGroups map[string]Secrets `config:"secrets"` +} + +type InstanceSettings struct { + ConfigFiles []string `config:"configs"` + Secrets []string `config:"secrets"` +} diff --git a/plugin/managed/config/config.go b/plugin/managed/config/config.go new file mode 100755 index 00000000..153739f3 --- /dev/null +++ b/plugin/managed/config/config.go @@ -0,0 +1,339 @@ +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +package config + +import ( + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/core" + "infini.sh/console/plugin/managed/common" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/env" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "infini.sh/framework/core/util/file" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" +) + +func init() { + api.HandleAPIMethod(api.GET, "/config/", h.listConfigAction) + api.HandleAPIMethod(api.PUT, "/config/", h.saveConfigAction) + api.HandleAPIMethod(api.DELETE, "/config/", h.deleteConfigAction) + api.HandleAPIMethod(api.POST, "/config/_reload", h.reloadConfigAction) + api.HandleAPIMethod(api.GET, "/config/runtime", h.getConfigAction) + api.HandleAPIMethod(api.GET, "/environments", h.getEnvAction) + +} + +var h = DefaultHandler{} + +type DefaultHandler struct { + core.Handler +} + +func (handler DefaultHandler) getEnvAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + handler.WriteJSONHeader(w) + handler.WriteJSON(w, os.Environ(), 200) +} + +func (handler DefaultHandler) getConfigAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + json := env.GetConfigAsJSON() + handler.WriteJSONHeader(w) + handler.Write(w, []byte(json)) +} + +func validateFile(cfgDir, name string) error { + + cfgDir, _ = filepath.Abs(cfgDir) + name, _ = filepath.Abs(name) + + //filter out the hidden files and go files + if strings.HasPrefix(filepath.Base(name), ".") || strings.HasSuffix(filepath.Base(name), ".go") { + return errors.Errorf("invalid config filename") + } + + //filetype checking + ext := filepath.Ext(name) + if len(global.Env().SystemConfig.Configs.ValidConfigsExtensions) > 0 && !util.ContainsAnyInArray(ext, global.Env().SystemConfig.Configs.ValidConfigsExtensions) { + return errors.Errorf("invalid config file: %s, only support: %v", name, global.Env().SystemConfig.Configs.ValidConfigsExtensions) + } + + //permission checking + if !util.IsFileWithinFolder(name, cfgDir) { + return errors.Errorf("invalid config file: %s, outside of path: %v", name, cfgDir) + } + return nil +} + +func DeleteConfig(name string) error { + + cfgDir, err := filepath.Abs(global.Env().GetConfigDir()) + if err != nil { + return err + } + + fileToDelete := path.Join(cfgDir, name) + + log.Info("delete config file: ", fileToDelete) + + //file checking + if err := validateFile(cfgDir, fileToDelete); err != nil { + return err + } + + if util.FileExists(fileToDelete) { + if global.Env().SystemConfig.Configs.SoftDelete { + err := util.Rename(fileToDelete, fmt.Sprintf("%v.%v.bak", fileToDelete, time.Now().UnixMilli())) + if err != nil { + return err + } + } else { + err := util.FileDelete(fileToDelete) + if err != nil { + return err + } + } + } else { + return errors.Errorf("file not exists: %s", fileToDelete) + } + + return nil +} + +func SaveConfig(name string, cfg common.ConfigFile) error { + + //update version + if cfg.Managed { + cfg.Content = updateConfigVersion(cfg.Content, cfg.Version) + cfg.Content = updateConfigManaged(cfg.Content, true) + } + + return SaveConfigStr(name, cfg.Content) +} + +func SaveConfigStr(name, content string) error { + + cfgDir, err := filepath.Abs(global.Env().GetConfigDir()) + if err != nil { + return err + } + + fileToSave := path.Join(cfgDir, name) + + log.Info("write config file: ", fileToSave) + log.Trace("file content: ", content) + + if err := validateFile(cfgDir, fileToSave); err != nil { + return err + } + + if util.FileExists(fileToSave) { + if global.Env().SystemConfig.Configs.SoftDelete { + err := util.Rename(fileToSave, fmt.Sprintf("%v.%v.bak", fileToSave, time.Now().UnixMilli())) + if err != nil { + return err + } + } + } + + _, err = util.FilePutContent(fileToSave, content) + if err != nil { + return err + } + + return nil +} + +func (handler DefaultHandler) deleteConfigAction(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + reqBody := common.ConfigDeleteRequest{} + err := handler.DecodeJSON(req, &reqBody) + if err != nil { + panic(err) + } + + for _, name := range reqBody.Configs { + err := DeleteConfig(name) + if err != nil { + panic(err) + } + } + handler.WriteAckOKJSON(w) +} + +var versionRegexp = regexp.MustCompile(`#MANAGED_CONFIG_VERSION:\s*(?P\d+)`) +var managedRegexp = regexp.MustCompile(`#MANAGED:\s*(?P\w+)`) + +func parseConfigVersion(input string) int64 { + matches := versionRegexp.FindStringSubmatch(util.TrimSpaces(input)) + if len(matches) > 0 { + ver := versionRegexp.SubexpIndex("version") + if ver >= 0 { + str := (matches[ver]) + v, err := util.ToInt64(util.TrimSpaces(str)) + if err != nil { + log.Error(err) + } + return v + } + } + return -1 +} + +func parseConfigManaged(input string, defaultManaged bool) bool { + matches := managedRegexp.FindStringSubmatch(util.TrimSpaces(input)) + if len(matches) > 0 { + v := managedRegexp.SubexpIndex("managed") + if v >= 0 { + str := util.TrimSpaces(strings.ToLower(matches[v])) + if str == "true" { + return true + } else if str == "false" { + return false + } + } + } + return defaultManaged +} + +func updateConfigManaged(input string, managed bool) string { + if managedRegexp.MatchString(input) { + return managedRegexp.ReplaceAllString(input, fmt.Sprintf("#MANAGED: %v", managed)) + } else { + return fmt.Sprintf("%v\n#MANAGED: %v", input, managed) + } +} + +func updateConfigVersion(input string, newVersion int64) string { + if versionRegexp.MatchString(input) { + return versionRegexp.ReplaceAllString(input, fmt.Sprintf("#MANAGED_CONFIG_VERSION: %d", newVersion)) + } else { + return fmt.Sprintf("%v\n#MANAGED_CONFIG_VERSION: %d", input, newVersion) + } +} + +func (handler DefaultHandler) listConfigAction(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + configs := GetConfigs(true, false) + handler.WriteJSON(w, configs, 200) +} + +func GetConfigs(returnContent, managedOnly bool) common.ConfigList { + + cfgDir, err := filepath.Abs(global.Env().GetConfigDir()) + if err != nil { + panic(err) + } + + configs := common.ConfigList{} + configs.Configs = make(map[string]common.ConfigFile) + + mainConfig, _ := filepath.Abs(global.Env().GetConfigFile()) + + info, err := file.Stat(mainConfig) + if err != nil { + panic(err) + } + c, err := util.FileGetContent(mainConfig) + if err != nil { + panic(err) + } + + configs.Main = common.ConfigFile{ + Name: util.TrimLeftStr(filepath.Base(mainConfig), cfgDir), + Location: mainConfig, + Readonly: true, + Managed: false, + Updated: info.ModTime().Unix(), + Size: info.Size(), + } + + if returnContent { + configs.Main.Content = string(c) + } + + //get files within folder + filepath.Walk(cfgDir, func(file string, f os.FileInfo, err error) error { + + cfg, err := GetConfigFromFile(cfgDir, file) + if cfg != nil { + + if !cfg.Managed && managedOnly { + return nil + } + + if !returnContent { + cfg.Content = "" + } + + configs.Configs[cfg.Name] = *cfg + } + return nil + }) + + return configs +} + +func GetConfigFromFile(cfgDir, filename string) (*common.ConfigFile, error) { + + //file checking + if err := validateFile(cfgDir, filename); err != nil { + return nil, err + } + + c, err := util.FileGetContent(filename) + if err != nil { + return nil, err + } + + f, err := file.Stat(filename) + if err != nil { + return nil, err + } + + content := string(c) + cfg := common.ConfigFile{ + Name: util.TrimLeftStr(filepath.Base(filename), cfgDir), + Location: filename, + Version: parseConfigVersion(content), + Updated: f.ModTime().Unix(), + Size: f.Size(), + } + + cfg.Content = content + cfg.Managed = parseConfigManaged(content, cfg.Version > 0 || global.Env().SystemConfig.Configs.ConfigFileManagedByDefault) + + return &cfg, nil +} + +func (handler DefaultHandler) saveConfigAction(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + reqBody := common.ConfigUpdateRequest{} + err := handler.DecodeJSON(req, &reqBody) + if err != nil { + panic(err) + } + + for name, content := range reqBody.Configs { + err := SaveConfigStr(name, content) + if err != nil { + panic(err) + } + } + handler.WriteAckOKJSON(w) +} + +func (handler DefaultHandler) reloadConfigAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + log.Infof("refresh config") + err := global.Env().RefreshConfig() + if err != nil { + panic(err) + } + handler.WriteAckOKJSON(w) +} diff --git a/plugin/managed/config/config_test.go b/plugin/managed/config/config_test.go new file mode 100644 index 00000000..19b8ef6d --- /dev/null +++ b/plugin/managed/config/config_test.go @@ -0,0 +1,43 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package config + +import ( + "fmt" + "github.com/magiconair/properties/assert" + "testing" +) + +func TestUpdateVersion(t *testing.T) { + input := `PEFFIX DON"T CHANGE,,, #MANAGED_CONFIG_VERSION: 1234, keep this as well` + output := updateConfigVersion(input, 1235) + fmt.Println(output) + assert.Equal(t, output, `PEFFIX DON"T CHANGE,,, #MANAGED_CONFIG_VERSION: 1235, keep this as well`) + + + input = `PEFFIX DON"T CHANGE, this is my config` + output = updateConfigVersion(input, 1235) + fmt.Println("new config:") + fmt.Println(output) + + assert.Equal(t, output, "PEFFIX DON\"T CHANGE, this is my config\n#MANAGED_CONFIG_VERSION: 1235") +} + +func TestVersion(t *testing.T) { + ver:=parseConfigVersion("#MANAGED_CONFIG_VERSION: 1234") + assert.Equal(t, ver, int64(1234)) + + ver=parseConfigVersion("#MANAGED_CONFIG_VERSION:1234") + assert.Equal(t, ver, int64(1234)) + + ver=parseConfigVersion("#MANAGED_CONFIG_VERSION:1234 ") + assert.Equal(t, ver, int64(1234)) + + ver=parseConfigVersion("##MANAGED_CONFIG_VERSION: 1234") + assert.Equal(t, ver, int64(1234)) + + ver=parseConfigVersion("what's the version, i think is 1234") + assert.Equal(t, ver, int64(-1)) +} diff --git a/plugin/managed/managed.go b/plugin/managed/managed.go new file mode 100644 index 00000000..944d06e4 --- /dev/null +++ b/plugin/managed/managed.go @@ -0,0 +1,5 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package managed diff --git a/plugin/managed/server/config.go b/plugin/managed/server/config.go new file mode 100644 index 00000000..1b913825 --- /dev/null +++ b/plugin/managed/server/config.go @@ -0,0 +1,286 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package server + +import ( + log "github.com/cihub/seelog" + "infini.sh/console/plugin/managed/common" + "infini.sh/console/plugin/managed/config" + httprouter "infini.sh/framework/core/api/router" + config3 "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + "net/http" + "path" + "sync" +) + +var configProvidersLock = sync.RWMutex{} +var configProviders = []func(instance model.Instance) []*common.ConfigFile{} + +func RegisterConfigProvider(provider func(instance model.Instance) []*common.ConfigFile) { + configProvidersLock.Lock() + defer configProvidersLock.Unlock() + configProviders = append(configProviders, provider) +} + +func refreshConfigsRepo() { + + //load config settings from file + if global.Env().SystemConfig.Configs.ManagerConfig.LocalConfigsRepoPath != "" { + configRepo = common.ConfigRepo{} + cfgPath := path.Join(global.Env().SystemConfig.Configs.ManagerConfig.LocalConfigsRepoPath, "/settings.yml") + + if !util.FileExists(cfgPath) { + log.Debugf("config not exists, skip loading: %v", cfgPath) + return + } + + setCfg, err := config3.LoadFiles(cfgPath) + if err != nil { + panic(err) + } + + err = setCfg.Unpack(&configRepo) + log.Debug("loading config_repo: ", configRepo) + if err != nil { + panic(err) + } + + if configRepo.InstanceGroups != nil { + for _, v := range configRepo.InstanceGroups { + cfgs := []string{} + for _, f := range v.ConfigGroups { + cfg, ok := configRepo.ConfigGroups[f] + if ok { + cfgs = append(cfgs, cfg.Files...) + } + } + + secrets := []common.Secrets{} + for _, f := range v.Secrets { + secret, ok := configRepo.SecretGroups[f] + if ok { + secrets = append(secrets, secret) + } + } + for _, x := range v.Instances { + instanceConfigFiles[x] = cfgs + instanceSecrets[x] = secrets + } + } + } + } +} + +func getSecretsForInstance(instance model.Instance) *common.Secrets { + secrets := common.Secrets{} + secrets.Keystore = map[string]common.KeystoreValue{} + + //get config files for static settings + serverInit.Do(func() { + refreshConfigsRepo() + }) + + if instanceSecrets != nil { + v, ok := instanceSecrets[instance.ID] + if ok { + for _, f := range v { + if ok { + for m, n := range f.Keystore { + secrets.Keystore[m] = n + } + } + } + } + } + return &secrets +} + +func getConfigsForInstance(instance model.Instance) []*common.ConfigFile { + result := []*common.ConfigFile{} + + //get config files for static settings + serverInit.Do(func() { + refreshConfigsRepo() + }) + + if instanceConfigFiles != nil { + v, ok := instanceConfigFiles[instance.ID] + if ok { + for _, x := range v { + file := path.Join(global.Env().SystemConfig.Configs.ManagerConfig.LocalConfigsRepoPath, x) + log.Debug("prepare config:", file) + cfg, err := config.GetConfigFromFile(global.Env().SystemConfig.Configs.ManagerConfig.LocalConfigsRepoPath, file) + if err != nil { + panic(err) + } + if cfg != nil { + cfg.Managed = true + result = append(result, cfg) + } + } + } + } + return result +} + +func (h APIHandler) refreshConfigsRepo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + refreshConfigsRepo() + h.WriteAckOKJSON(w) +} + +func (h APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + var obj = &common.ConfigSyncRequest{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + } + + if global.Env().IsDebug { + log.Trace("request:", util.MustToJSON(obj)) + } + + //TODO, check the client's and the server's hash, if same, skip the sync + + var res = common.ConfigSyncResponse{} + res.Configs.CreatedConfigs = map[string]common.ConfigFile{} + res.Configs.UpdatedConfigs = map[string]common.ConfigFile{} + res.Configs.DeletedConfigs = map[string]common.ConfigFile{} + + //check if client is enrolled + + //check if client was marked as deleted + + //find out the client belongs to which config group + + //if server's hash didn't change, skip + + //if client's hash don't change, skip + + //find out different configs, add or delete configs + cfgs := getConfigsForInstance(obj.Client) + + newCfgs := getConfigsFromExternalProviders(obj.Client) + + if newCfgs != nil && len(newCfgs) > 0 { + cfgs = append(cfgs, newCfgs...) + } + + if global.Env().IsDebug { + log.Debugf("get configs for agent(%v): %v", obj.Client.ID, util.MustToJSON(cfgs)) + } + + if cfgs == nil || len(cfgs) == 0 { + + if len(obj.Configs.Configs) > 0 { + //set everything is deleted + log.Debugf("no config get from manager, exists config should be all deleted for instance: %v", obj.Client.ID) + res.Configs.DeletedConfigs = obj.Configs.Configs + res.Changed = true + } else { + log.Debugf("no config found from manager for instance: %v", obj.Client.ID) + res.Changed = false + } + h.WriteJSON(w, res, 200) + return + } + + //get configs from repo, let's diff and send to client + if len(cfgs) > 0 { + + cfgMap := map[string]common.ConfigFile{} + for _, c := range cfgs { + cfgMap[c.Name] = *c + } + + //find out which config content was changed, replace to new content + if len(obj.Configs.Configs) > 0 { + //check diff + for k, v := range cfgMap { + x, ok := obj.Configs.Configs[k] + //both exists + if ok { + if global.Env().IsDebug { + log.Trace("both exists: ", k, ", checking version: ", v.Version, " vs ", x.Version) + } + + if !x.Managed { + log.Debugf("config %v was marked as not to be managed, skip", k) + continue + } + + if global.Env().IsDebug { + log.Debugf("check version for config %v, %v vs %v, %v", k, v.Version, x.Version, x.Managed) + } + + //let's diff the version + if v.Version > x.Version { + if global.Env().IsDebug { + log.Trace("get newly version from server, let's sync to client: ", k) + } + + res.Configs.UpdatedConfigs[k] = v + res.Changed = true + } else { + //this config no need to update + if global.Env().IsDebug { + log.Trace("config not changed: ", k) + } + } + } else { + if global.Env().IsDebug { + log.Trace("found new configs: ", k, ", version: ", v.Version) + } + res.Configs.CreatedConfigs[k] = v + res.Changed = true + } + } + + //check removed files + for k, v := range obj.Configs.Configs { + _, ok := cfgMap[k] + if !ok { + //missing in server's config + res.Configs.DeletedConfigs[k] = v + res.Changed = true + if global.Env().IsDebug { + log.Trace("config was removed from server, let's mark it as deleted: ", k) + } + } + } + } else { + if global.Env().IsDebug { + log.Tracef("found %v new configs", len(cfgs)) + } + res.Changed = true + res.Configs.CreatedConfigs = cfgMap + } + } + + //only if config changed, we change try to update the client's secrets, //TODO maybe there are coupled + if res.Changed { + secrets := getSecretsForInstance(obj.Client) + res.Secrets = secrets + } + + h.WriteJSON(w, res, 200) + +} + +func getConfigsFromExternalProviders(client model.Instance) []*common.ConfigFile { + configProvidersLock.Lock() + defer configProvidersLock.Unlock() + var cfgs []*common.ConfigFile + for _, p := range configProviders { + c := p(client) + if c != nil && len(c) > 0 { + cfgs = append(cfgs, c...) + } + } + return cfgs +} diff --git a/plugin/managed/server/instance.go b/plugin/managed/server/instance.go new file mode 100644 index 00000000..4547bebd --- /dev/null +++ b/plugin/managed/server/instance.go @@ -0,0 +1,498 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package server + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + log "github.com/cihub/seelog" + "infini.sh/console/core/security/enum" + "infini.sh/console/plugin/managed/common" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + elastic2 "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic" + common2 "infini.sh/framework/modules/elastic/common" +) + +var instanceConfigFiles = map[string][]string{} //map instance->config files TODO lru cache, short life instance should be removed +var instanceSecrets = map[string][]common.Secrets{} //map instance->secrets TODO lru cache, short life instance should be removed + +func init() { + //for public usage, agent can report self to server, usually need to enroll by manager + api.HandleAPIMethod(api.POST, common.REGISTER_API, handler.registerInstance) //client register self to config servers + + //for public usage, get install script + api.HandleAPIMethod(api.GET, common.GET_INSTALL_SCRIPT_API, handler.getInstallScript) + + api.HandleAPIMethod(api.POST, "/instance/_generate_install_script", handler.RequireLogin(handler.generateInstallCommand)) + + api.HandleAPIMethod(api.POST, "/instance", handler.RequirePermission(handler.createInstance, enum.PermissionGatewayInstanceWrite)) + api.HandleAPIMethod(api.GET, "/instance/:instance_id", handler.RequirePermission(handler.getInstance, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.PUT, "/instance/:instance_id", handler.RequirePermission(handler.updateInstance, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.DELETE, "/instance/:instance_id", handler.RequirePermission(handler.deleteInstance, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/instance/_enroll", handler.RequirePermission(handler.enrollInstance, enum.PermissionGatewayInstanceWrite)) //config server enroll clients + + api.HandleAPIMethod(api.GET, "/instance/_search", handler.RequirePermission(handler.searchInstance, enum.PermissionAgentInstanceRead)) + + api.HandleAPIMethod(api.POST, "/instance/stats", handler.RequirePermission(handler.getInstanceStatus, enum.PermissionAgentInstanceRead)) + + //delegate request to instance + api.HandleAPIMethod(api.POST, "/instance/:instance_id/_proxy", handler.RequirePermission(handler.proxy, enum.PermissionGatewayInstanceRead)) + api.HandleAPIMethod(api.POST, "/instance/:instance_id/elasticsearch/try_connect", handler.RequireLogin(handler.tryESConnect)) + + //try to connect to instance + api.HandleAPIMethod(api.POST, "/instance/try_connect", handler.RequireLogin(handler.tryConnect)) + +} + +func (h APIHandler) registerInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + var obj = &model.Instance{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + } + + oldInst := &model.Instance{} + oldInst.ID = obj.ID + exists, err := orm.Get(oldInst) + if exists { + errMsg := fmt.Sprintf("agent [%s] already exists", obj.ID) + h.WriteError(w, errMsg, http.StatusInternalServerError) + return + } + + err = orm.Create(nil, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + log.Infof("register instance: %v[%v], %v", obj.Name, obj.ID, obj.Endpoint) + + h.WriteAckOKJSON(w) +} + +func (h APIHandler) enrollInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + +} + +func (h *APIHandler) getInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + id := ps.MustGetParameter("instance_id") + + obj := model.Instance{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": id, + "_source": obj, + }, 200) +} + +func (h *APIHandler) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var obj = &model.Instance{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + res, err := h.getInstanceInfo(obj.Endpoint, obj.BasicAuth) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + obj.ID = res.ID + obj.Description = res.Description + if len(res.Tags) > 0 { + obj.Tags = res.Tags + } + if res.Name != "" && obj.Name == "" { + obj.Name = res.Name + } + obj.Application = res.Application + res.Network = res.Network + + exists, err := orm.Get(obj) + if err != nil && err != elastic.ErrNotFound { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if exists { + h.WriteError(w, "instance already registered", http.StatusInternalServerError) + return + } + err = orm.Create(nil, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "created", + }, 200) + +} + +func (h *APIHandler) deleteInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + + obj := model.Instance{} + obj.ID = id + + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + err = orm.Delete(nil, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteDeletedOKJSON(w, id) +} + +func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("instance_id") + obj := model.Instance{} + + obj.ID = id + exists, err := orm.Get(&obj) + if !exists || err != nil { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return + } + + id = obj.ID + create := obj.Created + obj = model.Instance{} + err = h.DecodeJSON(req, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + //protect + obj.ID = id + obj.Created = create + err = orm.Update(nil, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "updated", + }, 200) +} + +func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + var ( + application = h.GetParameterOrDefault(req, "application", "") + keyword = h.GetParameterOrDefault(req, "keyword", "") + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` + strSize = h.GetParameterOrDefault(req, "size", "20") + strFrom = h.GetParameterOrDefault(req, "from", "0") + mustBuilder = &strings.Builder{} + ) + if keyword != "" { + mustBuilder.WriteString(fmt.Sprintf(`{"query_string":{"default_field":"*","query": "%s"}}`, keyword)) + } + + if application != "" { + if mustBuilder.Len() > 0 { + mustBuilder.WriteString(",") + } + mustBuilder.WriteString(fmt.Sprintf(`{"term":{"application.name":"%s"}}`, application)) + } + + size, _ := strconv.Atoi(strSize) + if size <= 0 { + size = 20 + } + from, _ := strconv.Atoi(strFrom) + if from < 0 { + from = 0 + } + + q := orm.Query{} + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from) + q.RawQuery = []byte(queryDSL) + + err, res := orm.Search(&model.Instance{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.Write(w, res.Raw) +} + +// TODO replace proxy +func (h *APIHandler) getInstanceStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var instanceIDs = []string{} + err := h.DecodeJSON(req, &instanceIDs) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(instanceIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + q := orm.Query{} + queryDSL := util.MapStr{ + "size": len(instanceIDs), + "query": util.MapStr{ + "terms": util.MapStr{ + "_id": instanceIDs, + }, + }, + } + q.RawQuery = util.MustToJSONBytes(queryDSL) + + err, res := orm.Search(&model.Instance{}, &q) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + result := util.MapStr{} + for _, item := range res.Result { + instance := util.MapStr(item.(map[string]interface{})) + if err != nil { + log.Error(err) + continue + } + endpoint, _ := instance.GetValue("endpoint") + + gid, _ := instance.GetValue("id") + + //req := &proxy.Request{ + // Endpoint: endpoint.(string), + // Method: http.MethodGet, + // Path: "/stats", + //} + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + req := &util.Request{ + Method: http.MethodGet, + Path: "/stats", + Context: ctx, + } + + username, _ := instance.GetValue("basic_auth.username") + if username != nil && username.(string) != "" { + password, _ := instance.GetValue("basic_auth.password") + if password != nil && password.(string) != "" { + req.SetBasicAuth(username.(string), password.(string)) + } + } + + var resMap = util.MapStr{} + _, err := ProxyAgentRequest("runtime", endpoint.(string), req, &resMap) + if err != nil { + log.Error(endpoint, ",", err) + result[gid.(string)] = util.MapStr{} + continue + } + result[gid.(string)] = resMap + } + h.WriteJSON(w, result, http.StatusOK) +} + +func (h *APIHandler) proxy(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ( + method = h.Get(req, "method", "GET") + path = h.Get(req, "path", "") + ) + instanceID := ps.MustGetParameter("instance_id") + _, obj, err := GetRuntimeInstanceByID(instanceID) + if err != nil { + panic(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + reqBody, _ := h.GetRawBody(req) + req1 := &util.Request{ + Method: method, + Path: path, + Context: ctx, + Body: reqBody, + } + if obj.BasicAuth != nil { + req1.SetBasicAuth(obj.BasicAuth.Username, obj.BasicAuth.Password.Get()) + } + + res, err := ProxyAgentRequest("runtime", obj.GetEndpoint(), req1, nil) + if err != nil { + panic(err) + } + + h.WriteHeader(w, res.StatusCode) + h.Write(w, res.Body) +} + +func (h *APIHandler) getInstanceInfo(endpoint string, basicAuth *model.BasicAuth) (*model.Instance, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + req1 := &util.Request{ + Method: http.MethodGet, + Path: "/_info", + Context: ctx, + } + if basicAuth != nil { + req1.SetBasicAuth(basicAuth.Username, basicAuth.Password.Get()) + } + obj := &model.Instance{} + _, err := ProxyAgentRequest("runtime", endpoint, req1, obj) + if err != nil { + panic(err) + } + return obj, err + +} + +func (h *APIHandler) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var reqBody = struct { + Endpoint string `json:"endpoint"` + BasicAuth *model.BasicAuth `json:"basic_auth"` + }{} + err := h.DecodeJSON(req, &reqBody) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + connectRes, err := h.getInstanceInfo(reqBody.Endpoint, reqBody.BasicAuth) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, connectRes, http.StatusOK) +} + +func (h *APIHandler) tryESConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + instanceID := ps.MustGetParameter("instance_id") + + var reqBody = struct { + Host string `json:"host"` + Schema string `json:"schema"` + CredentialID string `json:"credential_id"` + BasicAuth *model.BasicAuth `json:"basic_auth"` + }{} + + err := h.DecodeJSON(req, &reqBody) + if err != nil { + panic(err) + } + + if reqBody.BasicAuth == nil { + //TODO remove `manual` + if reqBody.CredentialID != "" && reqBody.CredentialID != "manual" { + cred, err := common2.GetCredential(reqBody.CredentialID) + if err != nil { + panic(err) + } + auth, err := cred.DecodeBasicAuth() + reqBody.BasicAuth = auth + } + } + + _, instance, err := GetRuntimeInstanceByID(instanceID) + if err != nil { + panic(err) + } + + esConfig := elastic2.ElasticsearchConfig{Host: reqBody.Host, Schema: reqBody.Schema, BasicAuth: reqBody.BasicAuth} + body := util.MustToJSONBytes(esConfig) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + req1 := &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/try_connect", + Context: ctx, + Body: body, + } + if reqBody.BasicAuth != nil { + req1.SetBasicAuth(reqBody.BasicAuth.Username, reqBody.BasicAuth.Password.Get()) + } + + res, err := ProxyAgentRequest("runtime", instance.GetEndpoint(), req1, nil) + if err != nil { + panic(err) + } + + //res, err := ProxyRequestToRuntimeInstance(instance.Endpoint, "POST", "/elasticsearch/try_connect", + // body, int64(len(body)), reqBody.BasicAuth) + // + //if err != nil { + // panic(err) + //} + + h.WriteHeader(w, res.StatusCode) + h.Write(w, res.Body) +} + +// TODO check permission by user +func GetRuntimeInstanceByID(instanceID string) (bool, *model.Instance, error) { + obj := model.Instance{} + obj.ID = instanceID + exists, err := orm.Get(&obj) + if !exists || err != nil { + if !exists { + err = fmt.Errorf("instance not found") + } + return exists, nil, err + } + return true, &obj, err +} diff --git a/plugin/managed/server/manager.go b/plugin/managed/server/manager.go new file mode 100644 index 00000000..31dc24a6 --- /dev/null +++ b/plugin/managed/server/manager.go @@ -0,0 +1,106 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package server + +import ( + "crypto/tls" + "fmt" + log "github.com/cihub/seelog" + "infini.sh/console/core" + "infini.sh/console/plugin/managed/common" + "infini.sh/framework/core/api" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "net" + "net/http" + "net/url" + "sync" + "time" +) + +type APIHandler struct { + core.Handler +} + +var serverInit = sync.Once{} +var configRepo common.ConfigRepo +var handler = APIHandler{} + +func init() { + + api.HandleAPIMethod(api.POST, common.SYNC_API, handler.syncConfigs) //client sync configs from config servers + api.HandleAPIMethod(api.POST, "/configs/_reload", handler.refreshConfigsRepo) //client sync configs from config servers + //delegate api to instances + api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) { + log.Debug(req.RequestURI) + endpoint := req.URL.Query().Get("endpoint") + path := req.URL.Query().Get("path") + var tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + } + target, err := url.Parse(endpoint) + if err != nil { + panic(err) + } + newURL, err := url.Parse(path) + if err != nil { + panic(err) + } + req.URL.Path = newURL.Path + req.URL.RawPath = newURL.RawPath + req.URL.RawQuery = "" + req.RequestURI = req.URL.RequestURI() + req.Header.Set("HOST", target.Host) + req.Host = target.Host + wsProxy := NewSingleHostReverseProxy(target) + wsProxy.Dial = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial + wsProxy.TLSClientConfig = tlsConfig + wsProxy.ServeHTTP(w, req) + }) +} + +var mTLSClient *http.Client //TODO get mTLSClient +var initOnce = sync.Once{} + +func ProxyAgentRequest(tag, endpoint string, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + var err error + var res *util.Result + + initOnce.Do(func() { + cfg := global.Env().GetClientConfigByEndpoint(tag, endpoint) + if cfg != nil { + hClient, err := api.NewHTTPClient(cfg) + if err != nil { + panic(err) + } + mTLSClient = hClient + } + }) + + req.Url = endpoint + req.Path + + res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + if err != nil || res.StatusCode != 200 { + body := "" + if res != nil { + body = string(res.Body) + } + return res, errors.New(fmt.Sprintf("request error: %v, %v", err, body)) + } + + if res != nil { + if res.Body != nil { + if responseObjectToUnMarshall != nil { + return res, util.FromJSONBytes(res.Body, responseObjectToUnMarshall) + } + } + } + + return res, err +} diff --git a/plugin/managed/server/script.go b/plugin/managed/server/script.go new file mode 100644 index 00000000..30645c5f --- /dev/null +++ b/plugin/managed/server/script.go @@ -0,0 +1,153 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package server + +import ( + "fmt" + log "github.com/cihub/seelog" + "github.com/valyala/fasttemplate" + "infini.sh/console/core/security" + "infini.sh/console/modules/agent/common" + common2 "infini.sh/console/plugin/managed/common" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" + "net/url" + "os" + + "net/http" + "path" + "strings" + "time" +) + +type Token struct { + CreatedAt time.Time + UserID string +} + +const ExpiredIn = time.Millisecond * 1000 * 60 * 60 + +var expiredTokenCache = util.NewCacheWithExpireOnAdd(ExpiredIn, 100) + +func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + claims, ok := req.Context().Value("user").(*security.UserClaims) + if !ok { + h.WriteError(w, "user not found", http.StatusInternalServerError) + return + } + agCfg := common.GetAgentConfig() + if agCfg == nil || agCfg.Setup == nil { + h.WriteError(w, "agent setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) + return + } + var ( + t *Token + tokenStr string + ) + + //TODO: get location from request, validate it + location := "/opt/agent" + + tokenStr = util.GetUUID() + t = &Token{ + CreatedAt: time.Now(), + UserID: claims.UserId, + } + + expiredTokenCache.Put(tokenStr, t) + consoleEndpoint := agCfg.Setup.ConsoleEndpoint + if consoleEndpoint == "" { + consoleEndpoint = getDefaultEndpoint(req) + } + + endpoint, err := url.JoinPath(consoleEndpoint, common2.GET_INSTALL_SCRIPT_API) + if err != nil { + panic(err) + } + + h.WriteJSON(w, util.MapStr{ + "script": fmt.Sprintf(`curl -ksSL %s?token=%s |sudo bash -s -- -u %s -t %v`, + endpoint, tokenStr, agCfg.Setup.DownloadURL, location), + "token": tokenStr, + "expired_at": t.CreatedAt.Add(ExpiredIn), + }, http.StatusOK) +} + +func getDefaultEndpoint(req *http.Request) string { + scheme := "http" + if req.TLS != nil { + scheme = "https" + } + return fmt.Sprintf("%s://%s", scheme, req.Host) +} + +func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + tokenStr := h.GetParameter(req, "token") + if strings.TrimSpace(tokenStr) == "" { + h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + + v := expiredTokenCache.Get(tokenStr) + if v == nil { + h.WriteError(w, "token is invalid", http.StatusUnauthorized) + return + } + + t, ok := v.(*Token) + if !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) { + expiredTokenCache.Delete(tokenStr) + h.WriteError(w, "token was expired", http.StatusUnauthorized) + return + } + + agCfg := common.GetAgentConfig() + caCert, clientCertPEM, clientKeyPEM, err := common.GenerateServerCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + scriptTplPath := path.Join(global.Env().GetConfigDir(), "install_agent.tpl") + buf, err := os.ReadFile(scriptTplPath) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + tpl := fasttemplate.New(string(buf), "{{", "}}") + downloadURL := agCfg.Setup.DownloadURL + if downloadURL == "" { + downloadURL = "https://release.infinilabs.com/agent/stable/" + } + + port := agCfg.Setup.Port + if port == "" { + port = "8080" + } + + consoleEndpoint := agCfg.Setup.ConsoleEndpoint + if consoleEndpoint == "" { + consoleEndpoint = getDefaultEndpoint(req) + } + + _, err = tpl.Execute(w, map[string]interface{}{ + "base_url": agCfg.Setup.DownloadURL, + "console_endpoint": consoleEndpoint, + "client_crt": clientCertPEM, + "client_key": clientKeyPEM, + "ca_crt": caCert, + "port": port, + "token": tokenStr, + }) + + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + } +} diff --git a/plugin/managed/server/websocket_proxy.go b/plugin/managed/server/websocket_proxy.go new file mode 100644 index 00000000..a1450170 --- /dev/null +++ b/plugin/managed/server/websocket_proxy.go @@ -0,0 +1,186 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package server + +import ( + "crypto/tls" + log "github.com/cihub/seelog" + "io" + "net" + "net/http" + "net/url" + "strings" +) + +// ReverseProxy is a WebSocket reverse proxy. It will not work with a regular +// HTTP request, so it is the caller's responsiblity to ensure the incoming +// request is a WebSocket request. +type ReverseProxy struct { + // Director must be a function which modifies + // the request into a new request to be sent + // using Transport. Its response is then copied + // back to the original client unmodified. + Director func(*http.Request) + + // Dial specifies the dial function for dialing the proxied + // server over tcp. + // If Dial is nil, net.Dial is used. + Dial func(network, addr string) (net.Conn, error) + + // TLSClientConfig specifies the TLS configuration to use for 'wss'. + // If nil, the default configuration is used. + TLSClientConfig *tls.Config + + // ErrorLog specifies an optional logger for errors + // that occur when attempting to proxy the request. + // If nil, logging goes to os.Stderr via the log package's + // standard logger. + ErrorLog *log.LoggerInterface +} + +// stolen from net/http/httputil. singleJoiningSlash ensures that the route +// '/a/' joined with '/b' becomes '/a/b'. +func singleJoiningSlash(a, b string) string { + aslash := strings.HasSuffix(a, "/") + bslash := strings.HasPrefix(b, "/") + switch { + case aslash && bslash: + return a + b[1:] + case !aslash && !bslash: + return a + "/" + b + } + return a + b +} + +// NewSingleHostReverseProxy returns a new websocket ReverseProxy. The path +// rewrites follow the same rules as the httputil.ReverseProxy. If the target +// url has the path '/foo' and the incoming request '/bar', the request path +// will be updated to '/foo/bar' before forwarding. +// Scheme should specify if 'ws' or 'wss' should be used. +func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { + targetQuery := target.RawQuery + director := func(req *http.Request) { + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) + if targetQuery == "" || req.URL.RawQuery == "" { + req.URL.RawQuery = targetQuery + req.URL.RawQuery + } else { + req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery + } + } + return &ReverseProxy{Director: director} +} + +// Function to implement the http.Handler interface. +func (p *ReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + logFunc := log.Errorf + + if !IsWebSocketRequest(r) { + http.Error(w, "Cannot handle non-WebSocket requests", 500) + logFunc("Received a request that was not a WebSocket request") + return + } + + outreq := new(http.Request) + // shallow copying + *outreq = *r + p.Director(outreq) + host := outreq.URL.Host + + if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + // If we aren't the first proxy retain prior + // X-Forwarded-For information as a comma+space + // separated list and fold multiple headers into one. + if prior, ok := outreq.Header["X-Forwarded-For"]; ok { + clientIP = strings.Join(prior, ", ") + ", " + clientIP + } + outreq.Header.Set("X-Forwarded-For", clientIP) + } + + dial := p.Dial + if dial == nil { + dial = net.Dial + } + + // if host does not specify a port, use the default http port + if !strings.Contains(host, ":") { + if outreq.URL.Scheme == "wss" { + host = host + ":443" + } else { + host = host + ":80" + } + } + + if outreq.URL.Scheme == "wss" { + var tlsConfig *tls.Config + if p.TLSClientConfig == nil { + tlsConfig = &tls.Config{InsecureSkipVerify: true} + } else { + tlsConfig = p.TLSClientConfig + } + dial = func(network, address string) (net.Conn, error) { + return tls.Dial("tcp", host, tlsConfig) + } + } + + d, err := dial("tcp", host) + if err != nil { + http.Error(w, "Error forwarding request.", 500) + logFunc("Error dialing websocket backend %s: %v", outreq.URL, err) + return + } + // All request generated by the http package implement this interface. + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Not a hijacker?", 500) + return + } + // Hijack() tells the http package not to do anything else with the connection. + // After, it bcomes this functions job to manage it. `nc` is of type *net.Conn. + nc, _, err := hj.Hijack() + if err != nil { + logFunc("Hijack error: %v", err) + return + } + defer nc.Close() // must close the underlying net connection after hijacking + defer d.Close() + + // write the modified incoming request to the dialed connection + err = outreq.Write(d) + if err != nil { + logFunc("Error copying request to target: %v", err) + return + } + errc := make(chan error, 2) + cp := func(dst io.Writer, src io.Reader) { + _, err := io.Copy(dst, src) + errc <- err + } + go cp(d, nc) + go cp(nc, d) + <-errc +} + +// IsWebSocketRequest returns a boolean indicating whether the request has the +// headers of a WebSocket handshake request. +func IsWebSocketRequest(r *http.Request) bool { + contains := func(key, val string) bool { + vv := strings.Split(r.Header.Get(key), ",") + for _, v := range vv { + if val == strings.ToLower(strings.TrimSpace(v)) { + return true + } + } + return false + } + if !contains("Connection", "upgrade") { + return false + } + if !contains("Upgrade", "websocket") { + return false + } + return true +} diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index e20ca01e..f2c01539 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -3,10 +3,13 @@ package task import ( "bufio" "bytes" + "context" "crypto/md5" "encoding/hex" "fmt" + "infini.sh/console/core/security" "infini.sh/framework/lib/go-ucfg" + elastic2 "infini.sh/framework/modules/elastic" "io" "io/ioutil" "net/http" @@ -19,8 +22,9 @@ import ( log "github.com/cihub/seelog" "github.com/valyala/fasttemplate" "golang.org/x/crypto/bcrypt" + elastic3 "infini.sh/console/modules/elastic/api" + security2 "infini.sh/console/modules/security" "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/credential" "infini.sh/framework/core/elastic" @@ -36,12 +40,8 @@ import ( "infini.sh/framework/core/util" "infini.sh/framework/lib/fasthttp" keystore2 "infini.sh/framework/lib/keystore" - elastic2 "infini.sh/framework/modules/elastic" "infini.sh/framework/modules/elastic/adapter" - elastic3 "infini.sh/framework/modules/elastic/api" elastic1 "infini.sh/framework/modules/elastic/common" - "infini.sh/framework/modules/security" - _ "infini.sh/framework/modules/security" "infini.sh/framework/plugins/replay" ) @@ -315,7 +315,7 @@ func (module *Module) initTempClient(r *http.Request) (error, elastic.API, Setup elastic.UpdateConfig(cfg) elastic.UpdateClient(cfg, client) - health, err := client.ClusterHealth() + health, err := client.ClusterHealth(context.Background()) if err != nil { return err, nil, request } @@ -438,7 +438,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http } } //处理索引 - security.InitSchema() //register user index + security2.InitSchema() //register user index elastic2.InitSchema() toSaveCfg := cfg oldCfg := elastic.ElasticsearchConfig{} @@ -505,7 +505,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http if request.BootstrapUsername != "" && request.BootstrapPassword != "" { //Save bootstrap user - user := rbac.User{} + user := security.User{} user.ID = "default_user_" + request.BootstrapUsername user.Username = request.BootstrapUsername user.Nickname = request.BootstrapUsername @@ -516,10 +516,10 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http } user.Password = string(hash) - role := []rbac.UserRole{} - role = append(role, rbac.UserRole{ - ID: rbac.RoleAdminName, - Name: rbac.RoleAdminName, + role := []security.UserRole{} + role = append(role, security.UserRole{ + ID: security.RoleAdminName, + Name: security.RoleAdminName, }) user.Roles = role now := time.Now() @@ -800,14 +800,14 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, request.Cluster.Host = uri.Host request.Cluster.Schema = uri.Scheme } + var setupHTTPPool = fasthttp.NewRequestResponsePool("setup") + req := setupHTTPPool.AcquireRequest() + res := setupHTTPPool.AcquireResponse() - req := fasthttp.AcquireRequest() - res := fasthttp.AcquireResponse() + defer setupHTTPPool.ReleaseRequest(req) + defer setupHTTPPool.ReleaseResponse(res) - defer fasthttp.ReleaseRequest(req) - defer fasthttp.ReleaseResponse(res) - - _, err, _ = replay.ReplayLines(pipeline.AcquireContext(pipeline.PipelineConfigV2{}), lines, request.Cluster.Schema, request.Cluster.Host, request.Cluster.Username, request.Cluster.Password) + _, err, _ = replay.ReplayLines(req, res, pipeline.AcquireContext(pipeline.PipelineConfigV2{}), lines, request.Cluster.Schema, request.Cluster.Host, request.Cluster.Username, request.Cluster.Password) if err != nil { module.WriteJSON(w, util.MapStr{ "success": false, diff --git a/plugin/task_manager/api.go b/plugin/task_manager/api.go deleted file mode 100644 index c1db839d..00000000 --- a/plugin/task_manager/api.go +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package task_manager - -import ( - "infini.sh/framework/core/api" - "infini.sh/framework/core/api/rbac/enum" -) - -func InitAPI() { - handler := APIHandler{} - api.HandleAPIMethod(api.GET, "/migration/data/_search", handler.RequirePermission(handler.searchTask("cluster_migration"), enum.PermissionMigrationTaskRead)) - api.HandleAPIMethod(api.POST, "/migration/data", handler.RequirePermission(handler.createDataMigrationTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.DELETE, "/migration/data/:task_id", handler.RequirePermission(handler.deleteTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.POST, "/migration/data/:task_id/_start", handler.RequirePermission(handler.startTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.POST, "/migration/data/:task_id/_stop", handler.RequirePermission(handler.stopTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.POST, "/migration/data/:task_id/_pause", handler.RequirePermission(handler.pauseTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.POST, "/migration/data/:task_id/_resume", handler.RequirePermission(handler.resumeTask, enum.PermissionMigrationTaskWrite)) - api.HandleAPIMethod(api.GET, "/migration/data/:task_id/info", handler.RequirePermission(handler.getDataMigrationTaskInfo, enum.PermissionMigrationTaskRead)) - api.HandleAPIMethod(api.GET, "/migration/data/:task_id/info/:index", handler.RequirePermission(handler.getDataMigrationTaskOfIndex, enum.PermissionMigrationTaskRead)) - api.HandleAPIMethod(api.GET, "/migration/data/:task_id/logging/:index", handler.RequirePermission(handler.searchIndexLevelTaskLogging, enum.PermissionMigrationTaskRead)) - api.HandleAPIMethod(api.GET, "/migration/data/_search_values", handler.RequirePermission(handler.searchTaskFieldValues("cluster_migration"), enum.PermissionMigrationTaskRead)) - api.HandleAPIMethod(api.POST, "/migration/data/partition/_restart", handler.RequirePermission(handler.restartAllFailedPartitions, enum.PermissionMigrationTaskWrite)) - - api.HandleAPIMethod(api.GET, "/comparison/data/_search", handler.RequirePermission(handler.searchTask("cluster_comparison"), enum.PermissionComparisonTaskRead)) - api.HandleAPIMethod(api.POST, "/comparison/data", handler.RequirePermission(handler.createDataComparisonTask, enum.PermissionComparisonTaskWrite)) - api.HandleAPIMethod(api.DELETE, "/comparison/data/:task_id", handler.RequirePermission(handler.deleteTask, enum.PermissionComparisonTaskWrite)) - api.HandleAPIMethod(api.GET, "/comparison/data/:task_id/info", handler.RequirePermission(handler.getDataComparisonTaskInfo, enum.PermissionComparisonTaskRead)) - api.HandleAPIMethod(api.GET, "/comparison/data/:task_id/info/:index", handler.RequirePermission(handler.getDataComparisonTaskOfIndex, enum.PermissionComparisonTaskRead)) - api.HandleAPIMethod(api.POST, "/comparison/data/:task_id/_start", handler.RequirePermission(handler.startTask, enum.PermissionComparisonTaskWrite)) - api.HandleAPIMethod(api.POST, "/comparison/data/:task_id/_stop", handler.RequirePermission(handler.stopTask, enum.PermissionComparisonTaskWrite)) - api.HandleAPIMethod(api.POST, "/comparison/data/:task_id/_pause", handler.RequirePermission(handler.pauseTask, enum.PermissionComparisonTaskWrite)) - api.HandleAPIMethod(api.POST, "/comparison/data/:task_id/_resume", handler.RequirePermission(handler.resumeTask, enum.PermissionComparisonTaskWrite)) - - api.HandleAPIMethod(api.POST, "/migration/data/_validate", handler.RequireLogin(handler.validateDataMigration)) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/index/:index/_partition", handler.getIndexPartitionInfo) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/index/:index/_count", handler.countDocuments) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/index/:index/_refresh", handler.refreshIndex) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/index/:index/_init", handler.initIndex) - -} - -type APIHandler struct { - api.Handler -} diff --git a/plugin/task_manager/cluster_comparison/cluster_comparison.go b/plugin/task_manager/cluster_comparison/cluster_comparison.go deleted file mode 100644 index 72a07722..00000000 --- a/plugin/task_manager/cluster_comparison/cluster_comparison.go +++ /dev/null @@ -1,467 +0,0 @@ -package cluster_comparison - -import ( - "context" - "fmt" - "time" - - log "github.com/cihub/seelog" - "infini.sh/console/model" - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -type processor struct { - Elasticsearch string - IndexName string - scheduler migration_model.Scheduler -} - -func NewProcessor(elasticsearch, indexName string, scheduler migration_model.Scheduler) migration_model.Processor { - return &processor{ - Elasticsearch: elasticsearch, - IndexName: indexName, - scheduler: scheduler, - } -} - -func (p *processor) Process(t *task.Task) (err error) { - switch t.Status { - case task.StatusReady: - // split & schedule index_comparison tasks - err = p.handleReadyMajorTask(t) - case task.StatusRunning: - // check index_comparison tasks status - err = p.handleRunningMajorTask(t) - case task.StatusPendingStop: - // mark index_comparison as pending_stop - err = p.handlePendingStopMajorTask(t) - } - return err -} - -func (p *processor) handleReadyMajorTask(taskItem *task.Task) error { - if ok, _ := util.ExtractBool(taskItem.Metadata.Labels["is_split"]); !ok { - return p.splitMajorTask(taskItem) - } - // update status of subtask to ready - err := migration_util.UpdateStoppedChildTasksToReady(taskItem, "index_comparison") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - taskItem.RetryTimes++ - taskItem.StartTimeInMillis = time.Now().UnixMilli() - taskItem.Status = task.StatusRunning - taskItem.Metadata.Labels["total_diff_docs"] = 0 - taskItem.Metadata.Labels["only_in_source"] = 0 - taskItem.Metadata.Labels["only_in_target"] = 0 - taskItem.Metadata.Labels["diff_both"] = 0 - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("cluster comparison task [%s] started", taskItem.ID)) - p.sendMajorTaskNotification(taskItem) - return nil -} - -func (p *processor) splitMajorTask(taskItem *task.Task) error { - clusterComparisonTask := migration_model.ClusterComparisonTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &clusterComparisonTask) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return err - } - esSourceClient := elastic.GetClient(clusterComparisonTask.Cluster.Source.Id) - esTargetClient := elastic.GetClient(clusterComparisonTask.Cluster.Target.Id) - - err = migration_util.DeleteChildTasks(taskItem.ID, "index_comparison") - if err != nil { - log.Warnf("failed to clear child tasks, err: %v", err) - return nil - } - - current := migration_util.GetMapIntValue(taskItem.Metadata.Labels, "next_run_time") - var step time.Duration - if clusterComparisonTask.Settings.Execution.Repeat != nil { - step = clusterComparisonTask.Settings.Execution.Repeat.Interval - } - - var pids []string - pids = append(pids, taskItem.ParentId...) - pids = append(pids, taskItem.ID) - - var sourceTotalDocs int64 - var targetTotalDocs int64 - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Second*60)) - defer cancel() - - for i, index := range clusterComparisonTask.Indices { - sourceDump := migration_model.IndexComparisonDumpConfig{ - ClusterId: clusterComparisonTask.Cluster.Source.Id, - Indices: index.Source.Name, - DocCount: index.Source.Docs, - SliceSize: clusterComparisonTask.Settings.Dump.SliceSize, - BatchSize: clusterComparisonTask.Settings.Dump.Docs, - PartitionSize: clusterComparisonTask.Settings.Dump.PartitionSize, - ScrollTime: clusterComparisonTask.Settings.Dump.Timeout, - } - - // TODO: dump_hash can only handle 1G file - if sourceDump.PartitionSize <= 0 { - sourceDump.PartitionSize = 1 - } - - if v, ok := index.RawFilter.(string); ok { - sourceDump.QueryString = v - } else { - var must []interface{} - if index.RawFilter != nil { - must = append(must, index.RawFilter) - } - if index.Incremental != nil { - incrementalFilter, err := index.Incremental.BuildFilter(current, step) - if err != nil { - return err - } - must = append(must, incrementalFilter) - } - if len(must) > 0 { - sourceDump.QueryDSL = util.MapStr{ - "bool": util.MapStr{ - "must": must, - }, - } - } - } - - targetDump := sourceDump - targetDump.ClusterId = clusterComparisonTask.Cluster.Target.Id - targetDump.Indices = index.Target.Name - targetDump.DocCount = index.Target.Docs - - var countQuery = util.MapStr{} - if sourceDump.QueryDSL != nil { - countQuery = util.MapStr{ - "query": sourceDump.QueryDSL, - } - } - sourceCount, err := esSourceClient.Count(ctx, index.Source.Name, util.MustToJSONBytes(countQuery)) - if err != nil { - return err - } - clusterComparisonTask.Indices[i].Source.Docs = sourceCount.Count - sourceTotalDocs += sourceCount.Count - sourceDump.DocCount = sourceCount.Count - - targetCount, err := esTargetClient.Count(ctx, index.Target.Name, util.MustToJSONBytes(countQuery)) - if err != nil { - return err - } - clusterComparisonTask.Indices[i].Target.Docs = targetCount.Count - targetTotalDocs += targetCount.Count - targetDump.DocCount = targetCount.Count - - if index.Partition == nil { - indexComparisonTask := task.Task{ - ParentId: pids, - Cancellable: true, - Runnable: false, - Status: task.StatusReady, - Metadata: task.Metadata{ - Type: "index_comparison", - Labels: util.MapStr{ - "business_id": "index_comparison", - "source_cluster_id": clusterComparisonTask.Cluster.Source.Id, - "target_cluster_id": clusterComparisonTask.Cluster.Target.Id, - "partition_count": 1, - "index_name": index.Source.Name, - "unique_index_name": index.Source.GetUniqueIndexName(), - }, - }, - ConfigString: util.MustToJSON(migration_model.IndexComparisonTaskConfig{ - Source: sourceDump, - Target: targetDump, - Execution: clusterComparisonTask.Settings.Execution, - }), - } - indexComparisonTask.ID = util.GetUUID() - - err = orm.Create(nil, &indexComparisonTask) - if err != nil { - return fmt.Errorf("store index comparison task error: %w", err) - } - continue - } - - sourcePartitionQ := &elastic.PartitionQuery{ - IndexName: sourceDump.Indices, - FieldName: index.Partition.FieldName, - FieldType: index.Partition.FieldType, - Step: index.Partition.Step, - } - if sourceDump.QueryDSL != nil { - sourcePartitionQ.Filter = sourceDump.QueryDSL - } - sourcePartitions, err := elastic.GetPartitions(sourcePartitionQ, esSourceClient) - if err != nil { - return err - } - - targetPartitionQ := &elastic.PartitionQuery{ - IndexName: targetDump.Indices, - FieldName: index.Partition.FieldName, - FieldType: index.Partition.FieldType, - Step: index.Partition.Step, - } - if targetDump.QueryDSL != nil { - targetPartitionQ.Filter = targetDump.QueryDSL - } - targetPartitions, err := elastic.GetPartitions(targetPartitionQ, esTargetClient) - if err != nil { - return err - } - - partitions := elastic.MergePartitions(sourcePartitions, targetPartitions, index.Partition.FieldName, index.Partition.FieldType, targetPartitionQ.Filter) - - if len(partitions) == 0 { - continue - } - - var ( - partitionID int - ) - for _, partition := range partitions { - partitionID++ - partitionSourceDump := sourceDump - partitionSourceDump.Start = partition.Start - partitionSourceDump.End = partition.End - partitionSourceDump.DocCount = partition.Docs - partitionSourceDump.Step = index.Partition.Step - partitionSourceDump.PartitionId = partitionID - partitionSourceDump.QueryDSL = partition.Filter - partitionSourceDump.QueryString = "" - - partitionTargetDump := partitionSourceDump - partitionTargetDump.ClusterId = clusterComparisonTask.Cluster.Target.Id - partitionTargetDump.Indices = index.Target.Name - - partitionComparisonTask := task.Task{ - ParentId: pids, - Cancellable: false, - Runnable: true, - Status: task.StatusReady, - Metadata: task.Metadata{ - Type: "index_comparison", - Labels: util.MapStr{ - "business_id": "index_comparison", - "source_cluster_id": clusterComparisonTask.Cluster.Source.Id, - "target_cluster_id": clusterComparisonTask.Cluster.Target.Id, - "index_name": index.Source.Name, - "unique_index_name": index.Source.GetUniqueIndexName(), - }, - }, - ConfigString: util.MustToJSON(migration_model.IndexComparisonTaskConfig{ - Source: partitionSourceDump, - Target: partitionTargetDump, - Execution: clusterComparisonTask.Settings.Execution, - }), - } - partitionComparisonTask.ID = util.GetUUID() - err = orm.Create(nil, &partitionComparisonTask) - if err != nil { - return fmt.Errorf("store index comparison task (partition) error: %w", err) - } - } - } - - taskItem.ConfigString = util.MustToJSON(clusterComparisonTask) - taskItem.Metadata.Labels["is_split"] = true - taskItem.Metadata.Labels["source_total_docs"] = sourceTotalDocs - taskItem.Metadata.Labels["target_total_docs"] = targetTotalDocs - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("major task [%s] splitted", taskItem.ID)) - return nil -} - -func (p *processor) handleRunningMajorTask(taskItem *task.Task) error { - taskStatus, err := p.getMajorTaskState(taskItem) - if err != nil { - return err - } - if !(taskStatus == task.StatusComplete || taskStatus == task.StatusError) { - return nil - } - - var errMsg string - - if taskStatus == task.StatusError { - errMsg = "index comparison(s) failed" - } - - if errMsg == "" { - taskItem.Status = task.StatusComplete - } else { - taskItem.Status = task.StatusError - } - tn := time.Now() - taskItem.CompletedTime = &tn - p.sendMajorTaskNotification(taskItem) - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("cluster comparison task [%s] finished with status [%s]", taskItem.ID, taskItem.Status)) - return nil -} - -func (p *processor) getMajorTaskState(majorTask *task.Task) (string, error) { - query := util.MapStr{ - "size": 0, - "aggs": util.MapStr{ - "count": util.MapStr{ - "terms": util.MapStr{ - "field": "*", - }, - }, - "grp": util.MapStr{ - "terms": util.MapStr{ - "field": "status", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": majorTask.ID, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_comparison", - }, - }, - }, - }, - }, - }, - } - esClient := elastic.GetClient(p.Elasticsearch) - res, err := esClient.SearchWithRawQueryDSL(p.IndexName, util.MustToJSONBytes(query)) - if err != nil { - log.Errorf("search es failed, err: %v", err) - return "", nil - } - var ( - hasError bool - ) - for _, bk := range res.Aggregations["grp"].Buckets { - statusKey, _ := util.ExtractString(bk["key"]) - if migration_util.IsRunningState(statusKey) { - return task.StatusRunning, nil - } - if statusKey == task.StatusError { - hasError = true - } - } - if hasError { - return task.StatusError, nil - } - return task.StatusComplete, nil -} - -func (p *processor) handlePendingStopMajorTask(taskItem *task.Task) error { - err := migration_util.UpdatePendingChildTasksToPendingStop(taskItem, "index_comparison") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - - tasks, err := migration_util.GetPendingChildTasks(taskItem.ID, "index_comparison") - if err != nil { - log.Errorf("failed to get sub tasks, err: %v", err) - return nil - } - - // all subtask stopped or error or complete - if len(tasks) == 0 { - taskItem.Status = task.StatusStopped - p.sendMajorTaskNotification(taskItem) - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("cluster comparison task [%s] stopped", taskItem.ID)) - // NOTE: we don't know how many running index_comparison's stopped, so do a refresh from ES - p.scheduler.RefreshInstanceJobsFromES() - } - return nil -} - -func (p *processor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.Elasticsearch) - _, err := esClient.Index(p.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -func (p *processor) sendMajorTaskNotification(taskItem *task.Task) { - // don't send notification for repeating child tasks - if len(taskItem.ParentId) > 0 { - return - } - - config := migration_model.ClusterComparisonTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &config) - if err != nil { - log.Errorf("failed to parse config info from major task, id: %s, err: %v", taskItem.ID, err) - return - } - - creatorID := config.Creator.Id - - var title, body string - body = fmt.Sprintf("From Cluster: [%s (%s)], To Cluster: [%s (%s)]", config.Cluster.Source.Id, config.Cluster.Source.Name, config.Cluster.Target.Id, config.Cluster.Target.Name) - link := fmt.Sprintf("/#/data_tools/comparison/%s/detail", taskItem.ID) - switch taskItem.Status { - case task.StatusReady: - log.Debugf("skip sending notification for ready task, id: %s", taskItem.ID) - return - case task.StatusStopped: - title = fmt.Sprintf("Data Comparison Stopped") - case task.StatusComplete: - title = fmt.Sprintf("Data Comparison Completed") - case task.StatusError: - title = fmt.Sprintf("Data Comparison Failed") - case task.StatusRunning: - title = fmt.Sprintf("Data Comparison Started") - default: - log.Warnf("skip sending notification for invalid task status, id: %s", taskItem.ID) - return - } - notification := &model.Notification{ - UserId: util.ToString(creatorID), - Type: model.NotificationTypeNotification, - MessageType: model.MessageTypeComparison, - Status: model.NotificationStatusNew, - Title: title, - Body: body, - Link: link, - } - err = orm.Create(nil, notification) - if err != nil { - log.Errorf("failed to create notification, err: %v", err) - return - } - return -} diff --git a/plugin/task_manager/cluster_comparison/orm.go b/plugin/task_manager/cluster_comparison/orm.go deleted file mode 100644 index 51d92911..00000000 --- a/plugin/task_manager/cluster_comparison/orm.go +++ /dev/null @@ -1,132 +0,0 @@ -package cluster_comparison - -import ( - "errors" - "fmt" - - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/api/rbac" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func RepeatTask(oldTask *task.Task) (*task.Task, error) { - config := migration_model.ClusterComparisonTaskConfig{} - err := migration_util.GetTaskConfig(oldTask, &config) - if err != nil { - return nil, err - } - - t, err := buildTask(&config, &rbac.ShortUser{ - Username: config.Creator.Name, - UserId: config.Creator.Id, - }, true) - if err != nil { - return nil, err - } - t.ParentId = []string{oldTask.ID} - if len(oldTask.ParentId) > 0 { - t.ParentId = oldTask.ParentId - } - - migration_util.CopyRepeatState(oldTask.Metadata.Labels, t.Metadata.Labels) - err = migration_util.UpdateRepeatState(config.Settings.Execution.Repeat, t.Metadata.Labels) - if err != nil { - return nil, fmt.Errorf("repeat invalid: %v", err) - } - - err = orm.Create(nil, t) - if err != nil { - return nil, err - } - return t, nil -} - -func CreateTask(config *migration_model.ClusterComparisonTaskConfig, creator *rbac.ShortUser) (*task.Task, error) { - t, err := buildTask(config, creator, false) - if err != nil { - return nil, err - } - - err = migration_util.UpdateRepeatState(config.Settings.Execution.Repeat, t.Metadata.Labels) - if err != nil { - return nil, fmt.Errorf("repeat invalid: %v", err) - } - - ctx := &orm.Context{ - Refresh: "wait_for", - } - err = orm.Create(ctx, t) - if err != nil { - return nil, err - } - return t, nil -} - -func buildTask(config *migration_model.ClusterComparisonTaskConfig, creator *rbac.ShortUser, repeat bool) (*task.Task, error) { - if len(config.Indices) == 0 { - return nil, errors.New("indices must not be empty") - } - if creator == nil { - return nil, errors.New("missing creator info") - } - config.Creator.Name = creator.Username - config.Creator.Id = creator.UserId - - srcClusterCfg := elastic.GetConfig(config.Cluster.Source.Id) - config.Cluster.Source.Distribution = srcClusterCfg.Distribution - dstClusterCfg := elastic.GetConfig(config.Cluster.Target.Id) - config.Cluster.Target.Distribution = dstClusterCfg.Distribution - - clearTaskConfig(config) - - var sourceTotalDocs int64 - var targetTotalDocs int64 - - for _, index := range config.Indices { - if index.Incremental != nil { - if repeat { - index.Incremental.Full = false - } else { - index.Incremental.Full = true - } - } - sourceTotalDocs += index.Source.Docs - targetTotalDocs += index.Target.Docs - } - - t := task.Task{ - Metadata: task.Metadata{ - Type: "cluster_comparison", - Labels: util.MapStr{ - "business_id": "cluster_comparison", - "source_cluster_id": config.Cluster.Source.Id, - "target_cluster_id": config.Cluster.Target.Id, - "source_total_docs": sourceTotalDocs, - "target_total_docs": targetTotalDocs, - "permit_nodes": config.Settings.Execution.Nodes.Permit, - "name": config.Name, - }, - }, - Cancellable: true, - Runnable: false, - Status: task.StatusInit, - ConfigString: util.MustToJSON(config), - } - if len(config.Tags) > 0 { - t.Metadata.Labels["tags"] = config.Tags - } - t.ID = util.GetUUID() - return &t, nil -} - -// sync with getDataComparisonTaskInfo -func clearTaskConfig(config *migration_model.ClusterComparisonTaskConfig) { - for i := range config.Indices { - config.Indices[i].ScrollPercent = 0 - config.Indices[i].ErrorPartitions = 0 - } -} diff --git a/plugin/task_manager/cluster_migration/cluster_migration.go b/plugin/task_manager/cluster_migration/cluster_migration.go deleted file mode 100644 index 95636135..00000000 --- a/plugin/task_manager/cluster_migration/cluster_migration.go +++ /dev/null @@ -1,501 +0,0 @@ -package cluster_migration - -import ( - "context" - "fmt" - "time" - - log "github.com/cihub/seelog" - - "infini.sh/console/model" - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic/common" -) - -type processor struct { - Elasticsearch string - IndexName string - scheduler migration_model.Scheduler -} - -func NewProcessor(elasticsearch, indexName string, scheduler migration_model.Scheduler) migration_model.Processor { - return &processor{ - Elasticsearch: elasticsearch, - IndexName: indexName, - scheduler: scheduler, - } -} - -func (p *processor) Process(t *task.Task) (err error) { - switch t.Status { - case task.StatusReady: - // split & schedule index_migration tasks - err = p.handleReadyMajorTask(t) - case task.StatusRunning: - // check index_migration tasks status - err = p.handleRunningMajorTask(t) - case task.StatusPendingStop: - // mark index_migrations as pending_stop - err = p.handlePendingStopMajorTask(t) - } - return err -} - -func (p *processor) handleReadyMajorTask(taskItem *task.Task) error { - if ok, _ := util.ExtractBool(taskItem.Metadata.Labels["is_split"]); !ok { - return p.splitMajorMigrationTask(taskItem) - } - // update status of subtask to ready - err := migration_util.UpdateStoppedChildTasksToReady(taskItem, "index_migration") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - taskItem.RetryTimes++ - if taskItem.StartTimeInMillis == 0 { - taskItem.StartTimeInMillis = time.Now().UnixMilli() - } - taskItem.Status = task.StatusRunning - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("major task [%s] started", taskItem.ID)) - p.sendMajorTaskNotification(taskItem) - return nil -} - -func (p *processor) splitMajorMigrationTask(taskItem *task.Task) error { - clusterMigrationTask := migration_model.ClusterMigrationTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &clusterMigrationTask) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return err - } - esSourceClient := elastic.GetClient(clusterMigrationTask.Cluster.Source.Id) - targetType := common.GetClusterDocType(clusterMigrationTask.Cluster.Target.Id) - - err = migration_util.DeleteChildTasks(taskItem.ID, "index_migration") - if err != nil { - log.Warnf("failed to clear child tasks, err: %v", err) - return nil - } - - current := migration_util.GetMapIntValue(taskItem.Metadata.Labels, "next_run_time") - var step time.Duration - if clusterMigrationTask.Settings.Execution.Repeat != nil { - step = clusterMigrationTask.Settings.Execution.Repeat.Interval - } - - var pids []string - pids = append(pids, taskItem.ParentId...) - pids = append(pids, taskItem.ID) - - var totalDocs int64 - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Second*60)) - defer cancel() - - for i, index := range clusterMigrationTask.Indices { - source := migration_model.IndexMigrationSourceConfig{ - ClusterId: clusterMigrationTask.Cluster.Source.Id, - Indices: index.Source.Name, - DocCount: index.Source.Docs, - SliceSize: clusterMigrationTask.Settings.Scroll.SliceSize, - BatchSize: clusterMigrationTask.Settings.Scroll.Docs, - ScrollTime: clusterMigrationTask.Settings.Scroll.Timeout, - SkipCountCheck: clusterMigrationTask.Settings.SkipScrollCountCheck, - } - - if index.IndexRename != nil { - source.IndexRename = index.IndexRename - } - if index.Target.Name != "" { - source.IndexRename = util.MapStr{ - index.Source.Name: index.Target.Name, - } - } - if index.TypeRename != nil { - source.TypeRename = index.TypeRename - } - - if v, ok := index.RawFilter.(string); ok { - source.QueryString = v - } else { - var must []interface{} - if index.RawFilter != nil { - must = append(must, index.RawFilter) - } - if index.Incremental != nil { - incrementalFilter, err := index.Incremental.BuildFilter(current, step) - if source.Step == nil { - source.Step = step.String() - source.End = float64(current - index.Incremental.Delay.Milliseconds()) - if !index.Incremental.Full { - source.Start = source.End - float64(step.Milliseconds()) - } - } - if err != nil { - return err - } - must = append(must, incrementalFilter) - } - if index.Source.DocType != "" { - if index.Target.DocType != "" { - source.TypeRename = util.MapStr{ - index.Source.DocType: index.Target.DocType, - } - } - must = append(must, util.MapStr{ - "terms": util.MapStr{ - "_type": []string{index.Source.DocType}, - }, - }) - } else { - if targetType != "" { - source.TypeRename = util.MapStr{ - "*": index.Target.DocType, - } - } - } - if len(must) > 0 { - source.QueryDSL = util.MapStr{ - "bool": util.MapStr{ - "must": must, - }, - } - } - } - - // NOTE: for repeating tasks, frontend can't get the accurate doc count - // update here before we split the task, if the index has incremental config and correct delay value, - // the doc count will not change and will be stable - var countQuery = util.MapStr{} - if source.QueryDSL != nil { - countQuery = util.MapStr{ - "query": source.QueryDSL, - } - } - sourceCount, err := esSourceClient.Count(ctx, index.Source.Name, util.MustToJSONBytes(countQuery)) - if err != nil { - log.Errorf("failed to count docs, err: %v", err) - return err - } - - clusterMigrationTask.Indices[i].Source.Docs = sourceCount.Count - source.DocCount = sourceCount.Count - totalDocs += sourceCount.Count - - target := migration_model.IndexMigrationTargetConfig{ - ClusterId: clusterMigrationTask.Cluster.Target.Id, - SkipCountCheck: clusterMigrationTask.Settings.SkipBulkCountCheck, - Bulk: migration_model.IndexMigrationBulkConfig{ - BatchSizeInMB: clusterMigrationTask.Settings.Bulk.StoreSizeInMB, - BatchSizeInDocs: clusterMigrationTask.Settings.Bulk.Docs, - MaxWorkerSize: clusterMigrationTask.Settings.Bulk.MaxWorkerSize, - IdleTimeoutInSeconds: clusterMigrationTask.Settings.Bulk.IdleTimeoutInSeconds, - SliceSize: clusterMigrationTask.Settings.Bulk.SliceSize, - Compress: clusterMigrationTask.Settings.Bulk.Compress, - Operation: clusterMigrationTask.Settings.Bulk.Operation, - }, - } - - if index.Partition != nil { - partitionQ := &elastic.PartitionQuery{ - IndexName: index.Source.Name, - FieldName: index.Partition.FieldName, - FieldType: index.Partition.FieldType, - Step: index.Partition.Step, - UseEvenStrategy: index.Partition.UseEvenStrategy, - } - if source.QueryDSL != nil { - partitionQ.Filter = source.QueryDSL - } - partitions, err := elastic.GetPartitions(partitionQ, esSourceClient) - if err != nil { - return err - } - if len(partitions) == 0 { - continue - } - var ( - partitionID int - ) - for _, partition := range partitions { - //skip empty partition - if partition.Docs <= 0 { - continue - } - partitionID++ - partitionSource := source - partitionSource.Start = partition.Start - partitionSource.End = partition.End - partitionSource.DocCount = partition.Docs - partitionSource.Step = index.Partition.Step - partitionSource.PartitionId = partitionID - partitionSource.QueryDSL = partition.Filter - partitionSource.QueryString = "" - - partitionMigrationTask := task.Task{ - ParentId: pids, - Cancellable: false, - Runnable: true, - Status: task.StatusReady, - Metadata: task.Metadata{ - Type: "index_migration", - Labels: util.MapStr{ - "business_id": "index_migration", - "source_cluster_id": clusterMigrationTask.Cluster.Source.Id, - "target_cluster_id": clusterMigrationTask.Cluster.Target.Id, - "index_name": index.Source.Name, - "unique_index_name": index.Source.GetUniqueIndexName(), - }, - }, - ConfigString: util.MustToJSON(migration_model.IndexMigrationTaskConfig{ - Source: partitionSource, - Target: target, - Execution: clusterMigrationTask.Settings.Execution, - Version: migration_model.IndexMigrationV1, - }), - } - partitionMigrationTask.ID = util.GetUUID() - err = orm.Create(nil, &partitionMigrationTask) - if err != nil { - return fmt.Errorf("store index migration task(partition) error: %w", err) - } - - } - } else { - indexMigrationTask := task.Task{ - ParentId: pids, - Cancellable: true, - Runnable: false, - Status: task.StatusReady, - Metadata: task.Metadata{ - Type: "index_migration", - Labels: util.MapStr{ - "business_id": "index_migration", - "source_cluster_id": clusterMigrationTask.Cluster.Source.Id, - "target_cluster_id": clusterMigrationTask.Cluster.Target.Id, - "partition_count": 1, - "index_name": index.Source.Name, - "unique_index_name": index.Source.GetUniqueIndexName(), - }, - }, - ConfigString: util.MustToJSON(migration_model.IndexMigrationTaskConfig{ - Source: source, - Target: target, - Execution: clusterMigrationTask.Settings.Execution, - Version: migration_model.IndexMigrationV1, - }), - } - - indexMigrationTask.ID = util.GetUUID() - - err = orm.Create(nil, &indexMigrationTask) - if err != nil { - return fmt.Errorf("store index migration task error: %w", err) - } - } - } - taskItem.ConfigString = util.MustToJSON(clusterMigrationTask) - taskItem.Metadata.Labels["is_split"] = true - taskItem.Metadata.Labels["source_total_docs"] = totalDocs - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("major task [%s] splitted", taskItem.ID)) - return nil -} - -func (p *processor) handleRunningMajorTask(taskItem *task.Task) error { - ts, err := p.getMajorTaskState(taskItem) - if err != nil { - return err - } - if !(ts.Status == task.StatusComplete || ts.Status == task.StatusError) { - return nil - } - - totalDocs := migration_util.GetMapIntValue(util.MapStr(taskItem.Metadata.Labels), "source_total_docs") - var errMsg string - if ts.Status == task.StatusError { - errMsg = "index migration(s) failed" - } - - if errMsg == "" { - if totalDocs != ts.IndexDocs { - errMsg = fmt.Sprintf("cluster migration completed but docs count unmatch: %d / %d", ts.IndexDocs, totalDocs) - } - } - - if errMsg == "" { - taskItem.Status = task.StatusComplete - } else { - taskItem.Status = task.StatusError - } - taskItem.Metadata.Labels["target_total_docs"] = ts.IndexDocs - tn := time.Now() - taskItem.CompletedTime = &tn - p.sendMajorTaskNotification(taskItem) - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("major task [%s] finished with status [%s]", taskItem.ID, taskItem.Status)) - return nil -} - -func (p *processor) getMajorTaskState(majorTask *task.Task) (taskState migration_model.ClusterMigrationTaskState, err error) { - query := util.MapStr{ - "size": 0, - "aggs": util.MapStr{ - "total_docs": util.MapStr{ - "sum": util.MapStr{ - "field": "metadata.labels.index_docs", - }, - }, - "grp": util.MapStr{ - "terms": util.MapStr{ - "field": "status", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": majorTask.ID, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_migration", - }, - }, - }, - }, - }, - }, - } - esClient := elastic.GetClient(p.Elasticsearch) - res, err := esClient.SearchWithRawQueryDSL(p.IndexName, util.MustToJSONBytes(query)) - if err != nil { - log.Errorf("search es failed, err: %v", err) - return taskState, nil - } - if v, err := util.ExtractInt(res.Aggregations["total_docs"].Value); err == nil { - taskState.IndexDocs = v - } - var ( - hasError bool - ) - for _, bk := range res.Aggregations["grp"].Buckets { - status, _ := util.ExtractString(bk["key"]) - if migration_util.IsRunningState(status) { - taskState.Status = task.StatusRunning - return taskState, nil - } - if status == task.StatusError { - hasError = true - } - } - if hasError { - taskState.Status = task.StatusError - } else { - taskState.Status = task.StatusComplete - } - return taskState, nil -} - -func (p *processor) handlePendingStopMajorTask(taskItem *task.Task) error { - err := migration_util.UpdatePendingChildTasksToPendingStop(taskItem, "index_migration") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - - tasks, err := migration_util.GetPendingChildTasks(taskItem.ID, "index_migration") - if err != nil { - log.Errorf("failed to get sub tasks, err: %v", err) - return nil - } - - // all subtask stopped or error or complete - if len(tasks) == 0 { - taskItem.Status = task.StatusStopped - p.sendMajorTaskNotification(taskItem) - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("task [%s] stopped", taskItem.ID)) - // NOTE: we don't know how many running index_migration's stopped, so do a refresh from ES - p.scheduler.RefreshInstanceJobsFromES() - } - return nil -} - -func (p *processor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.Elasticsearch) - _, err := esClient.Index(p.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -func (p *processor) sendMajorTaskNotification(taskItem *task.Task) { - // don't send notification for repeating child tasks - if len(taskItem.ParentId) > 0 { - return - } - - config := migration_model.ClusterMigrationTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &config) - if err != nil { - log.Errorf("failed to parse config info from major task, id: %s, err: %v", taskItem.ID, err) - return - } - - creatorID := config.Creator.Id - - var title, body string - body = fmt.Sprintf("From Cluster: [%s (%s)], To Cluster: [%s (%s)]", config.Cluster.Source.Id, config.Cluster.Source.Name, config.Cluster.Target.Id, config.Cluster.Target.Name) - link := fmt.Sprintf("/#/data_tools/migration/%s/detail", taskItem.ID) - switch taskItem.Status { - case task.StatusReady: - log.Debugf("skip sending notification for ready task, id: %s", taskItem.ID) - return - case task.StatusStopped: - title = fmt.Sprintf("Data Migration Stopped") - case task.StatusComplete: - title = fmt.Sprintf("Data Migration Completed") - case task.StatusError: - title = fmt.Sprintf("Data Migration Failed") - case task.StatusRunning: - title = fmt.Sprintf("Data Migration Started") - default: - log.Warnf("skip sending notification for invalid task status, id: %s", taskItem.ID) - return - } - notification := &model.Notification{ - UserId: util.ToString(creatorID), - Type: model.NotificationTypeNotification, - MessageType: model.MessageTypeMigration, - Status: model.NotificationStatusNew, - Title: title, - Body: body, - Link: link, - } - err = orm.Create(nil, notification) - if err != nil { - log.Errorf("failed to create notification, err: %v", err) - return - } - return -} diff --git a/plugin/task_manager/cluster_migration/orm.go b/plugin/task_manager/cluster_migration/orm.go deleted file mode 100644 index 4dbde582..00000000 --- a/plugin/task_manager/cluster_migration/orm.go +++ /dev/null @@ -1,129 +0,0 @@ -package cluster_migration - -import ( - "errors" - "fmt" - - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/api/rbac" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func RepeatTask(oldTask *task.Task) (*task.Task, error) { - config := migration_model.ClusterMigrationTaskConfig{} - err := migration_util.GetTaskConfig(oldTask, &config) - if err != nil { - return nil, err - } - - t, err := buildTask(&config, &rbac.ShortUser{ - Username: config.Creator.Name, - UserId: config.Creator.Id, - }, true) - if err != nil { - return nil, err - } - t.ParentId = []string{oldTask.ID} - if len(oldTask.ParentId) > 0 { - t.ParentId = oldTask.ParentId - } - - migration_util.CopyRepeatState(oldTask.Metadata.Labels, t.Metadata.Labels) - err = migration_util.UpdateRepeatState(config.Settings.Execution.Repeat, t.Metadata.Labels) - if err != nil { - return nil, fmt.Errorf("repeat invalid: %v", err) - } - - err = orm.Create(nil, t) - if err != nil { - return nil, err - } - return t, nil -} - -func CreateTask(config *migration_model.ClusterMigrationTaskConfig, creator *rbac.ShortUser) (*task.Task, error) { - t, err := buildTask(config, creator, false) - if err != nil { - return nil, err - } - - err = migration_util.UpdateRepeatState(config.Settings.Execution.Repeat, t.Metadata.Labels) - if err != nil { - return nil, fmt.Errorf("repeat invalid: %v", err) - } - - ctx := &orm.Context{ - Refresh: "wait_for", - } - err = orm.Create(ctx, t) - if err != nil { - return nil, err - } - return t, nil -} - -func buildTask(config *migration_model.ClusterMigrationTaskConfig, creator *rbac.ShortUser, repeat bool) (*task.Task, error) { - if len(config.Indices) == 0 { - return nil, errors.New("indices must not be empty") - } - if creator == nil { - return nil, errors.New("missing creator info") - } - - config.Creator.Name = creator.Username - config.Creator.Id = creator.UserId - - srcClusterCfg := elastic.GetConfig(config.Cluster.Source.Id) - config.Cluster.Source.Distribution = srcClusterCfg.Distribution - dstClusterCfg := elastic.GetConfig(config.Cluster.Target.Id) - config.Cluster.Target.Distribution = dstClusterCfg.Distribution - clearTaskConfig(config) - - var totalDocs int64 - for _, index := range config.Indices { - if index.Incremental != nil { - if repeat { - index.Incremental.Full = false - } else { - index.Incremental.Full = true - } - } - totalDocs += index.Source.Docs - } - - t := task.Task{ - Metadata: task.Metadata{ - Type: "cluster_migration", - Labels: util.MapStr{ - "business_id": "cluster_migration", - "source_cluster_id": config.Cluster.Source.Id, - "target_cluster_id": config.Cluster.Target.Id, - "source_total_docs": totalDocs, - "permit_nodes": config.Settings.Execution.Nodes.Permit, - "name": config.Name, - }, - }, - Cancellable: true, - Runnable: false, - Status: task.StatusInit, - ConfigString: util.MustToJSON(config), - } - if len(config.Tags) > 0 { - t.Metadata.Labels["tags"] = config.Tags - } - t.ID = util.GetUUID() - return &t, nil -} - -// sync with getDataMigrationTaskInfo -func clearTaskConfig(config *migration_model.ClusterMigrationTaskConfig) { - for i := range config.Indices { - config.Indices[i].Target.Docs = 0 - config.Indices[i].Percent = 0 - config.Indices[i].ErrorPartitions = 0 - } -} diff --git a/plugin/task_manager/common_api.go b/plugin/task_manager/common_api.go deleted file mode 100644 index 3f43a3a8..00000000 --- a/plugin/task_manager/common_api.go +++ /dev/null @@ -1,808 +0,0 @@ -package task_manager - -import ( - "errors" - "fmt" - model2 "infini.sh/console/model" - migration_model "infini.sh/console/plugin/task_manager/model" - "infini.sh/framework/core/global" - "net/http" - "strconv" - "strings" - "time" - - log "github.com/cihub/seelog" - - migration_util "infini.sh/console/plugin/task_manager/util" - - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" - elastic2 "infini.sh/framework/modules/elastic" -) - -type TaskInfoResponse struct { - TaskID string `json:"task_id"` - Step interface{} `json:"step"` - StartTime int64 `json:"start_time"` - CompletedTime int64 `json:"completed_time"` - Duration int64 `json:"duration"` - DataPartition int `json:"data_partition"` - CompletedPartitions int `json:"completed_partitions"` - Partitions []util.MapStr `json:"partitions"` - Repeating bool `json:"repeating"` - Workers []util.MapStr `json:"workers"` - Incremental *migration_model.IndexIncremental `json:"incremental"` - NextRunTime int64 `json:"next_run_time"` -} - -func (h *APIHandler) searchTask(taskType string) func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - keyword = h.GetParameterOrDefault(req, "keyword", "") - strSize = h.GetParameterOrDefault(req, "size", "20") - strFrom = h.GetParameterOrDefault(req, "from", "0") - mustQ []interface{} - ) - mustQ = append(mustQ, util.MapStr{ - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": taskType, - }, - }, - }) - - if keyword != "" { - mustQ = append(mustQ, util.MapStr{ - "query_string": util.MapStr{ - "default_field": "*", - "query": keyword, - }, - }) - } - size, _ := strconv.Atoi(strSize) - if size <= 0 { - size = 20 - } - from, _ := strconv.Atoi(strFrom) - if from < 0 { - from = 0 - } - - queryDSL := util.MapStr{ - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "desc", - }, - }, - }, - "size": size, - "from": from, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - "must_not": util.MapStr{ - "exists": util.MapStr{ - "field": "parent_id", - }, - }, - }, - }, - } - - q := orm.Query{} - q.RawQuery = util.MustToJSONBytes(queryDSL) - - err, res := orm.Search(&task.Task{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - searchRes := &elastic.SearchResponse{} - err = util.FromJSONBytes(res.Raw, searchRes) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - for _, hit := range searchRes.Hits.Hits { - sourceM := util.MapStr(hit.Source) - h.populateMajorTaskInfo(hit.ID, sourceM) - } - - h.WriteJSON(w, searchRes, http.StatusOK) - } -} - -func (h *APIHandler) populateMajorTaskInfo(taskID string, sourceM util.MapStr) { - buf := util.MustToJSONBytes(sourceM) - majorTask := task.Task{} - err := util.FromJSONBytes(buf, &majorTask) - if err != nil { - log.Errorf("failed to unmarshal major task info, err: %v", err) - return - } - _, repeatStatus, err := h.calcRepeatingStatus(&majorTask) - if err != nil { - log.Warnf("failed to calc repeat info, err: %v", err) - return - } - sourceM.Put("repeat", repeatStatus) - switch majorTask.Metadata.Type { - case "cluster_migration": - ts, _, err := h.getMigrationMajorTaskInfo(taskID) - if err != nil { - log.Warnf("fetch progress info of task error: %v", err) - return - } - sourceM.Put("metadata.labels.target_total_docs", ts.IndexDocs) - sourceM.Put("metadata.labels.source_total_docs", ts.SourceDocs) - sourceM.Put("metadata.labels.error_partitions", ts.ErrorPartitions) - count, err := migration_util.CountRunningChildren(taskID, "index_migration") - if err != nil { - log.Warnf("failed to count running children, err: %v", err) - return - } - sourceM.Put("running_children", count) - if repeatStatus.IsRepeat && repeatStatus.LastRunChildTaskID != "" { - ts, _, err = h.getMigrationMajorTaskInfo(repeatStatus.LastRunChildTaskID) - if err != nil { - log.Warnf("fetch progress info of task error: %v", err) - return - } - sourceM.Put("metadata.labels.target_total_docs", ts.IndexDocs) - sourceM.Put("metadata.labels.source_total_docs", ts.SourceDocs) - } - case "cluster_comparison": - targetTaskId := taskID - if repeatStatus.IsRepeat && repeatStatus.LastRunChildTaskID != "" { - targetTaskId = repeatStatus.LastRunChildTaskID - } - ts, _, err := h.getComparisonMajorTaskInfo(targetTaskId) - if err != nil { - log.Warnf("fetch progress info of task error: %v", err) - return - } - sourceM.Put("metadata.labels.source_scroll_docs", ts.SourceScrollDocs) - sourceM.Put("metadata.labels.source_total_docs", ts.SourceTotalDocs) - sourceM.Put("metadata.labels.target_total_docs", ts.TargetTotalDocs) - sourceM.Put("metadata.labels.target_scroll_docs", ts.TargetScrollDocs) - sourceM.Put("metadata.labels.total_diff_docs", ts.TotalDiffDocs) - count, err := migration_util.CountRunningChildren(taskID, "index_comparison") - if err != nil { - log.Warnf("failed to count running children, err: %v", err) - return - } - sourceM.Put("running_children", count) - } -} - -func (h *APIHandler) startTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - taskID := ps.MustGetParameter("task_id") - obj := task.Task{} - - obj.ID = taskID - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("task [%s] not found", taskID), http.StatusInternalServerError) - return - } - if obj.Metadata.Type != "pipeline" && (obj.Status == task.StatusComplete && obj.Metadata.Type != "cluster_comparison") { - h.WriteError(w, fmt.Sprintf("[%s] task [%s] completed, can't start anymore", obj.Metadata.Type, taskID), http.StatusInternalServerError) - return - } - obj.Status = task.StatusReady - if _, ok := obj.Metadata.Labels["next_run_time"]; !ok { - startTime := time.Now().UnixMilli() - // only set for initial cluster-level tasks - if len(obj.ParentId) == 0 { - obj.Metadata.Labels["next_run_time"] = startTime - } - } - - ctx := &orm.Context{ - Refresh: "wait_for", - } - err = orm.Update(ctx, &obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - migration_util.WriteLog(&obj, &task.TaskResult{ - Success: true, - }, "task status manually set to ready") - - // update status of parent task to running - for _, parentTaskID := range obj.ParentId { - parentTask := task.Task{} - parentTask.ID = parentTaskID - exists, err := orm.Get(&parentTask) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("parent task [%s] not found", parentTaskID), http.StatusInternalServerError) - return - } - parentTask.Status = task.StatusRunning - err = orm.Update(nil, &parentTask) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - migration_util.WriteLog(&parentTask, nil, fmt.Sprintf("child [%s] task [%s] manually started", obj.Metadata.Type, taskID)) - } - - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) -} - -// delete task and all sub tasks -func (h *APIHandler) deleteTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - obj := task.Task{} - obj.ID = id - - _, err := orm.Get(&obj) - if err != nil { - if errors.Is(err, elastic2.ErrNotFound) { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "result": "not_found", - }, http.StatusNotFound) - return - } - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - if util.StringInArray([]string{task.StatusReady, task.StatusRunning, task.StatusPendingStop}, obj.Status) { - h.WriteError(w, fmt.Sprintf("can not delete task [%s] with status [%s]", obj.ID, obj.Status), http.StatusInternalServerError) - return - } - ctx := &orm.Context{ - Refresh: "wait_for", - } - err = orm.Delete(ctx, &obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - q := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": id, - }, - }, - }, - }, - }, - }, - } - err = orm.DeleteBy(&obj, util.MustToJSONBytes(q)) - if err != nil { - log.Error(err) - } - - h.WriteJSON(w, util.MapStr{ - "_id": obj.ID, - "result": "deleted", - }, 200) -} - -// resume an repeating task -func (h *APIHandler) resumeTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - taskID := ps.MustGetParameter("task_id") - obj := task.Task{} - - obj.ID = taskID - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("task [%s] not found", taskID), http.StatusInternalServerError) - return - } - if len(obj.ParentId) > 0 { - h.WriteError(w, fmt.Sprintf("can't resume on a child task", taskID), http.StatusInternalServerError) - return - } - lastRepeatingChild, repeatStatus, err := h.calcRepeatingStatus(&obj) - if err != nil { - h.WriteError(w, fmt.Sprintf("failed to get repeating status", taskID), http.StatusInternalServerError) - return - } - if !repeatStatus.IsRepeat { - h.WriteError(w, fmt.Sprintf("not a repeating task", taskID), http.StatusInternalServerError) - return - } - if repeatStatus.Done { - h.WriteError(w, fmt.Sprintf("repeat task done", taskID), http.StatusInternalServerError) - return - } - if repeatStatus.Repeating { - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) - return - } - - lastRepeatingChild.Metadata.Labels["repeat_triggered"] = false - ctx := &orm.Context{ - Refresh: "wait_for", - } - err = orm.Update(ctx, lastRepeatingChild) - if err != nil { - log.Errorf("failed to update last child, err: %v", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - migration_util.WriteLog(&obj, nil, fmt.Sprintf("repeating task [%s] manually resumed", obj.Metadata.Type, taskID)) - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) - return -} - -// query index level task logging -func (h *APIHandler) searchIndexLevelTaskLogging(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - uniqueIndexName := ps.MustGetParameter("index") - cfg := global.MustLookup("cluster_migration_config") - var ( - migrationConfig *DispatcherConfig - ok bool - ) - if migrationConfig, ok = cfg.(*DispatcherConfig); !ok { - h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) - return - } - client := elastic.GetClient(migrationConfig.Elasticsearch) - var ( - strSize = h.GetParameterOrDefault(req, "size", "500") - min = h.GetParameterOrDefault(req, "min", "") - max = h.GetParameterOrDefault(req, "max", "") - ) - size, _ := strconv.Atoi(strSize) - if size <= 0 { - size = 500 - } - rangeObj := util.MapStr{} - if min != "" { - rangeObj["gte"] = min - } - if max != "" { - rangeObj["lt"] = max - } - mustQ := []util.MapStr{ - { - "term": util.MapStr{ - "metadata.category": util.MapStr{ - "value": "task", - }, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.parent_task_id": util.MapStr{ - "value": id, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.unique_index_name": util.MapStr{ - "value": uniqueIndexName, - }, - }, - }, - } - if len(rangeObj) > 0 { - mustQ = append(mustQ, util.MapStr{ - "range": util.MapStr{ - "timestamp": rangeObj, - }, - }) - } - query := util.MapStr{ - "size": size, - "_source": []string{"payload.task.logging.message", "timestamp"}, - "sort": []util.MapStr{ - { - "timestamp": util.MapStr{ - "order": "desc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - }, - }, - } - searchRes, err := client.SearchWithRawQueryDSL(migrationConfig.LogIndexName, util.MustToJSONBytes(query)) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, searchRes, http.StatusOK) -} - -type RepeatStatus struct { - IsRepeat bool `json:"is_repeat"` - Done bool `json:"done"` - Repeating bool `json:"repeating"` - LastRunTime int64 `json:"last_run_time"` - NextRunTime int64 `json:"next_run_time"` - LastRunChildTaskID string `json:"last_run_child_task_id"` - LastCompleteTime int64 `json:"last_complete_time"` -} - -func (h *APIHandler) calcRepeatingStatus(taskItem *task.Task) (*task.Task, *RepeatStatus, error) { - ret := &RepeatStatus{} - lastRepeatingChild, lastRunChild, err := migration_util.GetLastRepeatingChildTask(taskItem.ID, taskItem.Metadata.Type) - if err != nil { - return nil, nil, err - } - if lastRepeatingChild == nil { - lastRepeatingChild = taskItem - } - - isRepeat := migration_util.GetMapBoolValue(lastRepeatingChild.Metadata.Labels, "is_repeat") - if !isRepeat { - return lastRepeatingChild, ret, nil - } - ret.IsRepeat = isRepeat - - repeatDone := migration_util.GetMapBoolValue(lastRepeatingChild.Metadata.Labels, "repeat_done") - if repeatDone { - ret.Done = true - return lastRepeatingChild, ret, nil - } - repeatTriggered := migration_util.GetMapBoolValue(lastRepeatingChild.Metadata.Labels, "repeat_triggered") - if !repeatTriggered { - ret.Repeating = true - } - ret.NextRunTime = migration_util.GetMapIntValue(lastRepeatingChild.Metadata.Labels, "next_run_time") - ret.LastRunTime = lastRepeatingChild.StartTimeInMillis - if ret.LastRunTime == 0 && lastRunChild != nil { - ret.LastRunTime = lastRunChild.StartTimeInMillis - if lastRunChild.CompletedTime != nil && !lastRunChild.CompletedTime.IsZero(){ - ret.LastCompleteTime = lastRunChild.CompletedTime.UnixMilli() - } - } - if lastRunChild != nil { - ret.LastRunChildTaskID = lastRunChild.ID - } - return lastRepeatingChild, ret, nil -} - -func (h *APIHandler) stopTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - obj := task.Task{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - if task.IsEnded(obj.Status) { - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) - return - } - obj.Status = task.StatusPendingStop - err = orm.Update(nil, &obj) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - migration_util.WriteLog(&obj, &task.TaskResult{ - Success: true, - }, "task status manually set to pending stop") - - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) -} - -// pause an repeating task -func (h *APIHandler) pauseTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - taskID := ps.MustGetParameter("task_id") - obj := task.Task{} - - obj.ID = taskID - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("task [%s] not found", taskID), http.StatusInternalServerError) - return - } - if len(obj.ParentId) > 0 { - h.WriteError(w, fmt.Sprintf("can't pause on a child task", taskID), http.StatusInternalServerError) - return - } - lastRepeatingChild, repeatStatus, err := h.calcRepeatingStatus(&obj) - if err != nil { - h.WriteError(w, fmt.Sprintf("failed to get repeating status", taskID), http.StatusInternalServerError) - return - } - if !repeatStatus.IsRepeat { - h.WriteError(w, fmt.Sprintf("not a repeating task", taskID), http.StatusInternalServerError) - return - } - if repeatStatus.Done { - h.WriteError(w, fmt.Sprintf("repeat task done", taskID), http.StatusInternalServerError) - return - } - if !repeatStatus.Repeating { - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) - return - } - - lastRepeatingChild.Metadata.Labels["repeat_triggered"] = true - err = orm.Update(nil, lastRepeatingChild) - if err != nil { - log.Errorf("failed to update last child, err: %v", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - migration_util.WriteLog(&obj, nil, fmt.Sprintf("repeating task [%s] manually paused", taskID)) - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) - return -} - -func (h *APIHandler) validateDataMigration(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - typ := h.GetParameter(req, "type") - switch typ { - case "multi_type": - h.validateMultiType(w, req, ps) - return - } - h.WriteError(w, "unknown parameter type", http.StatusOK) -} - -func (h *APIHandler) validateMultiType(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var reqBody = struct { - Cluster struct { - SourceID string `json:"source_id"` - TargetID string `json:"target_id"` - } `json:"cluster"` - Indices []string - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - sourceClient := elastic.GetClient(reqBody.Cluster.SourceID) - // get source type - indexNames := strings.Join(reqBody.Indices, ",") - typeInfo, err := elastic.GetIndexTypes(sourceClient, indexNames) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteJSON(w, util.MapStr{ - "result": typeInfo, - }, http.StatusOK) -} - -func (h *APIHandler) getChildTaskInfosByIndex(id string, uniqueIndexName string) (subTasks []task.Task, runningPipelineTaskIDs map[string][]string, pipelineSubParentIDs map[string]string, parentIDPipelineTasks map[string][]task.Task, err error) { - queryDsl := util.MapStr{ - "size": 9999, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "asc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": id, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.unique_index_name": util.MapStr{ - "value": uniqueIndexName, - }, - }, - }, - }, - }, - }, - } - allTasks, err := migration_util.GetTasks(queryDsl) - if err != nil { - return - } - - runningPipelineTaskIDs = map[string][]string{} - pipelineSubParentIDs = map[string]string{} - parentIDPipelineTasks = map[string][]task.Task{} - - for _, subTask := range allTasks { - if subTask.Metadata.Type != "pipeline" { - subTasks = append(subTasks, subTask) - continue - } - - if pl := len(subTask.ParentId); pl != 2 { - continue - } - parentID := migration_util.GetDirectParentId(subTask.ParentId) - - pipelineSubParentIDs[subTask.ID] = parentID - instID := migration_util.GetMapStringValue(util.MapStr(subTask.Metadata.Labels), "execution_instance_id") - if instID == "" { - continue - } - if subTask.Status == task.StatusRunning { - runningPipelineTaskIDs[instID] = append(runningPipelineTaskIDs[instID], subTask.ID) - } - parentIDPipelineTasks[parentID] = append(parentIDPipelineTasks[parentID], subTask) - } - - return -} - -func (h *APIHandler) getChildPipelineInfosFromGateway(pipelineTaskIDs map[string][]string) (pipelineContexts map[string]util.MapStr) { - pipelineContexts = map[string]util.MapStr{} - var err error - - for instID, taskIDs := range pipelineTaskIDs { - inst := &model2.TaskWorker{} - inst.ID = instID - _, err = orm.Get(&inst.Instance) - if err != nil { - log.Errorf("failed to get instance info, id: %s, err: %v", instID, err) - continue - } - pipelines, err := inst.GetPipelinesByIDs(taskIDs) - if err != nil { - log.Errorf("failed to get pipelines info, err: %v", err) - continue - } - - for pipelineID, status := range pipelines { - pipelineContexts[pipelineID] = status.Context - } - } - - return -} - -func (h *APIHandler) calcMajorTaskInfo(subTasks []task.Task, repeating bool) (startTime int64, completedTime int64, duration int64, completedPartitions int) { - if len(subTasks) == 0 { - return - } - - for _, subTask := range subTasks { - if subTask.StartTimeInMillis > 0 { - if startTime == 0 { - startTime = subTask.StartTimeInMillis - } - if subTask.StartTimeInMillis < startTime { - startTime = subTask.StartTimeInMillis - } - } - if subTask.CompletedTime != nil { - subCompletedTime := subTask.CompletedTime.UnixMilli() - if subCompletedTime > completedTime { - completedTime = subCompletedTime - } - } - - if subTask.Status == task.StatusComplete || subTask.Status == task.StatusError { - completedPartitions++ - } - } - if len(subTasks) != completedPartitions || repeating { - completedTime = 0 - duration = time.Now().UnixMilli() - startTime - } else { - duration = completedTime - startTime - } - - return -} - -func (h *APIHandler) searchTaskFieldValues(taskType string) func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - field = h.GetParameterOrDefault(req, "field", "") - keyword = h.GetParameterOrDefault(req, "keyword", "") - mustQ []interface{} - ) - mustQ = append(mustQ, util.MapStr{ - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": taskType, - }, - }, - }) - - if v := strings.TrimSpace(keyword); v != ""{ - mustQ = append(mustQ, util.MapStr{ - "query_string": util.MapStr{ - "default_field": field, - "query": fmt.Sprintf("*%s*", v), - }, - }) - } - queryDSL := util.MapStr{ - "aggs": util.MapStr{ - "items": util.MapStr{ - "terms": util.MapStr{ - "field": field, - "size": 20, - }, - }, - }, - "size": 0, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - }, - }, - } - q := orm.Query{ - RawQuery: util.MustToJSONBytes(queryDSL), - } - err, result := orm.Search(task.Task{}, &q) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - searchRes := elastic.SearchResponse{} - err = util.FromJSONBytes(result.Raw, &searchRes) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - items := []string{} - for _, bk := range searchRes.Aggregations["items"].Buckets { - if v, ok := bk["key"].(string); ok { - if strings.Contains(v, keyword){ - items = append(items, v) - } - } - } - h.WriteJSON(w, items, http.StatusOK) - } -} - diff --git a/plugin/task_manager/comparison_api.go b/plugin/task_manager/comparison_api.go deleted file mode 100644 index a4382f0f..00000000 --- a/plugin/task_manager/comparison_api.go +++ /dev/null @@ -1,400 +0,0 @@ -package task_manager - -import ( - "fmt" - "net/http" - "time" - - log "github.com/cihub/seelog" - - "infini.sh/console/plugin/task_manager/cluster_comparison" - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/api/rbac" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/global" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func (h *APIHandler) createDataComparisonTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - clusterTaskConfig := &migration_model.ClusterComparisonTaskConfig{} - err := h.DecodeJSON(req, clusterTaskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - user, err := rbac.FromUserContext(req.Context()) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - t, err := cluster_comparison.CreateTask(clusterTaskConfig, user) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - h.WriteCreatedOKJSON(w, t.ID) -} - -func (h *APIHandler) getDataComparisonTaskInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - - obj := task.Task{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - taskConfig := &migration_model.ClusterComparisonTaskConfig{} - err = migration_util.GetTaskConfig(&obj, taskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - _, indexState, err := h.getComparisonMajorTaskInfo(id) - if err != nil { - log.Errorf("failed to get major task info, err: %v", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - var completedIndices int - for i, index := range taskConfig.Indices { - indexName := index.Source.GetUniqueIndexName() - count := indexState[indexName].TotalScrollDocs - percent := float64(count) / float64(index.Source.Docs+index.Target.Docs) * 100 - if percent > 100 { - percent = 100 - } - taskConfig.Indices[i].Source.Docs = indexState[indexName].SourceTotalDocs - taskConfig.Indices[i].Target.Docs = indexState[indexName].TargetTotalDocs - taskConfig.Indices[i].ScrollPercent = util.ToFixed(percent, 2) - taskConfig.Indices[i].TotalScrollDocs = count - taskConfig.Indices[i].ErrorPartitions = indexState[indexName].ErrorPartitions - taskConfig.Indices[i].RunningChildren = indexState[indexName].RunningChildren - if count == index.Source.Docs+index.Target.Docs { - completedIndices++ - } - } - cfg := global.MustLookup("cluster_migration_config") - if migrationConfig, ok := cfg.(*DispatcherConfig); ok { - if obj.Metadata.Labels == nil { - obj.Metadata.Labels = util.MapStr{} - } - obj.Metadata.Labels["log_info"] = util.MapStr{ - "cluster_id": migrationConfig.Elasticsearch, - "index_name": migrationConfig.LogIndexName, - } - } - - _, repeatStatus, err := h.calcRepeatingStatus(&obj) - if err != nil { - log.Warnf("failed to calc repeat info, err: %v", err) - } - obj.Metadata.Labels["repeat"] = repeatStatus - - obj.ConfigString = util.MustToJSON(taskConfig) - obj.Metadata.Labels["completed_indices"] = completedIndices - h.WriteJSON(w, obj, http.StatusOK) -} - -type ClusterComparisonTaskState struct { - SourceTotalDocs int64 - SourceScrollDocs int64 - TargetTotalDocs int64 - TargetScrollDocs int64 - TotalDiffDocs int64 - RunningChildren int -} - -type ComparisonIndexStateInfo struct { - ErrorPartitions int - SourceTotalDocs int64 - SourceScrollDocs int64 - TargetTotalDocs int64 - TargetScrollDocs int64 - TotalDiffDocs int64 - RunningChildren int - TotalScrollDocs int64 -} - -func (h *APIHandler) getComparisonMajorTaskInfo(taskID string) (taskStats ClusterComparisonTaskState, indexState map[string]ComparisonIndexStateInfo, err error) { - var pipelineTaskIDs = map[string][]string{} - var pipelineIndexNames = map[string]string{} - indexState = map[string]ComparisonIndexStateInfo{} - const size = 500 - var ( - from = -size - hasMore = true - ) - for hasMore { - from += size - taskQuery := util.MapStr{ - "size": 500, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskID, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_comparison", - }, - }, - }, - }, - }, - }, - } - subTasks, err := migration_util.GetTasks(taskQuery) - if err != nil { - return taskStats, indexState, err - } - if len(subTasks) < size { - hasMore = false - } - - var indexMigrationTaskIDs []string - for _, subTask := range subTasks { - taskLabels := util.MapStr(subTask.Metadata.Labels) - indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") - if indexName == "" { - continue - } - - cfg := migration_model.IndexComparisonTaskConfig{} - err = migration_util.GetTaskConfig(&subTask, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - continue - } - totalDiffDocs := migration_util.GetMapIntValue(taskLabels, "total_diff_docs") - taskStats.SourceTotalDocs += cfg.Source.DocCount - taskStats.TargetTotalDocs += cfg.Target.DocCount - taskStats.TotalDiffDocs += totalDiffDocs - st := indexState[indexName] - st.SourceTotalDocs += cfg.Source.DocCount - st.TargetTotalDocs += cfg.Target.DocCount - st.TotalDiffDocs += totalDiffDocs - if subTask.Status == task.StatusError { - st.ErrorPartitions += 1 - } - if subTask.Status == task.StatusRunning { - st.RunningChildren++ - indexState[indexName] = st - indexMigrationTaskIDs = append(indexMigrationTaskIDs, subTask.ID) - continue - } - sourceDocs := migration_util.GetMapIntValue(taskLabels, "source_scrolled") - targetDocs := migration_util.GetMapIntValue(taskLabels, "target_scrolled") - st.SourceScrollDocs += sourceDocs - st.TargetScrollDocs += targetDocs - st.TotalScrollDocs += sourceDocs + targetDocs - taskStats.TargetScrollDocs += targetDocs - taskStats.SourceScrollDocs += sourceDocs - indexState[indexName] = st - } - - if len(indexMigrationTaskIDs) == 0 { - continue - } - - taskQuery = util.MapStr{ - "size": len(indexMigrationTaskIDs) * 2, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "parent_id": indexMigrationTaskIDs, - }, - }, - }, - }, - }, - } - subTasks, err = migration_util.GetTasks(taskQuery) - if err != nil { - return taskStats, indexState, err - } - - for _, subTask := range subTasks { - taskLabels := util.MapStr(subTask.Metadata.Labels) - indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") - if indexName == "" { - continue - } - - pipelineIndexNames[subTask.ID] = indexName - - if instID := migration_util.GetMapStringValue(taskLabels, "execution_instance_id"); instID != "" { - pipelineTaskIDs[instID] = append(pipelineTaskIDs[instID], subTask.ID) - } - } - } - - pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs) - for pipelineID, pipelineContext := range pipelineContexts { - // add scrolledDocs of running tasks - scrollDocs := migration_util.GetMapIntValue(pipelineContext, "dump_hash.scrolled_docs") - indexName := pipelineIndexNames[pipelineID] - st := indexState[indexName] - st.TotalScrollDocs += scrollDocs - indexState[indexName] = st - } - return taskStats, indexState, nil -} - -func (h *APIHandler) getDataComparisonTaskOfIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - uniqueIndexName := ps.MustGetParameter("index") - majorTask := task.Task{} - majorTask.ID = id - exists, err := orm.Get(&majorTask) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("task [%s] not found", id), http.StatusInternalServerError) - return - } - - taskConfig := &migration_model.ClusterComparisonTaskConfig{} - err = migration_util.GetTaskConfig(&majorTask, taskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - taskInfo := &TaskInfoResponse{ - TaskID: id, - StartTime: majorTask.StartTimeInMillis, - Repeating: migration_util.IsRepeating(taskConfig.Settings.Execution.Repeat, majorTask.Metadata.Labels), - } - - subTasks, pipelineTaskIDs, _, parentIDPipelineTasks, err := h.getChildTaskInfosByIndex(id, uniqueIndexName) - - taskInfo.DataPartition = len(subTasks) - if len(subTasks) == 0 { - h.WriteJSON(w, taskInfo, http.StatusOK) - return - } - - pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs) - startTime, completedTime, duration, completedPartitions := h.calcMajorTaskInfo(subTasks, taskInfo.Repeating) - - var partitionTaskInfos []util.MapStr - var workers = map[string]struct{}{} - - for i, subTask := range subTasks { - cfg := migration_model.IndexComparisonTaskConfig{} - err := migration_util.GetTaskConfig(&subTask, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - continue - } - if i == 0 { - taskInfo.Step = cfg.Source.Step - } - instID := migration_util.GetMapStringValue(subTask.Metadata.Labels, "execution_instance_id") - if instID != "" { - workers[instID] = struct{}{} - } - - var durationInMS int64 - var subCompletedTime int64 - if subTask.StartTimeInMillis > 0 { - if migration_util.IsPendingState(subTask.Status) { - durationInMS = time.Now().UnixMilli() - subTask.StartTimeInMillis - } else if subTask.CompletedTime != nil { - subCompletedTime = subTask.CompletedTime.UnixMilli() - durationInMS = subCompletedTime - subTask.StartTimeInMillis - } - } - subTaskLabels := util.MapStr(subTask.Metadata.Labels) - sourceScrollDocs := migration_util.GetMapIntValue(subTaskLabels, "source_scrolled") - targetScrollDocs := migration_util.GetMapIntValue(subTaskLabels, "target_scrolled") - onlyInSource := migration_util.GetMapIntValue(subTaskLabels, "only_in_source") - onlyInTarget := migration_util.GetMapIntValue(subTaskLabels, "only_in_target") - diffBoth := migration_util.GetMapIntValue(subTaskLabels, "diff_both") - - partitionTaskInfo := util.MapStr{ - "task_id": subTask.ID, - "status": subTask.Status, - "start_time": subTask.StartTimeInMillis, - "completed_time": subCompletedTime, - "start": cfg.Source.Start, - "end": cfg.Source.End, - "duration": durationInMS, - "source_total_docs": cfg.Source.DocCount, - "target_total_docs": cfg.Target.DocCount, - "only_in_source": onlyInSource, - "only_in_target": onlyInTarget, - "diff_both": diffBoth, - } - sourceDumpTask, targetDumpTask, _ := migration_util.SplitIndexComparisonTasks(parentIDPipelineTasks[subTask.ID], &cfg) - if sourceDumpTask != nil { - partitionTaskInfo["source_scroll_task"] = util.MapStr{ - "id": sourceDumpTask.ID, - "status": sourceDumpTask.Status, - } - pipelineID := sourceDumpTask.ID - pipelineContext, ok := pipelineContexts[pipelineID] - if ok { - if vv := migration_util.GetMapIntValue(pipelineContext, "dump_hash.scrolled_docs"); vv > 0 { - sourceScrollDocs = vv - } - } - } - if targetDumpTask != nil { - partitionTaskInfo["target_scroll_task"] = util.MapStr{ - "id": targetDumpTask.ID, - "status": targetDumpTask.Status, - } - pipelineID := targetDumpTask.ID - pipelineContext, ok := pipelineContexts[pipelineID] - if ok { - if vv := migration_util.GetMapIntValue(pipelineContext, "dump_hash.scrolled_docs"); vv > 0 { - targetScrollDocs = vv - } - } - } - partitionTaskInfo["source_scroll_docs"] = sourceScrollDocs - partitionTaskInfo["target_scroll_docs"] = targetScrollDocs - partitionTaskInfos = append(partitionTaskInfos, partitionTaskInfo) - } - taskInfo.CompletedTime = completedTime - taskInfo.Duration = duration - // NOTE: overwrite major task start time with the first started sub task - if taskInfo.StartTime == 0 { - taskInfo.StartTime = startTime - } - for _, node := range taskConfig.Settings.Execution.Nodes.Permit { - if _, ok := workers[node.ID]; ok { - taskInfo.Workers = append(taskInfo.Workers, util.MapStr{ - "id": node.ID, - "name": node.Name, - }) - } - } - taskInfo.Partitions = partitionTaskInfos - taskInfo.CompletedPartitions = completedPartitions - h.WriteJSON(w, taskInfo, http.StatusOK) -} diff --git a/plugin/task_manager/es_api.go b/plugin/task_manager/es_api.go deleted file mode 100644 index ae77ed41..00000000 --- a/plugin/task_manager/es_api.go +++ /dev/null @@ -1,156 +0,0 @@ -package task_manager - -import ( - "net/http" - - log "github.com/cihub/seelog" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/util" -) - -func (h *APIHandler) getIndexPartitionInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - index = ps.MustGetParameter("index") - clusterID = ps.MustGetParameter("id") - ) - client := elastic.GetClient(clusterID) - pq := &elastic.PartitionQuery{ - IndexName: index, - } - err := h.DecodeJSON(req, pq) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - partitions, err := elastic.GetPartitions(pq, client) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, partitions, http.StatusOK) -} - -func (h *APIHandler) countDocuments(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - index = ps.MustGetParameter("index") - clusterID = ps.MustGetParameter("id") - ) - client := elastic.GetClient(clusterID) - reqBody := struct { - Filter interface{} `json:"filter"` - }{} - err := h.DecodeJSON(req, &reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - var query []byte - if reqBody.Filter != nil { - query = util.MustToJSONBytes(util.MapStr{ - "query": reqBody.Filter, - }) - } - - ctx := req.Context() - - countRes, err := client.Count(ctx, index, query) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteJSON(w, countRes, http.StatusOK) -} - -func (h *APIHandler) refreshIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - index = ps.MustGetParameter("index") - clusterID = ps.MustGetParameter("id") - ) - client := elastic.GetClient(clusterID) - err := client.Refresh(index) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) - return - } - h.WriteJSON(w, util.MapStr{ - "success": true, - }, 200) -} - -type InitIndexRequest struct { - Mappings map[string]interface{} `json:"mappings"` - Settings map[string]interface{} `json:"settings"` -} - -func (h *APIHandler) initIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - targetClusterID := ps.MustGetParameter("id") - indexName := ps.MustGetParameter("index") - reqBody := &InitIndexRequest{} - err := h.DecodeJSON(req, reqBody) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - client := elastic.GetClient(targetClusterID) - exists, err := client.Exists(indexName) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if exists { - if len(reqBody.Settings) > 0 { - err = client.UpdateIndexSettings(indexName, reqBody.Settings) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - } - if ml := len(reqBody.Mappings); ml > 0 { - var ( - docType = "" - mapping interface{} = reqBody.Mappings - ) - if ml == 1 { - for key, _ := range reqBody.Mappings { - if key != "properties" { - docType = key - mapping = reqBody.Mappings[key] - } - } - } - mappingBytes := util.MustToJSONBytes(mapping) - _, err = client.UpdateMapping(indexName, docType, mappingBytes) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - } - } else { - indexSettings := map[string]interface{}{} - if len(reqBody.Settings) > 0 { - indexSettings["settings"] = reqBody.Settings - } - if len(reqBody.Mappings) > 0 { - indexSettings["mappings"] = reqBody.Mappings - } - err = client.CreateIndex(indexName, indexSettings) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - } - h.WriteJSON(w, util.MapStr{ - "success": true, - }, http.StatusOK) -} diff --git a/plugin/task_manager/index_comparison/index_comparison.go b/plugin/task_manager/index_comparison/index_comparison.go deleted file mode 100644 index cdcb1bc5..00000000 --- a/plugin/task_manager/index_comparison/index_comparison.go +++ /dev/null @@ -1,481 +0,0 @@ -package index_comparison - -import ( - "fmt" - "time" - - log "github.com/cihub/seelog" - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -type processor struct { - Elasticsearch string - IndexName string - scheduler migration_model.Scheduler -} - -func NewProcessor(elasticsearch, indexName string, scheduler migration_model.Scheduler) migration_model.Processor { - return &processor{ - Elasticsearch: elasticsearch, - IndexName: indexName, - scheduler: scheduler, - } -} - -func (p *processor) Process(t *task.Task) (err error) { - switch t.Status { - case task.StatusReady: - // split & schedule pipline tasks - err = p.handleReadySubTask(t) - case task.StatusRunning: - // check pipeline tasks status - err = p.handleRunningSubTask(t) - case task.StatusPendingStop: - // mark pipeline tasks as pending_stop - err = p.handlePendingStopSubTask(t) - } - return err -} - -func (p *processor) handleReadySubTask(taskItem *task.Task) error { - if ok, _ := util.ExtractBool(taskItem.Metadata.Labels["is_split"]); !ok { - return p.handleSplitSubTask(taskItem) - } - - return p.handleScheduleSubTask(taskItem) -} - -// split task to two dump_hash and on index_diff pipeline and then persistent -func (p *processor) handleSplitSubTask(taskItem *task.Task) error { - cfg := migration_model.IndexComparisonTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - return fmt.Errorf("got wrong config [%v] with task [%s], err: %v", taskItem.ConfigString, taskItem.ID, err) - } - - if len(taskItem.ParentId) == 0 { - return fmt.Errorf("got wrong parent id of task [%v]", *taskItem) - } - - err = migration_util.DeleteChildTasks(taskItem.ID, "pipeline") - if err != nil { - log.Warnf("failed to clear child tasks, err: %v", err) - return nil - } - - var pids []string - pids = append(pids, taskItem.ParentId...) - pids = append(pids, taskItem.ID) - sourceClusterID := cfg.Source.ClusterId - targetClusterID := cfg.Target.ClusterId - - sourceDumpID := util.GetUUID() - sourceDumpTask := &task.Task{ - ParentId: pids, - Runnable: true, - Cancellable: true, - Metadata: task.Metadata{ - Type: "pipeline", - Labels: util.MapStr{ - "cluster_id": sourceClusterID, - "pipeline_id": "dump_hash", - "index_name": cfg.Source.Indices, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - }, - Status: task.StatusInit, - RetryTimes: taskItem.RetryTimes, - ConfigString: util.MustToJSON(migration_model.PipelineTaskConfig{ - Name: sourceDumpID, - Logging: migration_model.PipelineTaskLoggingConfig{ - Enabled: true, - }, - Labels: util.MapStr{ - "parent_task_id": pids, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - AutoStart: true, - KeepRunning: false, - Processor: []util.MapStr{ - { - "dump_hash": util.MapStr{ - "slice_size": cfg.Source.SliceSize, - "batch_size": cfg.Source.BatchSize, - "indices": cfg.Source.Indices, - "elasticsearch": sourceClusterID, - "output_queue": sourceDumpID, - "clean_old_files": true, - "partition_size": cfg.Source.PartitionSize, - "scroll_time": cfg.Source.ScrollTime, - "query_dsl": util.MustToJSON(util.MapStr{ - "query": cfg.Source.QueryDSL, - }), - }, - }, - }, - }), - } - sourceDumpTask.ID = sourceDumpID - - targetDumpID := util.GetUUID() - targetDumpTask := &task.Task{ - ParentId: pids, - Runnable: true, - Cancellable: true, - Metadata: task.Metadata{ - Type: "pipeline", - Labels: util.MapStr{ - "cluster_id": targetClusterID, - "pipeline_id": "dump_hash", - "index_name": cfg.Target.Indices, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - }, - Status: task.StatusInit, - RetryTimes: taskItem.RetryTimes, - ConfigString: util.MustToJSON(migration_model.PipelineTaskConfig{ - Name: targetDumpID, - Logging: migration_model.PipelineTaskLoggingConfig{ - Enabled: true, - }, - Labels: util.MapStr{ - "parent_task_id": pids, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - AutoStart: true, - KeepRunning: false, - Processor: []util.MapStr{ - { - "dump_hash": util.MapStr{ - "slice_size": cfg.Target.SliceSize, - "batch_size": cfg.Target.BatchSize, - "indices": cfg.Target.Indices, - "elasticsearch": targetClusterID, - "output_queue": targetDumpID, - "clean_old_files": true, - "partition_size": cfg.Target.PartitionSize, - "scroll_time": cfg.Target.ScrollTime, - "query_dsl": util.MustToJSON(util.MapStr{ - "query": cfg.Target.QueryDSL, - }), - }, - }, - }, - }), - } - targetDumpTask.ID = targetDumpID - - diffID := util.GetUUID() - diffTask := &task.Task{ - ParentId: pids, - Runnable: true, - Cancellable: true, - Metadata: task.Metadata{ - Type: "pipeline", - Labels: util.MapStr{ - "pipeline_id": "index_diff", - }, - }, - Status: task.StatusInit, - RetryTimes: taskItem.RetryTimes, - ConfigString: util.MustToJSON(migration_model.PipelineTaskConfig{ - Name: diffID, - Logging: migration_model.PipelineTaskLoggingConfig{ - Enabled: true, - }, - Labels: util.MapStr{ - "parent_task_id": pids, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - AutoStart: true, - KeepRunning: false, - Processor: []util.MapStr{ - { - "index_diff": util.MapStr{ - "text_report": false, - "keep_source": false, - "buffer_size": 1, - "clean_old_files": true, - // NOTE: source & target must have same partition_size - "partition_size": cfg.Source.PartitionSize, - "source_queue": sourceDumpID, - "target_queue": targetDumpID, - "output_queue": util.MapStr{ - "name": diffID, - "labels": util.MapStr{ - "comparison_task_id": taskItem.ID, - }, - }, - }, - }, - }, - }), - } - diffTask.ID = diffID - - err = orm.Create(nil, sourceDumpTask) - if err != nil { - return fmt.Errorf("create source dump pipeline task error: %w", err) - } - err = orm.Create(nil, targetDumpTask) - if err != nil { - return fmt.Errorf("create target dump pipeline task error: %w", err) - } - err = orm.Create(nil, diffTask) - if err != nil { - return fmt.Errorf("create diff pipeline task error: %w", err) - } - - taskItem.Metadata.Labels["is_split"] = true - taskItem.Status = task.StatusReady - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("comparison task [%s] splitted", taskItem.ID)) - return nil -} - -func (p *processor) handleScheduleSubTask(taskItem *task.Task) error { - cfg := migration_model.IndexComparisonTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return fmt.Errorf("got wrong config of task [%s]", taskItem.ID) - } - - sourceDumpTask, targetDumpTask, diffTask, err := p.getPipelineTasks(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return nil - } - if sourceDumpTask == nil || targetDumpTask == nil || diffTask == nil { - // ES might not synced yet - log.Warnf("task [%s] pipeline task(s) not found", taskItem.ID) - return nil - } - - taskItem.RetryTimes++ - - // get a new instanceID - executionConfig := cfg.Execution - instance, err := p.scheduler.GetPreferenceInstance(executionConfig) - if err != nil { - if err == migration_model.ErrHitMax { - log.Debug("hit max tasks per instance, skip dispatch") - return nil - } - return fmt.Errorf("get preference intance error: %w", err) - } - instanceID := instance.ID - - // try to clear disk queue before running dump_hash - p.cleanGatewayQueue(taskItem) - - sourceDumpTask.RetryTimes = taskItem.RetryTimes - sourceDumpTask.Metadata.Labels["execution_instance_id"] = instanceID - sourceDumpTask.Status = task.StatusReady - err = orm.Update(nil, sourceDumpTask) - if err != nil { - return fmt.Errorf("update source dump pipeline task error: %w", err) - } - targetDumpTask.RetryTimes = taskItem.RetryTimes - targetDumpTask.Metadata.Labels["execution_instance_id"] = instanceID - targetDumpTask.Status = task.StatusReady - err = orm.Update(nil, targetDumpTask) - if err != nil { - return fmt.Errorf("update target dump pipeline task error: %w", err) - } - diffTask.RetryTimes = taskItem.RetryTimes - diffTask.Metadata.Labels["execution_instance_id"] = instanceID - diffTask.Status = task.StatusInit - err = orm.Update(nil, diffTask) - if err != nil { - return fmt.Errorf("update diff pipeline task error: %w", err) - } - - // update sub migration task status to running and save task log - taskItem.Metadata.Labels["execution_instance_id"] = instanceID - p.clearTaskState(taskItem) - taskItem.Status = task.StatusRunning - taskItem.StartTimeInMillis = time.Now().UnixMilli() - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("index comparison task [%s] started", taskItem.ID)) - // update dispatcher state - p.scheduler.IncrInstanceJobs(instanceID) - return nil -} - -func (p *processor) handleRunningSubTask(taskItem *task.Task) error { - cfg := migration_model.IndexComparisonTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return fmt.Errorf("got wrong config of task [%s]", taskItem.ID) - } - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - - sourceDumpTask, targetDumpTask, diffTask, err := p.getPipelineTasks(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return nil - } - if sourceDumpTask == nil || targetDumpTask == nil || diffTask == nil { - // ES might not synced yet - log.Warnf("task [%s] pipeline task(s) not found", taskItem.ID) - return nil - } - - if migration_util.IsRunningState(sourceDumpTask.Status) || migration_util.IsRunningState(targetDumpTask.Status) || migration_util.IsRunningState(diffTask.Status) { - return nil - } - if sourceDumpTask.Status == task.StatusComplete && targetDumpTask.Status == task.StatusComplete { - sourceDocs := migration_util.GetMapIntValue(util.MapStr(sourceDumpTask.Metadata.Labels), "scrolled_docs") - targetDocs := migration_util.GetMapIntValue(util.MapStr(targetDumpTask.Metadata.Labels), "scrolled_docs") - - taskItem.Metadata.Labels["source_scrolled"] = sourceDocs - taskItem.Metadata.Labels["target_scrolled"] = targetDocs - if diffTask.Status == task.StatusInit { - diffTask.Status = task.StatusReady - p.saveTaskAndWriteLog(diffTask, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("source/target dump completed, diff pipeline started")) - return nil - } - if diffTask.Status == task.StatusComplete { - m := util.MapStr(diffTask.Metadata.Labels) - var ( - onlyInSource = migration_util.GetMapIntValue(m, "only_in_source_count") - onlyInSourceKeys = migration_util.GetMapStringValue(m, "only_in_source_keys") - onlyInTarget = migration_util.GetMapIntValue(m, "only_in_target_count") - onlyInTargetKeys = migration_util.GetMapStringValue(m, "only_in_target_keys") - diffBoth = migration_util.GetMapIntValue(m, "diff_both_count") - diffBothKeys = migration_util.GetMapStringValue(m, "diff_both_keys") - ) - if onlyInSource > 0 || onlyInTarget > 0 || diffBoth > 0 { - now := time.Now() - taskItem.Metadata.Labels["total_diff_docs"] = onlyInSource + onlyInTarget + diffBoth - taskItem.Metadata.Labels["only_in_source"] = onlyInSource - taskItem.Metadata.Labels["only_in_target"] = onlyInTarget - taskItem.Metadata.Labels["diff_both"] = diffBoth - taskItem.CompletedTime = &now - taskItem.Status = task.StatusError - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: "data unmatch", - }, fmt.Sprintf("index comparison failed, only in source: %d, only in target: %d, diff in both: %d (sample doc ids: [%s], [%s] [%s])", onlyInSource, onlyInTarget, diffBoth, onlyInSourceKeys, onlyInTargetKeys, diffBothKeys)) - p.cleanGatewayQueue(taskItem) - p.scheduler.DecrInstanceJobs(instanceID) - return nil - } - } - } - if sourceDumpTask.Status == task.StatusError || targetDumpTask.Status == task.StatusError || diffTask.Status == task.StatusError { - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Status = task.StatusError - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: "pipeline task failed", - }, "index comparison failed") - p.cleanGatewayQueue(taskItem) - p.scheduler.DecrInstanceJobs(instanceID) - return nil - } - - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Status = task.StatusComplete - taskItem.Metadata.Labels["total_diff_docs"] = 0 - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, "index comparison completed") - p.cleanGatewayQueue(taskItem) - p.scheduler.DecrInstanceJobs(instanceID) - - return nil -} - -func (p *processor) handlePendingStopSubTask(taskItem *task.Task) error { - err := migration_util.UpdatePendingChildTasksToPendingStop(taskItem, "pipeline") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - - tasks, err := migration_util.GetPendingChildTasks(taskItem.ID, "pipeline") - if err != nil { - log.Errorf("failed to get sub tasks, err: %v", err) - return nil - } - - // all subtask stopped or error or complete - if len(tasks) == 0 { - taskItem.Status = task.StatusStopped - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("index comparison task [%s] stopped", taskItem.ID)) - } - return nil -} - -func (p *processor) getPipelineTasks(taskItem *task.Task, cfg *migration_model.IndexComparisonTaskConfig) (sourceDumpTask *task.Task, targetDumpTask *task.Task, diffTask *task.Task, err error) { - ptasks, err := migration_util.GetChildTasks(taskItem.ID, "pipeline", nil) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return - } - if len(ptasks) != 3 { - err = fmt.Errorf("invalid pipeline task count: %d", len(ptasks)) - return - } - sourceDumpTask, targetDumpTask, diffTask = migration_util.SplitIndexComparisonTasks(ptasks, cfg) - return -} - -// NOTE: only index_diff have an output queue, others are local files -func (p *processor) cleanGatewayQueue(taskItem *task.Task) { - log.Debugf("cleaning gateway queue for task [%s]", taskItem.ID) - - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - if instanceID == "" { - log.Debugf("task [%s] not scheduled yet, skip cleaning queue", taskItem.ID) - return - } - instance, err := p.scheduler.GetInstance(instanceID) - if err != nil { - log.Errorf("failed to get instance, err: %v", err) - return - } - - selector := util.MapStr{ - "labels": util.MapStr{ - "comparison_task_id": taskItem.ID, - }, - } - err = instance.DeleteQueueBySelector(selector) - if err != nil { - log.Errorf("failed to delete queue, err: %v", err) - } -} - -func (p *processor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.Elasticsearch) - _, err := esClient.Index(p.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -func (p *processor) clearTaskState(taskItem *task.Task) { - delete(taskItem.Metadata.Labels, "source_scrolled") - delete(taskItem.Metadata.Labels, "target_scrolled") -} diff --git a/plugin/task_manager/index_migration/index_migration.go b/plugin/task_manager/index_migration/index_migration.go deleted file mode 100644 index df780e68..00000000 --- a/plugin/task_manager/index_migration/index_migration.go +++ /dev/null @@ -1,577 +0,0 @@ -package index_migration - -import ( - "errors" - "fmt" - "time" - - log "github.com/cihub/seelog" - - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic/common" -) - -type processor struct { - Elasticsearch string - IndexName string - scheduler migration_model.Scheduler -} - -func NewProcessor(elasticsearch, indexName string, scheduler migration_model.Scheduler) migration_model.Processor { - return &processor{ - Elasticsearch: elasticsearch, - IndexName: indexName, - scheduler: scheduler, - } -} - -func (p *processor) Process(t *task.Task) (err error) { - switch t.Status { - case task.StatusReady: - // split & schedule pipline tasks - err = p.handleReadySubTask(t) - case task.StatusRunning: - // check pipeline tasks status - err = p.handleRunningSubTask(t) - case task.StatusPendingStop: - // mark pipeline tasks as pending_stop - err = p.handlePendingStopSubTask(t) - } - return err -} - -func (p *processor) handlePendingStopSubTask(taskItem *task.Task) error { - err := migration_util.UpdatePendingChildTasksToPendingStop(taskItem, "pipeline") - if err != nil { - log.Errorf("failed to update sub task status, err: %v", err) - return nil - } - - tasks, err := migration_util.GetPendingChildTasks(taskItem.ID, "pipeline") - if err != nil { - log.Errorf("failed to get sub tasks, err: %v", err) - return nil - } - - // all subtask stopped or error or complete - if len(tasks) == 0 { - taskItem.Status = task.StatusStopped - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("index migration task [%s] stopped", taskItem.ID)) - } - return nil -} - -func (p *processor) handleReadySubTask(taskItem *task.Task) error { - if ok, _ := util.ExtractBool(taskItem.Metadata.Labels["is_split"]); !ok { - return p.handleSplitSubTask(taskItem) - } - - return p.handleScheduleSubTask(taskItem) -} - -func (p *processor) handleSplitSubTask(taskItem *task.Task) error { - //split task to scroll/bulk_indexing pipeline and then persistent - var pids []string - pids = append(pids, taskItem.ParentId...) - pids = append(pids, taskItem.ID) - scrollID := util.GetUUID() - cfg := migration_model.IndexMigrationTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - return fmt.Errorf("got wrong config [%v] with task [%s], err: %v", taskItem.ConfigString, taskItem.ID, err) - } - sourceClusterID := cfg.Source.ClusterId - targetClusterID := cfg.Target.ClusterId - docType := common.GetClusterDocType(targetClusterID) - if len(taskItem.ParentId) == 0 { - return fmt.Errorf("got wrong parent id of task [%v]", *taskItem) - } - - err = migration_util.DeleteChildTasks(taskItem.ID, "pipeline") - if err != nil { - log.Warnf("failed to clear child tasks, err: %v", err) - return nil - } - - indexName := cfg.Source.Indices - scrollTask := &task.Task{ - ParentId: pids, - Runnable: true, - Cancellable: true, - Metadata: task.Metadata{ - Type: "pipeline", - Labels: util.MapStr{ - "cluster_id": sourceClusterID, - "pipeline_id": "es_scroll", - "index_name": indexName, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - }, - Status: task.StatusInit, - RetryTimes: taskItem.RetryTimes, - ConfigString: util.MustToJSON(migration_model.PipelineTaskConfig{ - Name: scrollID, - Logging: migration_model.PipelineTaskLoggingConfig{ - Enabled: true, - }, - Labels: util.MapStr{ - "parent_task_id": pids, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - AutoStart: true, - KeepRunning: false, - Processor: []util.MapStr{ - { - "es_scroll": util.MapStr{ - "remove_type": docType == "", - "slice_size": cfg.Source.SliceSize, - "batch_size": cfg.Source.BatchSize, - "indices": indexName, - "elasticsearch": sourceClusterID, - "bulk_operation": cfg.Target.Bulk.Operation, - "queue": util.MapStr{ - "name": scrollID, - "labels": util.MapStr{ - "migration_task_id": taskItem.ID, - }, - }, - "partition_size": 1, - "scroll_time": cfg.Source.ScrollTime, - "query_dsl": util.MustToJSON(util.MapStr{ - "query": cfg.Source.QueryDSL, - }), - "index_rename": cfg.Source.IndexRename, - "type_rename": cfg.Source.TypeRename, - }, - }, - }, - }), - } - scrollTask.ID = scrollID - - bulkID := util.GetUUID() - bulkTask := &task.Task{ - ParentId: pids, - Runnable: true, - Cancellable: true, - Metadata: task.Metadata{ - Type: "pipeline", - Labels: util.MapStr{ - "cluster_id": targetClusterID, - "pipeline_id": "bulk_indexing", - "index_name": indexName, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - }, - Status: task.StatusInit, - RetryTimes: taskItem.RetryTimes, - ConfigString: util.MustToJSON(migration_model.PipelineTaskConfig{ - Name: bulkID, - Logging: migration_model.PipelineTaskLoggingConfig{ - Enabled: true, - }, - Labels: util.MapStr{ - "parent_task_id": pids, - "unique_index_name": taskItem.Metadata.Labels["unique_index_name"], - }, - AutoStart: true, - KeepRunning: false, - Processor: []util.MapStr{ - { - "bulk_indexing": util.MapStr{ - "detect_active_queue": false, - "bulk": util.MapStr{ - "batch_size_in_mb": cfg.Target.Bulk.BatchSizeInMB, - "batch_size_in_docs": cfg.Target.Bulk.BatchSizeInDocs, - "invalid_queue": "bulk_indexing_400", - "compress": cfg.Target.Bulk.Compress, - "bulk_operation": cfg.Target.Bulk.Operation, - }, - "max_worker_size": cfg.Target.Bulk.MaxWorkerSize, - "num_of_slices": cfg.Target.Bulk.SliceSize, - "idle_timeout_in_seconds": cfg.Target.Bulk.IdleTimeoutInSeconds, - "elasticsearch": targetClusterID, - "queues": util.MapStr{ - "type": "scroll_docs", - "migration_task_id": taskItem.ID, - }, - }, - }, - }, - }), - } - bulkTask.ID = bulkID - - err = orm.Create(nil, scrollTask) - if err != nil { - return fmt.Errorf("create scroll pipeline task error: %w", err) - } - err = orm.Create(nil, bulkTask) - if err != nil { - return fmt.Errorf("create bulk_indexing pipeline task error: %w", err) - } - - taskItem.Metadata.Labels["is_split"] = true - taskItem.Status = task.StatusReady - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("task [%s] splitted", taskItem.ID)) - return nil -} - -func (p *processor) handleScheduleSubTask(taskItem *task.Task) error { - cfg := migration_model.IndexMigrationTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return fmt.Errorf("got wrong config of task [%s]", taskItem.ID) - } - - scrollTask, bulkTask, err := p.getScrollBulkPipelineTasks(taskItem) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return nil - } - if scrollTask == nil || bulkTask == nil { - // ES might not synced yet - log.Warnf("task [%s] es_scroll or bulk_indexing pipeline task not found", taskItem.ID) - return nil - } - - taskItem.RetryTimes++ - - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - totalDocs := cfg.Source.DocCount - scrolled, _, err := p.checkScrollPipelineTaskStatus(scrollTask, &cfg, totalDocs) - - redoScroll := true - if cfg.Version >= migration_model.IndexMigrationV1 { - // skip scroll if possible - if scrolled && err == nil { - redoScroll = false - // reset queue consumer offset - // NOTE: we only trigger this flow when restart index_migration - // Restart bulk task will not reset queue offset - err = p.resetGatewayQueue(taskItem) - if err != nil { - log.Infof("task [%s] failed to reset gateway queue, redo scroll", taskItem.ID) - redoScroll = true - } - } - } - if !redoScroll { - migration_util.WriteLog(taskItem, nil, fmt.Sprintf("task [%s] skiping scroll", taskItem.ID)) - } else { - var executionConfig migration_model.ExecutionConfig - if cfg.Version >= migration_model.IndexMigrationV1 { - executionConfig = cfg.Execution - } else { - executionConfig, err = p.getExecutionConfigFromMajorTask(taskItem) - if err != nil { - return fmt.Errorf("get execution config from parent task failed, err: %v", err) - } - } - - // get a new instanceID - instance, err := p.scheduler.GetPreferenceInstance(executionConfig) - if err != nil { - if err == migration_model.ErrHitMax { - log.Debug("hit max tasks per instance, skip dispatch") - return nil - } - return fmt.Errorf("get preference intance error: %w", err) - } - instanceID = instance.ID - - scrollTask.RetryTimes = taskItem.RetryTimes - // update instance info first - scrollTask.Metadata.Labels["execution_instance_id"] = instanceID - // try to clear disk queue before running es_scroll - p.cleanGatewayQueue(taskItem) - // update scroll task to ready - scrollTask.Status = task.StatusReady - err = orm.Update(nil, scrollTask) - if err != nil { - return fmt.Errorf("update scroll pipeline task error: %w", err) - } - } - - // update bulk task to init - bulkTask.RetryTimes = taskItem.RetryTimes - bulkTask.Metadata.Labels["execution_instance_id"] = instanceID - bulkTask.Status = task.StatusInit - err = orm.Update(nil, bulkTask) - if err != nil { - return fmt.Errorf("update bulk_indexing pipeline task error: %w", err) - } - - // update sub migration task status to running and save task log - taskItem.Metadata.Labels["execution_instance_id"] = instanceID - p.clearTaskState(taskItem) - taskItem.Status = task.StatusRunning - taskItem.StartTimeInMillis = time.Now().UnixMilli() - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("task [%s] started", taskItem.ID)) - // update dispatcher state - p.scheduler.IncrInstanceJobs(instanceID) - return nil -} - -func (p *processor) handleRunningSubTask(taskItem *task.Task) error { - cfg := migration_model.IndexMigrationTaskConfig{} - err := migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return fmt.Errorf("got wrong config of task [%s]", taskItem.ID) - } - totalDocs := cfg.Source.DocCount - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - - if totalDocs == 0 { - taskItem.Status = task.StatusComplete - p.clearTaskState(taskItem) - now := time.Now() - taskItem.CompletedTime = &now - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, "empty index migration completed") - p.cleanGatewayQueue(taskItem) - p.scheduler.DecrInstanceJobs(instanceID) - return nil - } - - scrollTask, bulkTask, err := p.getScrollBulkPipelineTasks(taskItem) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return nil - } - if scrollTask == nil || bulkTask == nil { - return errors.New("scroll/bulk pipeline task missing") - } - - scrolled, scrolledDocs, err := p.checkScrollPipelineTaskStatus(scrollTask, &cfg, totalDocs) - if !scrolled { - return nil - } - if err != nil { - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Status = task.StatusError - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: err.Error(), - }, "index scroll failed") - p.scheduler.DecrInstanceJobs(instanceID) - // clean disk queue if scroll failed - p.cleanGatewayQueue(taskItem) - return nil - } - - if migration_util.GetMapIntValue(util.MapStr(taskItem.Metadata.Labels), "scrolled_docs") == 0 { - taskItem.Metadata.Labels["scrolled_docs"] = scrolledDocs - p.saveTaskAndWriteLog(taskItem, nil, "") - } - - bulked, successDocs, err := p.checkBulkPipelineTaskStatus(bulkTask, &cfg, totalDocs) - if !bulked { - return nil - } - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Metadata.Labels["scrolled_docs"] = scrolledDocs - taskItem.Metadata.Labels["index_docs"] = successDocs - if err != nil { - taskItem.Status = task.StatusError - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: err.Error(), - }, "index bulk failed") - } else { - taskItem.Status = task.StatusComplete - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, "index migration completed") - // clean disk queue if bulk completed - p.cleanGatewayQueue(taskItem) - } - p.scheduler.DecrInstanceJobs(instanceID) - return nil -} - -func (p *processor) checkScrollPipelineTaskStatus(scrollTask *task.Task, cfg *migration_model.IndexMigrationTaskConfig, totalDocs int64) (scrolled bool, scrolledDocs int64, err error) { - if scrollTask.Status == task.StatusError { - return true, 0, errors.New("scroll pipeline failed") - } - // NOTE: old-version pipeline tasks has empty status - if scrollTask.Status == "" { - return true, 0, errors.New("task was started by an old-version console, need to manually restart it") - } - - // scroll not finished yet - if scrollTask.Status != task.StatusComplete { - return false, 0, nil - } - - var ( - scrollLabels = util.MapStr(scrollTask.Metadata.Labels) - ) - scrolledDocs = migration_util.GetMapIntValue(scrollLabels, "scrolled_docs") - - if !cfg.Source.SkipCountCheck && scrolledDocs != totalDocs { - return true, scrolledDocs, fmt.Errorf("scroll complete but docs count unmatch: %d / %d", scrolledDocs, totalDocs) - } - - return true, scrolledDocs, nil -} - -func (p *processor) checkBulkPipelineTaskStatus(bulkTask *task.Task, cfg *migration_model.IndexMigrationTaskConfig, totalDocs int64) (bulked bool, successDocs int64, err error) { - // NOTE: old-version pipeline tasks has empty status - if bulkTask.Status == "" { - return true, 0, errors.New("task was started by an old-version console, need to manually restart it") - } - - // start bulk as needed - if bulkTask.Status == task.StatusInit { - bulkTask.Status = task.StatusReady - p.saveTaskAndWriteLog(bulkTask, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("scroll completed, bulk pipeline started")) - return false, 0, nil - } - - // bulk not finished yet - if bulkTask.Status != task.StatusComplete && bulkTask.Status != task.StatusError { - return false, 0, nil - } - - var ( - bulkLabels = util.MapStr(bulkTask.Metadata.Labels) - invalidDocs = migration_util.GetMapStringValue(bulkLabels, "invalid_docs") - invalidReasons = migration_util.GetMapStringValue(bulkLabels, "invalid_reasons") - failureDocs = migration_util.GetMapStringValue(bulkLabels, "failure_docs") - failureReasons = migration_util.GetMapStringValue(bulkLabels, "failure_reasons") - ) - successDocs = migration_util.GetMapIntValue(bulkLabels, "success_docs") - - if successDocs != totalDocs { - // check count - if !cfg.Target.SkipCountCheck { - return true, successDocs, fmt.Errorf("bulk complete but docs count unmatch: %d / %d, invalid docs: [%s] (reasons: [%s]), failure docs: [%s] (reasons: [%s])", successDocs, totalDocs, invalidDocs, invalidReasons, failureDocs, failureReasons) - } - // has errors - if bulkTask.Status == task.StatusError { - return true, successDocs, fmt.Errorf("bulk pipeline failed") - } - } - - // successDocs matched, return ok - return true, successDocs, nil -} - -func (p *processor) getExecutionConfigFromMajorTask(taskItem *task.Task) (config migration_model.ExecutionConfig, err error) { - majorTaskID := migration_util.GetDirectParentId(taskItem.ParentId) - majorTask := task.Task{} - majorTask.ID = majorTaskID - _, err = orm.Get(&majorTask) - if err != nil { - log.Errorf("failed to get major task, err: %v", err) - return - } - cfg := migration_model.ClusterMigrationTaskConfig{} - err = migration_util.GetTaskConfig(&majorTask, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return - } - config = cfg.Settings.Execution - return -} - -func (p *processor) getScrollBulkPipelineTasks(taskItem *task.Task) (scrollTask *task.Task, bulkTask *task.Task, err error) { - ptasks, err := migration_util.GetChildTasks(taskItem.ID, "pipeline", nil) - if err != nil { - log.Errorf("failed to get pipeline tasks, err: %v", err) - return - } - if len(ptasks) != 2 { - err = fmt.Errorf("invalid pipeline task count: %d", len(ptasks)) - return - } - scrollTask, bulkTask = migration_util.SplitIndexMigrationTasks(ptasks) - return -} - -func (p *processor) cleanGatewayQueue(taskItem *task.Task) { - log.Debugf("cleaning gateway queue for task [%s]", taskItem.ID) - - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - if instanceID == "" { - log.Debugf("task [%s] not scheduled yet, skip cleaning queue", taskItem.ID) - return - } - instance, err := p.scheduler.GetInstance(instanceID) - if err != nil { - log.Errorf("failed to get instance, err: %v", err) - return - } - - selector := util.MapStr{ - "labels": util.MapStr{ - "migration_task_id": taskItem.ID, - }, - } - err = instance.DeleteQueueBySelector(selector) - if err != nil { - log.Errorf("failed to delete queue, err: %v", err) - } -} - -func (p *processor) resetGatewayQueue(taskItem *task.Task) error { - log.Debugf("resetting gateway queue offset for task [%s]", taskItem.ID) - - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - instance, err := p.scheduler.GetInstance(instanceID) - if err != nil { - log.Errorf("failed to get instance, err: %v", err) - return err - } - - selector := util.MapStr{ - "labels": util.MapStr{ - "migration_task_id": taskItem.ID, - }, - } - err = instance.DeleteQueueConsumersBySelector(selector) - if err != nil { - log.Errorf("failed to delete queue consumers, err: %v", err) - return err - } - - return nil -} - -func (p *processor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.Elasticsearch) - _, err := esClient.Index(p.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -func (p *processor) clearTaskState(taskItem *task.Task) { - delete(taskItem.Metadata.Labels, "index_docs") - delete(taskItem.Metadata.Labels, "scrolled_docs") -} diff --git a/plugin/task_manager/migration_api.go b/plugin/task_manager/migration_api.go deleted file mode 100644 index 06b423c8..00000000 --- a/plugin/task_manager/migration_api.go +++ /dev/null @@ -1,505 +0,0 @@ -package task_manager - -import ( - "fmt" - "net/http" - "strings" - "time" - - log "github.com/cihub/seelog" - - "infini.sh/console/plugin/task_manager/cluster_migration" - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/api/rbac" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/global" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - clusterTaskConfig := &migration_model.ClusterMigrationTaskConfig{} - err := h.DecodeJSON(req, clusterTaskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - user, err := rbac.FromUserContext(req.Context()) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - t, err := cluster_migration.CreateTask(clusterTaskConfig, user) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteCreatedOKJSON(w, t.ID) -} - -func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - - obj := task.Task{} - obj.ID = id - - exists, err := orm.Get(&obj) - if !exists || err != nil { - h.WriteJSON(w, util.MapStr{ - "_id": id, - "found": false, - }, http.StatusNotFound) - return - } - taskConfig := &migration_model.ClusterMigrationTaskConfig{} - err = migration_util.GetTaskConfig(&obj, taskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - _, indexState, err := h.getMigrationMajorTaskInfo(id) - if err != nil { - log.Errorf("failed to get major task info, err: %v", err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - var completedIndices int - for i, index := range taskConfig.Indices { - indexName := index.Source.GetUniqueIndexName() - count := indexState[indexName].IndexDocs - sourceDocs := index.Source.Docs - var percent float64 - var exportedPercent float64 - if sourceDocs <= 0 { - percent = 100 - exportedPercent = 100 - }else{ - percent = float64(count) / float64(sourceDocs) * 100 - if percent > 100 { - percent = 100 - } - exportedPercent = float64(indexState[indexName].ScrollDocs)/float64(sourceDocs) * 100 - if exportedPercent > 100 { - exportedPercent = 100 - } - } - //taskConfig.Indices[i].Source.Docs = sourceDocs - taskConfig.Indices[i].Target.Docs = count - taskConfig.Indices[i].Percent = util.ToFixed(percent, 2) - taskConfig.Indices[i].ErrorPartitions = indexState[indexName].ErrorPartitions - taskConfig.Indices[i].RunningChildren = indexState[indexName].RunningChildren - taskConfig.Indices[i].ExportedPercent = util.ToFixed(exportedPercent, 2) - if count == index.Source.Docs { - completedIndices++ - } - } - - cfg := global.MustLookup("cluster_migration_config") - if migrationConfig, ok := cfg.(*DispatcherConfig); ok { - if obj.Metadata.Labels == nil { - obj.Metadata.Labels = util.MapStr{} - } - obj.Metadata.Labels["log_info"] = util.MapStr{ - "cluster_id": migrationConfig.Elasticsearch, - "index_name": migrationConfig.LogIndexName, - } - } - - _, repeatStatus, err := h.calcRepeatingStatus(&obj) - if err != nil { - log.Warnf("failed to calc repeat info, err: %v", err) - } - obj.Metadata.Labels["repeat"] = repeatStatus - - obj.ConfigString = util.MustToJSON(taskConfig) - obj.Metadata.Labels["completed_indices"] = completedIndices - h.WriteJSON(w, obj, http.StatusOK) -} - -func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.MustGetParameter("task_id") - uniqueIndexName := ps.MustGetParameter("index") - majorTask := task.Task{} - majorTask.ID = id - exists, err := orm.Get(&majorTask) - if !exists || err != nil { - h.WriteError(w, fmt.Sprintf("task [%s] not found", id), http.StatusInternalServerError) - return - } - - taskConfig := &migration_model.ClusterMigrationTaskConfig{} - err = migration_util.GetTaskConfig(&majorTask, taskConfig) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - taskInfo := &TaskInfoResponse{ - TaskID: id, - StartTime: majorTask.StartTimeInMillis, - Repeating: migration_util.IsRepeating(taskConfig.Settings.Execution.Repeat, majorTask.Metadata.Labels), - } - if taskInfo.Repeating { - _, repeatStatus, err := h.calcRepeatingStatus(&majorTask) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - taskInfo.NextRunTime = repeatStatus.NextRunTime - } - indexParts := strings.Split(uniqueIndexName, ":") - for _, index := range taskConfig.Indices { - if index.Source.Name == indexParts[0] { - taskInfo.Incremental = index.Incremental - } - } - - subTasks, pipelineTaskIDs, pipelineSubParentIDs, parentIDPipelineTasks, err := h.getChildTaskInfosByIndex(id, uniqueIndexName) - - taskInfo.DataPartition = len(subTasks) - if len(subTasks) == 0 { - h.WriteJSON(w, taskInfo, http.StatusOK) - return - } - - var scrollStats = map[string]int64{} - var bulkStats = map[string]int64{} - pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs) - for pipelineID, pipelineContext := range pipelineContexts { - if pid, ok := pipelineSubParentIDs[pipelineID]; ok { - if vv := migration_util.GetMapIntValue(pipelineContext, "es_scroll.scrolled_docs"); vv > 0 { - scrollStats[pid] = vv - } - if vv := migration_util.GetMapIntValue(pipelineContext, "bulk_indexing.success.count"); vv > 0 { - bulkStats[pid] = vv - } - } - } - - startTime, completedTime, duration, completedPartitions := h.calcMajorTaskInfo(subTasks, taskInfo.Repeating) - - var partitionTaskInfos []util.MapStr - var workers = map[string]struct{}{} - - for i, ptask := range subTasks { - cfg := migration_model.IndexMigrationTaskConfig{} - err := migration_util.GetTaskConfig(&ptask, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - continue - } - if i == 0 { - taskInfo.Step = cfg.Source.Step - } - instID := migration_util.GetMapStringValue(ptask.Metadata.Labels, "execution_instance_id") - if instID != "" { - workers[instID] = struct{}{} - } - var durationInMS int64 - var subCompletedTime int64 - if ptask.StartTimeInMillis > 0 { - if migration_util.IsPendingState(ptask.Status) { - durationInMS = time.Now().UnixMilli() - ptask.StartTimeInMillis - } else if ptask.CompletedTime != nil { - subCompletedTime = ptask.CompletedTime.UnixMilli() - durationInMS = subCompletedTime - ptask.StartTimeInMillis - } - } - var ( - scrollDocs int64 - indexDocs int64 - ) - ptaskLabels := util.MapStr(ptask.Metadata.Labels) - if vv, ok := scrollStats[ptask.ID]; ok { - scrollDocs = vv - } else { - scrollDocs = migration_util.GetMapIntValue(ptaskLabels, "scrolled_docs") - } - if vv, ok := bulkStats[ptask.ID]; ok { - indexDocs = vv - } else { - indexDocs = migration_util.GetMapIntValue(ptaskLabels, "index_docs") - } - - partitionTotalDocs := cfg.Source.DocCount - partitionTaskInfo := util.MapStr{ - "task_id": ptask.ID, - "status": ptask.Status, - "start_time": ptask.StartTimeInMillis, - "completed_time": subCompletedTime, - "start": cfg.Source.Start, - "end": cfg.Source.End, - "duration": durationInMS, - "scroll_docs": scrollDocs, - "index_docs": indexDocs, - "total_docs": partitionTotalDocs, - } - scrollTask, bulkTask := migration_util.SplitIndexMigrationTasks(parentIDPipelineTasks[ptask.ID]) - if scrollTask != nil { - partitionTaskInfo["scroll_task"] = util.MapStr{ - "id": scrollTask.ID, - "status": scrollTask.Status, - } - } - if bulkTask != nil { - partitionTaskInfo["bulk_task"] = util.MapStr{ - "id": bulkTask.ID, - "status": bulkTask.Status, - } - } - partitionTaskInfos = append(partitionTaskInfos, partitionTaskInfo) - } - taskInfo.CompletedTime = completedTime - taskInfo.Duration = duration - // NOTE: overwrite major task start time with the first started sub task - if taskInfo.StartTime == 0 { - taskInfo.StartTime = startTime - } - taskInfo.Partitions = partitionTaskInfos - taskInfo.CompletedPartitions = completedPartitions - for _, node := range taskConfig.Settings.Execution.Nodes.Permit { - if _, ok := workers[node.ID]; ok { - taskInfo.Workers = append(taskInfo.Workers, util.MapStr{ - "id": node.ID, - "name": node.Name, - }) - } - } - h.WriteJSON(w, taskInfo, http.StatusOK) -} - -type MigrationIndexStateInfo struct { - ErrorPartitions int - IndexDocs int64 - SourceDocs int64 - RunningChildren int - ScrollDocs int64 -} - -/* -We count data from two sources: - - index_migrations with complete/error status - - plus index_migration.index_docs with realtime bulk indexing info - - realtime bulk indexing info is only available for running index_migrations -*/ -func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_model.ClusterMigrationTaskState, indexState map[string]MigrationIndexStateInfo, err error) { - var pipelineTaskIDs = map[string][]string{} - var pipelineIndexNames = map[string]string{} - indexState = map[string]MigrationIndexStateInfo{} - const size = 500 - var ( - from = -size - hasMore = true - ) - for hasMore { - from += size - taskQuery := util.MapStr{ - "from": from, - "size": size, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "asc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": id, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_migration", - }, - }, - }, - }, - }, - }, - } - subTasks, err := migration_util.GetTasks(taskQuery) - if err != nil { - return taskStats, indexState, err - } - if len(subTasks) < size { - hasMore = false - } - - var indexMigrationTaskIDs []string - for _, subTask := range subTasks { - taskLabels := util.MapStr(subTask.Metadata.Labels) - indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") - if indexName == "" { - continue - } - - cfg := migration_model.IndexMigrationTaskConfig{} - err = migration_util.GetTaskConfig(&subTask, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - continue - } - - taskStats.SourceDocs += cfg.Source.DocCount - st := indexState[indexName] - st.SourceDocs += cfg.Source.DocCount - scrollDocs := migration_util.GetMapIntValue(taskLabels, "scrolled_docs") - st.ScrollDocs += scrollDocs - - if subTask.Status == task.StatusRunning { - st.RunningChildren++ - indexState[indexName] = st - indexMigrationTaskIDs = append(indexMigrationTaskIDs, subTask.ID) - continue - } - - indexDocs := migration_util.GetMapIntValue(taskLabels, "index_docs") - taskStats.IndexDocs += indexDocs - st.IndexDocs += indexDocs - - if subTask.Status == task.StatusError { - st.ErrorPartitions += 1 - taskStats.ErrorPartitions += 1 - } - indexState[indexName] = st - indexMigrationTaskIDs = append(indexMigrationTaskIDs, subTask.ID) - } - - if len(indexMigrationTaskIDs) == 0 { - continue - } - - taskQuery = util.MapStr{ - "size": len(indexMigrationTaskIDs) * 2, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "parent_id": indexMigrationTaskIDs, - }, - }, - //{ - // "term": util.MapStr{ - // "metadata.labels.pipeline_id": util.MapStr{ - // "value": "bulk_indexing", - // }, - // }, - //}, - }, - }, - }, - } - subTasks, err = migration_util.GetTasks(taskQuery) - if err != nil { - return taskStats, indexState, err - } - - for _, subTask := range subTasks { - taskLabels := util.MapStr(subTask.Metadata.Labels) - indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") - if indexName == "" { - continue - } - - pipelineIndexNames[subTask.ID] = indexName - - if instID := migration_util.GetMapStringValue(taskLabels, "execution_instance_id"); instID != "" { - pipelineTaskIDs[instID] = append(pipelineTaskIDs[instID], subTask.ID) - } - } - } - - pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs) - for pipelineID, pipelineContext := range pipelineContexts { - // add indexDocs of running tasks - indexDocs := migration_util.GetMapIntValue(pipelineContext, "bulk_indexing.success.count") - scrollDocs := migration_util.GetMapIntValue(pipelineContext, "es_scroll.scrolled_docs") - taskStats.IndexDocs += indexDocs - indexName := pipelineIndexNames[pipelineID] - st := indexState[indexName] - st.IndexDocs += indexDocs - st.ScrollDocs += scrollDocs - indexState[indexName] = st - } - return taskStats, indexState, nil -} - -func (h *APIHandler) restartAllFailedPartitions(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - mustQ := []util.MapStr{ - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "cluster_migration", - }, - }, - }, - { - "term": util.MapStr{ - "status": util.MapStr{ - "value": "error", - }, - }, - }, - } - - queryDSL := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - }, - }, - "script": util.MapStr{ - "source": fmt.Sprintf("ctx._source['status'] = '%s'", task.StatusRunning), - }, - } - - body := util.MustToJSONBytes(queryDSL) - - err := orm.UpdateBy(&task.Task{}, body) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - //update status of sub task - mustQ[0] = util.MapStr{ - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_migration", - }, - }, - } - queryDSL["script"] = util.MapStr{ - "source": fmt.Sprintf("ctx._source['status'] = '%s'", task.StatusReady), - } - body = util.MustToJSONBytes(queryDSL) - err = orm.UpdateBy(&task.Task{}, body) - if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteAckOKJSON(w) -} \ No newline at end of file diff --git a/plugin/task_manager/model/common.go b/plugin/task_manager/model/common.go deleted file mode 100644 index 343e8382..00000000 --- a/plugin/task_manager/model/common.go +++ /dev/null @@ -1,93 +0,0 @@ -package model - -import ( - "fmt" - "infini.sh/framework/core/task" - "time" - - "infini.sh/framework/core/util" -) - -type ExecutionConfig struct { - TimeWindow []TimeWindowItem `json:"time_window"` - Repeat *Repeat `json:"repeat"` - Nodes struct { - Permit []ExecutionNode `json:"permit"` - } `json:"nodes"` -} - -type ExecutionNode struct { - ID string `json:"id"` - Name string `json:"name"` -} - -type Repeat struct { - NextRunTime *time.Time `json:"next_run_time"` - Interval time.Duration `json:"interval"` - TotalRun int64 `json:"total_run"` -} - -type TimeWindowItem struct { - Start string `json:"start"` - End string `json:"end"` -} - -type IndexPartition struct { - FieldType string `json:"field_type"` - FieldName string `json:"field_name"` - Step interface{} `json:"step"` - //only worked when field type equals number - UseEvenStrategy bool `json:"use_even_strategy"` -} - -type IndexIncremental struct { - FieldName string `json:"field_name"` - // Optional, data ingest delay - Delay time.Duration `json:"delay"` - // If full, run the data from -inf, else from current - step - Full bool `json:"full"` -} - -type IndexInfo struct { - Name string `json:"name"` - DocType string `json:"doc_type"` - // NOTE: == 0 for migration target index - Docs int64 `json:"docs"` - StoreSizeInBytes int `json:"store_size_in_bytes"` -} - -func (ii *IndexInfo) GetUniqueIndexName() string { - return fmt.Sprintf("%s:%s", ii.Name, ii.DocType) -} - -type ClusterInfo struct { - Id string `json:"id"` - Name string `json:"name"` - Distribution string `json:"distribution,omitempty"` -} - -// BuildFilter generate a query filter, used by split task -func (incremental *IndexIncremental) BuildFilter(current int64, step time.Duration) (util.MapStr, error) { - if incremental == nil { - return util.MapStr{}, nil - } - - rv := util.MapStr{ - "lt": int64(current) - incremental.Delay.Milliseconds(), - "format": "epoch_millis", - } - if !incremental.Full { - rv["gte"] = int64(current) - step.Milliseconds() - incremental.Delay.Milliseconds() - } - return util.MapStr{ - "range": util.MapStr{ - incremental.FieldName: rv, - }, - }, nil -} - -type QueryTask struct { - Type string - Status []string - TaskHandler func(taskItem *task.Task) error -} \ No newline at end of file diff --git a/plugin/task_manager/model/comparison.go b/plugin/task_manager/model/comparison.go deleted file mode 100644 index b64d61ce..00000000 --- a/plugin/task_manager/model/comparison.go +++ /dev/null @@ -1,67 +0,0 @@ -package model - -import ( - "infini.sh/framework/core/util" -) - -type ClusterComparisonTaskConfig struct { - Name string `json:"name"` - Tags []string `json:"tags"` - Cluster struct { - Source ClusterInfo `json:"source"` - Target ClusterInfo `json:"target"` - } `json:"cluster"` - Indices []ClusterComparisonIndexConfig `json:"indices"` - Settings struct { - Dump DumpHashConfig `json:"dump"` - Diff IndexDiffConfig `json:"diff"` - Execution ExecutionConfig `json:"execution"` - } `json:"settings"` - Creator struct { - Name string `json:"name"` - Id string `json:"id"` - } `json:"creator"` -} - -type ClusterComparisonIndexConfig struct { - Source IndexInfo `json:"source"` - Target IndexInfo `json:"target"` - RawFilter interface{} `json:"raw_filter"` - Incremental *IndexIncremental `json:"incremental"` - Partition *IndexPartition `json:"partition,omitempty"` - - // only used in API - ScrollPercent float64 `json:"scroll_percent,omitempty"` - TotalScrollDocs int64 `json:"total_scroll_docs,omitempty"` - ErrorPartitions int `json:"error_partitions,omitempty"` - RunningChildren int `json:"running_children,omitempty"` -} - -type IndexComparisonTaskConfig struct { - Source IndexComparisonDumpConfig `json:"source"` - Target IndexComparisonDumpConfig `json:"target"` - Diff IndexComparisonDiffConfig `json:"diff"` - Execution ExecutionConfig `json:"execution"` -} - -type IndexComparisonDumpConfig struct { - ClusterId string `json:"cluster_id"` - Indices string `json:"indices"` - - SliceSize int `json:"slice_size"` - BatchSize int `json:"batch_size"` - PartitionSize int `json:"partition_size"` - ScrollTime string `json:"scroll_time"` - QueryString string `json:"query_string,omitempty"` - QueryDSL util.MapStr `json:"query_dsl,omitempty"` - DocCount int64 `json:"doc_count"` - - // Only populated for partitioned tasks - Start float64 `json:"start"` - End float64 `json:"end"` - Step interface{} `jsno:"step"` - PartitionId int `json:"partition_id"` -} - -type IndexComparisonDiffConfig struct { -} diff --git a/plugin/task_manager/model/migration.go b/plugin/task_manager/model/migration.go deleted file mode 100644 index dde4bfbc..00000000 --- a/plugin/task_manager/model/migration.go +++ /dev/null @@ -1,103 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package model - -import ( - "infini.sh/framework/core/util" -) - -type ClusterMigrationTaskConfig struct { - Name string `json:"name"` - Tags []string `json:"tags"` - Cluster struct { - Source ClusterInfo `json:"source"` - Target ClusterInfo `json:"target"` - } `json:"cluster"` - Indices []ClusterMigrationIndexConfig `json:"indices"` - Settings struct { - Scroll EsScrollConfig `json:"scroll"` - Bulk BulkIndexingConfig `json:"bulk"` - SkipScrollCountCheck bool `json:"skip_scroll_count_check"` - SkipBulkCountCheck bool `json:"skip_bulk_count_check"` - Execution ExecutionConfig `json:"execution"` - } `json:"settings"` - Creator struct { - Name string `json:"name"` - Id string `json:"id"` - } `json:"creator"` -} - -type ClusterMigrationIndexConfig struct { - Source IndexInfo `json:"source"` - Target IndexInfo `json:"target"` - RawFilter interface{} `json:"raw_filter"` - IndexRename map[string]interface{} `json:"index_rename"` - TypeRename map[string]interface{} `json:"type_rename"` - Incremental *IndexIncremental `json:"incremental"` - Partition *IndexPartition `json:"partition,omitempty"` - - // only used in API - Percent float64 `json:"percent,omitempty"` - ErrorPartitions int `json:"error_partitions,omitempty"` - RunningChildren int `json:"running_children,omitempty"` - ExportedPercent float64 `json:"exported_percent,omitempty"` -} - -type ClusterMigrationTaskState struct { - IndexDocs int64 - SourceDocs int64 - ErrorPartitions int64 - Status string -} - -const ( - IndexMigrationV0 = 0 - IndexMigrationV1 = 1 -) - -type IndexMigrationTaskConfig struct { - Source IndexMigrationSourceConfig `json:"source"` - Target IndexMigrationTargetConfig `json:"target"` - Execution ExecutionConfig `json:"execution"` - Version int `json:"version"` -} - -type IndexMigrationSourceConfig struct { - SkipCountCheck bool `json:"skip_count_check"` - - ClusterId string `json:"cluster_id"` - Indices string `json:"indices"` - SliceSize int `json:"slice_size"` - BatchSize int `json:"batch_size"` - ScrollTime string `json:"scroll_time"` - IndexRename util.MapStr `json:"index_rename,omitempty"` - TypeRename util.MapStr `json:"type_rename,omitempty"` - QueryString string `json:"query_string,omitempty"` - QueryDSL util.MapStr `json:"query_dsl,omitempty"` - DocCount int64 `json:"doc_count"` - - // Parition configs - Start float64 `json:"start"` - End float64 `json:"end"` - Step interface{} `json:"step"` - PartitionId int `json:"partition_id"` -} - -type IndexMigrationBulkConfig struct { - BatchSizeInDocs int `json:"batch_size_in_docs"` - BatchSizeInMB int `json:"batch_size_in_mb"` - MaxWorkerSize int `json:"max_worker_size"` - IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"` - SliceSize int `json:"slice_size"` - Compress bool `json:"compress"` - Operation string `json:"operation"` -} - -type IndexMigrationTargetConfig struct { - SkipCountCheck bool `json:"skip_count_check"` - - ClusterId string `json:"cluster_id"` - Bulk IndexMigrationBulkConfig `json:"bulk"` -} diff --git a/plugin/task_manager/model/pipeline.go b/plugin/task_manager/model/pipeline.go deleted file mode 100644 index 823ff56f..00000000 --- a/plugin/task_manager/model/pipeline.go +++ /dev/null @@ -1,46 +0,0 @@ -package model - -import "infini.sh/framework/core/util" - -// tunable `es_scroll` configurations -type EsScrollConfig struct { - SliceSize int `json:"slice_size"` - Docs int `json:"docs"` - Timeout string `json:"timeout"` -} - -// tunable `dump_hash` configurations -type DumpHashConfig struct { - SliceSize int `json:"slice_size"` - PartitionSize int `json:"partition_size"` - Docs int `json:"docs"` - Timeout string `json:"timeout"` -} - -// tunable `index_diff` configurations -type IndexDiffConfig struct { -} - -// tunable `bulk_indexing` configurations -type BulkIndexingConfig struct { - Docs int `json:"docs"` - StoreSizeInMB int `json:"store_size_in_mb"` - MaxWorkerSize int `json:"max_worker_size"` - IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"` - SliceSize int `json:"slice_size"` - Compress bool `json:"compress"` - Operation string `json:"operation"` -} - -type PipelineTaskLoggingConfig struct { - Enabled bool `json:"enabled"` -} - -type PipelineTaskConfig struct { - Name string `json:"name"` - Logging PipelineTaskLoggingConfig `json:"logging"` - Labels util.MapStr `json:"labels"` - AutoStart bool `json:"auto_start"` - KeepRunning bool `json:"keep_running"` - Processor []util.MapStr `json:"processor"` -} diff --git a/plugin/task_manager/model/processor.go b/plugin/task_manager/model/processor.go deleted file mode 100644 index 477abdb4..00000000 --- a/plugin/task_manager/model/processor.go +++ /dev/null @@ -1,7 +0,0 @@ -package model - -import "infini.sh/framework/core/task" - -type Processor interface { - Process(t *task.Task) (err error) -} diff --git a/plugin/task_manager/model/scheduler.go b/plugin/task_manager/model/scheduler.go deleted file mode 100644 index cd595ae2..00000000 --- a/plugin/task_manager/model/scheduler.go +++ /dev/null @@ -1,16 +0,0 @@ -package model - -import ( - "errors" - "infini.sh/console/model" -) - -type Scheduler interface { - GetPreferenceInstance(config ExecutionConfig) (instance *model.TaskWorker, err error) - GetInstance(instanceID string) (instance *model.TaskWorker, err error) - IncrInstanceJobs(instanceID string) - DecrInstanceJobs(instanceID string) - RefreshInstanceJobsFromES() error -} - -var ErrHitMax = errors.New("instance hit max job limit") diff --git a/plugin/task_manager/module.go b/plugin/task_manager/module.go deleted file mode 100644 index ae8d0ca3..00000000 --- a/plugin/task_manager/module.go +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package task_manager - -import ( - log "github.com/cihub/seelog" - "infini.sh/framework/core/env" - "infini.sh/framework/core/module" -) - -func (module *Module) Name() string { - return "migration" -} - -func (module *Module) Setup() { - exists, err := env.ParseConfig("migration", module) - if exists && err != nil { - log.Error(err) - } - InitAPI() -} -func (module *Module) Start() error { - return nil -} - -func (module *Module) Stop() error { - return nil -} - -type Module struct { -} - -func init() { - module.RegisterUserPlugin(&Module{}) -} diff --git a/plugin/task_manager/pipeline.go b/plugin/task_manager/pipeline.go deleted file mode 100644 index afd4a65d..00000000 --- a/plugin/task_manager/pipeline.go +++ /dev/null @@ -1,248 +0,0 @@ -/* Copyright © INFINI Ltd. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package task_manager - -import ( - "errors" - "fmt" - "time" - - log "github.com/cihub/seelog" - - "infini.sh/console/plugin/task_manager/cluster_comparison" - "infini.sh/console/plugin/task_manager/cluster_migration" - "infini.sh/console/plugin/task_manager/index_comparison" - "infini.sh/console/plugin/task_manager/index_migration" - migration_model "infini.sh/console/plugin/task_manager/model" - "infini.sh/console/plugin/task_manager/pipeline_task" - "infini.sh/console/plugin/task_manager/scheduler" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/config" - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/env" - "infini.sh/framework/core/global" - "infini.sh/framework/core/pipeline" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic/common" -) - -type DispatcherProcessor struct { - id string - config *DispatcherConfig - - scheduler migration_model.Scheduler - pipelineTaskProcessor migration_model.Processor - clusterMigrationTaskProcessor migration_model.Processor - indexMigrationTaskProcessor migration_model.Processor - clusterComparisonTaskProcessor migration_model.Processor - indexComparisonTaskProcessor migration_model.Processor - queryTasks []migration_model.QueryTask -} - -type DispatcherConfig struct { - Elasticsearch string `config:"elasticsearch,omitempty"` - IndexName string `config:"index_name"` - LogIndexName string `config:"log_index_name"` - MaxTasksPerInstance int `config:"max_tasks_per_instance"` - CheckInstanceAvailable bool `config:"check_instance_available"` - TaskBatchSize int `config:"task_batch_size"` -} - -func init() { - pipeline.RegisterProcessorPlugin("migration_dispatcher", newMigrationDispatcherProcessor) -} - -func newMigrationDispatcherProcessor(c *config.Config) (pipeline.Processor, error) { - - cfg := DispatcherConfig{} - if err := c.Unpack(&cfg); err != nil { - log.Errorf("failed to unpack config, err: %v", err) - return nil, fmt.Errorf("failed to unpack the configuration of migration dispatcher processor: %s", err) - } - if cfg.IndexName == "" || cfg.LogIndexName == "" { - ormConfig := common.ORMConfig{} - ok, err := env.ParseConfig("elastic.orm", &ormConfig) - if ok && err == nil { - if cfg.IndexName == "" { - cfg.IndexName = fmt.Sprintf("%stask", ormConfig.IndexPrefix) - } - if cfg.LogIndexName == "" { - cfg.LogIndexName = fmt.Sprintf("%slogs", ormConfig.IndexPrefix) - } - } else { - err = fmt.Errorf("parse config elastic.orm error: %w", err) - log.Errorf("failed to parse elastic.orm, err: %v", err) - return nil, err - } - } - global.Register("cluster_migration_config", &cfg) - if cfg.MaxTasksPerInstance <= 0 { - cfg.MaxTasksPerInstance = 10 - } - if cfg.TaskBatchSize <= 0 { - cfg.TaskBatchSize = 50 - } - - //query and then init dispatcher state - processor := DispatcherProcessor{ - id: util.GetUUID(), - config: &cfg, - } - var err error - processor.scheduler, err = scheduler.NewScheduler(cfg.Elasticsearch, cfg.IndexName, cfg.CheckInstanceAvailable, cfg.MaxTasksPerInstance) - if err != nil { - return nil, err - } - processor.pipelineTaskProcessor = pipeline_task.NewProcessor(cfg.Elasticsearch, cfg.IndexName, cfg.LogIndexName, processor.scheduler) - processor.indexMigrationTaskProcessor = index_migration.NewProcessor(cfg.Elasticsearch, cfg.IndexName, processor.scheduler) - processor.clusterMigrationTaskProcessor = cluster_migration.NewProcessor(cfg.Elasticsearch, cfg.IndexName, processor.scheduler) - processor.indexComparisonTaskProcessor = index_comparison.NewProcessor(cfg.Elasticsearch, cfg.IndexName, processor.scheduler) - processor.clusterComparisonTaskProcessor = cluster_comparison.NewProcessor(cfg.Elasticsearch, cfg.IndexName, processor.scheduler) - processor.queryTasks = []migration_model.QueryTask{ - // handle pipeline task - {"pipeline", []string{task.StatusReady, task.StatusRunning, task.StatusPendingStop}, processor.pipelineTaskProcessor.Process}, - // handle comparison tasks - {"cluster_comparison", []string{task.StatusPendingStop}, processor.clusterComparisonTaskProcessor.Process}, - {"index_comparison", []string{task.StatusPendingStop}, processor.indexComparisonTaskProcessor.Process}, - {"index_comparison", []string{task.StatusPendingStop}, processor.indexComparisonTaskProcessor.Process}, - {"index_comparison", []string{task.StatusRunning}, processor.indexComparisonTaskProcessor.Process}, - {"index_comparison", []string{task.StatusReady}, processor.indexComparisonTaskProcessor.Process}, - {"cluster_comparison", []string{task.StatusRunning}, processor.clusterComparisonTaskProcessor.Process}, - {"cluster_comparison", []string{task.StatusReady}, processor.clusterComparisonTaskProcessor.Process}, - // handle migration tasks - {"cluster_migration", []string{task.StatusPendingStop}, processor.clusterMigrationTaskProcessor.Process}, - {"index_migration", []string{task.StatusPendingStop}, processor.indexMigrationTaskProcessor.Process}, - {"index_migration", []string{task.StatusRunning}, processor.indexMigrationTaskProcessor.Process}, - {"index_migration", []string{task.StatusReady}, processor.indexMigrationTaskProcessor.Process}, - {"cluster_migration", []string{task.StatusRunning}, processor.clusterMigrationTaskProcessor.Process}, - {"cluster_migration", []string{task.StatusReady}, processor.clusterMigrationTaskProcessor.Process}, - } - - return &processor, nil -} - -func (p *DispatcherProcessor) Name() string { - return "migration_dispatcher" -} - -var ( - repeatingTaskTypes = []string{"cluster_comparison", "cluster_migration"} -) - -func (p *DispatcherProcessor) getTasks() error { - return nil -} - -func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error { - var handledTaskNum int - // handle repeating tasks - for _, taskType := range repeatingTaskTypes { - handledTaskNum += p.handleRepeatingTasks(ctx, taskType) - - } - for _, tsk := range p.queryTasks { - handledTaskNum += p.handleTasks(ctx, tsk.Type, tsk.Status, tsk.TaskHandler) - } - if handledTaskNum == 0 { - ctx.Finished() - } - return nil -} - -func (p *DispatcherProcessor) handleTasks(ctx *pipeline.Context, taskType string, taskStatus []string, taskHandler func(taskItem *task.Task) error) int { - tasks, err := p.getMigrationTasks(taskType, taskStatus, p.config.TaskBatchSize) - if err != nil { - log.Errorf("failed to get [%s] with status %s, err: %v", taskType, taskStatus, err) - return 0 - } - if len(tasks) == 0 { - return 0 - } - log.Debugf("handling [%s] with status [%s], count: %d", taskType, taskStatus, len(tasks)) - // refresh index after each batch - defer func() { - p.refreshTask() - }() - for i := range tasks { - if ctx.IsCanceled() { - return 0 - } - taskItem := &tasks[i] - err := p.handleTask(taskItem, taskHandler) - if err != nil { - log.Errorf("failed to handle task [%s]: [%v]", taskItem.ID, err) - - taskItem.Status = task.StatusError - tn := time.Now() - taskItem.CompletedTime = &tn - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: err.Error(), - }, fmt.Sprintf("failed to handle task [%s]", taskItem.ID)) - } - } - return len(tasks) -} - -func (p *DispatcherProcessor) handleTask(taskItem *task.Task, taskHandler func(taskItem *task.Task) error) error { - if taskItem.Metadata.Labels == nil { - log.Errorf("got migration task [%s] with empty labels, skip handling", taskItem.ID) - return errors.New("missing labels") - } - return taskHandler(taskItem) -} - -func (p *DispatcherProcessor) getMigrationTasks(taskType string, taskStatus []string, size int) ([]task.Task, error) { - queryDsl := util.MapStr{ - "size": size, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "asc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "terms": util.MapStr{ - "status": taskStatus, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": taskType, - }, - }, - }, - }, - }, - }, - } - return migration_util.GetTasks(queryDsl) -} - -func (p *DispatcherProcessor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.config.Elasticsearch) - _, err := esClient.Index(p.config.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -func (p *DispatcherProcessor) refreshTask() { - esClient := elastic.GetClient(p.config.Elasticsearch) - err := esClient.Refresh(p.config.IndexName) - if err != nil { - log.Errorf("failed to refresh state, err: %v", err) - } -} diff --git a/plugin/task_manager/pipeline_task/bulk_indexing.go b/plugin/task_manager/pipeline_task/bulk_indexing.go deleted file mode 100644 index 2f8cee83..00000000 --- a/plugin/task_manager/pipeline_task/bulk_indexing.go +++ /dev/null @@ -1,99 +0,0 @@ -package pipeline_task - -import ( - "fmt" - "strings" - "time" - - log "github.com/cihub/seelog" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/task" -) - -func (p *processor) handleRunningBulkIndexingPipelineTask(taskItem *task.Task) error { - successDocs, indexDocs, bulked, totalInvalidDocs, totalInvalidReasons, totalFailureDocs, totalFailureReasons, errs := p.getBulkIndexingTaskState(taskItem) - if !bulked { - return nil - } - - var errMsg string - if len(errs) > 0 { - errMsg = fmt.Sprintf("bulk finished with error(s): %v", errs) - } - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Metadata.Labels["index_docs"] = indexDocs - taskItem.Metadata.Labels["success_docs"] = successDocs - taskItem.Metadata.Labels["invalid_docs"] = strings.Join(totalInvalidDocs, ",") - taskItem.Metadata.Labels["invalid_reasons"] = strings.Join(totalInvalidReasons, ",") - taskItem.Metadata.Labels["failure_docs"] = strings.Join(totalFailureDocs, ",") - taskItem.Metadata.Labels["failure_reasons"] = strings.Join(totalFailureReasons, ",") - if errMsg != "" { - taskItem.Status = task.StatusError - } else { - taskItem.Status = task.StatusComplete - } - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("[bulk_indexing] pipeline task [%s] completed", taskItem.ID)) - p.cleanGatewayPipeline(taskItem) - return nil -} - -func (p *processor) getBulkIndexingTaskState(taskItem *task.Task) (successDocs int64, indexDocs int64, bulked bool, totalInvalidDocs []string, totalInvalidReasons []string, totalFailureDocs []string, totalFailureReasons []string, errs []string) { - newHits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, taskItem.Updated.UnixMilli()) - if err != nil { - log.Errorf("failed to get latest pipeline logs for task [%s], err: %v", taskItem.ID, err) - return - } - if len(newHits) == 0 { - log.Debugf("bulk task [%s] not finished yet since last start", taskItem.ID) - return - } - - hits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, 0) - if err != nil { - log.Errorf("failed to get all pipeline logs for task [%s], err: %v", taskItem.ID, err) - return - } - - for _, m := range hits { - bulked = true - - errStr := migration_util.GetMapStringValue(m, "payload.pipeline.logging.result.error") - if errStr != "" { - errs = append(errs, errStr) - } - - var ( - success = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.bulk_indexing.success.count") - failure = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.bulk_indexing.failure.count") - invalid = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.bulk_indexing.invalid.count") - ) - successDocs += success - indexDocs += success + invalid + failure - - var ( - invalidDocs = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.bulk_indexing.detail.invalid.documents") - invalidReasons = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.bulk_indexing.detail.invalid.reasons") - failureDocs = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.bulk_indexing.detail.failure.documents") - failureReasons = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.bulk_indexing.detail.failure.reasons") - ) - totalInvalidDocs = append(totalInvalidDocs, invalidDocs...) - totalInvalidReasons = append(invalidReasons, invalidReasons...) - totalFailureDocs = append(totalFailureDocs, failureDocs...) - totalFailureReasons = append(totalFailureReasons, failureReasons...) - } - return -} - -func (p *processor) clearBulkIndexLabels(labels map[string]interface{}) { - delete(labels, "index_docs") - delete(labels, "success_docs") - delete(labels, "invalid_docs") - delete(labels, "invalid_reasons") - delete(labels, "failure_docs") - delete(labels, "failure_reasons") -} diff --git a/plugin/task_manager/pipeline_task/dump_hash.go b/plugin/task_manager/pipeline_task/dump_hash.go deleted file mode 100644 index 09e49024..00000000 --- a/plugin/task_manager/pipeline_task/dump_hash.go +++ /dev/null @@ -1,80 +0,0 @@ -package pipeline_task - -import ( - "errors" - "fmt" - "time" - - log "github.com/cihub/seelog" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/task" -) - -func (p *processor) handleRunningDumpHashPipelineTask(taskItem *task.Task) error { - scrolledDocs, totalHits, scrolled, err := p.getDumpHashTaskState(taskItem) - - if !scrolled { - return nil - } - - var errMsg string - if err != nil { - errMsg = err.Error() - } - if errMsg == "" { - if scrolledDocs < totalHits { - errMsg = fmt.Sprintf("scrolled finished but docs count unmatch: %d / %d", scrolledDocs, totalHits) - } - } - - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Metadata.Labels["scrolled_docs"] = scrolledDocs - if errMsg != "" { - taskItem.Status = task.StatusError - } else { - taskItem.Status = task.StatusComplete - } - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("[dump_hash] pipeline task [%s] completed", taskItem.ID)) - p.cleanGatewayPipeline(taskItem) - return nil -} - -func (p *processor) getDumpHashTaskState(taskItem *task.Task) (scrolledDocs int64, totalHits int64, scrolled bool, err error) { - hits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, taskItem.Updated.UnixMilli()) - if err != nil { - log.Errorf("failed to get pipeline logs for task [%s], err: %v", taskItem.ID, err) - err = nil - return - } - if len(hits) == 0 { - log.Debugf("scroll task [%s] not finished yet since last start", taskItem.ID) - return - } - for _, m := range hits { - scrolled = true - - errStr := migration_util.GetMapStringValue(m, "payload.pipeline.logging.result.error") - if errStr != "" { - err = errors.New(errStr) - return - } - - var ( - scroll = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.dump_hash.scrolled_docs") - total = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.dump_hash.total_hits") - ) - - scrolledDocs += scroll - totalHits += total - } - return -} - -func (p *processor) clearDumpHashLabels(labels map[string]interface{}) { - delete(labels, "scrolled_docs") -} diff --git a/plugin/task_manager/pipeline_task/es_scroll.go b/plugin/task_manager/pipeline_task/es_scroll.go deleted file mode 100644 index be682cfb..00000000 --- a/plugin/task_manager/pipeline_task/es_scroll.go +++ /dev/null @@ -1,81 +0,0 @@ -package pipeline_task - -import ( - "errors" - "fmt" - "time" - - log "github.com/cihub/seelog" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/task" -) - -func (p *processor) handleRunningEsScrollPipelineTask(taskItem *task.Task) error { - scrolledDocs, totalHits, scrolled, err := p.getEsScrollTaskState(taskItem) - - if !scrolled { - return nil - } - - var errMsg string - if err != nil { - errMsg = err.Error() - } - if errMsg == "" { - if scrolledDocs < totalHits { - errMsg = fmt.Sprintf("scrolled finished but docs count unmatch: %d / %d", scrolledDocs, totalHits) - } - } - - now := time.Now() - taskItem.CompletedTime = &now - taskItem.Metadata.Labels["scrolled_docs"] = scrolledDocs - if errMsg != "" { - taskItem.Status = task.StatusError - } else { - taskItem.Status = task.StatusComplete - } - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("[es_scroll] pipeline task [%s] completed", taskItem.ID)) - p.cleanGatewayPipeline(taskItem) - return nil -} - -func (p *processor) getEsScrollTaskState(taskItem *task.Task) (scrolledDocs int64, totalHits int64, scrolled bool, err error) { - hits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, taskItem.Updated.UnixMilli()) - if err != nil { - log.Errorf("failed to get pipeline logs for task [%s], err: %v", taskItem.ID, err) - err = nil - return - } - if len(hits) == 0 { - log.Debugf("scroll task [%s] not finished yet since last start", taskItem.ID) - return - } - // NOTE: we only check the last run of es_scroll - for _, m := range hits { - scrolled = true - - errStr := migration_util.GetMapStringValue(m, "payload.pipeline.logging.result.error") - if errStr != "" { - err = errors.New(errStr) - return - } - - var ( - scroll = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.es_scroll.scrolled_docs") - total = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.es_scroll.total_hits") - ) - - scrolledDocs += scroll - totalHits += total - } - return -} - -func (p *processor) clearEsScrollLabels(labels map[string]interface{}) { - delete(labels, "scrolled_docs") -} diff --git a/plugin/task_manager/pipeline_task/index_diff.go b/plugin/task_manager/pipeline_task/index_diff.go deleted file mode 100644 index 9ebdab6e..00000000 --- a/plugin/task_manager/pipeline_task/index_diff.go +++ /dev/null @@ -1,102 +0,0 @@ -package pipeline_task - -import ( - "fmt" - "strings" - "time" - - log "github.com/cihub/seelog" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/task" -) - -func (p *processor) handleRunningIndexDiffPipelineTask(taskItem *task.Task) error { - - diffed, onlyInSourceCount, totalOnlyInSourceDocs, onlyInTargetCount, totalOnlyInTargetDocs, diffBothCount, totalDiffBothDocs, errs := p.getIndexDiffTaskState(taskItem) - if !diffed { - return nil - } - - var errMsg string - if len(errs) > 0 { - errMsg = fmt.Sprintf("index diff finished with error(s): %v", errs) - } - - now := time.Now() - taskItem.CompletedTime = &now - // NOTE: only_in_source/only_in_target is likely useless because we'll skip diff if doc count unmatch - taskItem.Metadata.Labels["only_in_source_count"] = onlyInSourceCount - taskItem.Metadata.Labels["only_in_source_keys"] = strings.Join(totalOnlyInSourceDocs, ",") - taskItem.Metadata.Labels["only_in_target_count"] = onlyInTargetCount - taskItem.Metadata.Labels["only_in_target_keys"] = strings.Join(totalOnlyInTargetDocs, ",") - taskItem.Metadata.Labels["diff_both_count"] = diffBothCount - taskItem.Metadata.Labels["diff_both_keys"] = strings.Join(totalDiffBothDocs, ",") - - if errMsg != "" { - taskItem.Status = task.StatusError - } else { - taskItem.Status = task.StatusComplete - } - - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: errMsg == "", - Error: errMsg, - }, fmt.Sprintf("[index_diff] pipeline task [%s] finished with status [%s]", taskItem.ID, taskItem.Status)) - p.cleanGatewayPipeline(taskItem) - return nil -} - -func (p *processor) getIndexDiffTaskState(taskItem *task.Task) (diffed bool, onlyInSourceCount int64, totalOnlyInSourceDocs []string, onlyInTargetCount int64, totalOnlyInTargetDocs []string, diffBothCount int64, totalDiffBothDocs []string, errs []string) { - newHits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, taskItem.Updated.UnixMilli()) - if err != nil { - log.Errorf("failed to get latest pipeline logs for task [%s], err: %v", taskItem.ID, err) - return - } - if len(newHits) == 0 { - log.Debugf("bulk task [%s] not finished yet since last start", taskItem.ID) - return - } - - hits, err := p.getPipelineLogs(taskItem, []string{"FINISHED", "FAILED"}, 0) - if err != nil { - log.Errorf("failed to get all pipeline logs for task [%s], err: %v", taskItem.ID, err) - return - } - - for _, m := range hits { - diffed = true - - errStr := migration_util.GetMapStringValue(m, "payload.pipeline.logging.result.error") - if errStr != "" { - errs = append(errs, errStr) - } - - var ( - onlyInSource = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.index_diff.only_in_source.count") - onlyInTarget = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.index_diff.only_in_target.count") - diffBoth = migration_util.GetMapIntValue(m, "payload.pipeline.logging.context.index_diff.diff_both.count") - ) - onlyInSourceCount += onlyInSource - onlyInTargetCount += onlyInTarget - diffBothCount += diffBoth - - var ( - onlyInSourceDocs = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.index_diff.only_in_source.keys") - onlyInTargetDocs = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.index_diff.only_in_target.keys") - diffBothDocs = migration_util.GetMapStringSliceValue(m, "payload.pipeline.logging.context.index_diff.diff_both.keys") - ) - totalOnlyInSourceDocs = append(totalOnlyInSourceDocs, onlyInSourceDocs...) - totalOnlyInTargetDocs = append(totalOnlyInTargetDocs, onlyInTargetDocs...) - totalDiffBothDocs = append(totalDiffBothDocs, diffBothDocs...) - } - return -} - -func (p *processor) clearIndexDiffLabels(labels map[string]interface{}) { - delete(labels, "only_in_source_count") - delete(labels, "only_in_source_keys") - delete(labels, "only_in_target_count") - delete(labels, "only_in_target_keys") - delete(labels, "diff_both_count") - delete(labels, "diff_both_keys") -} diff --git a/plugin/task_manager/pipeline_task/pipeline_task.go b/plugin/task_manager/pipeline_task/pipeline_task.go deleted file mode 100644 index 6b41326e..00000000 --- a/plugin/task_manager/pipeline_task/pipeline_task.go +++ /dev/null @@ -1,306 +0,0 @@ -package pipeline_task - -import ( - "errors" - "fmt" - "infini.sh/console/model" - "strconv" - "strings" - "time" - - log "github.com/cihub/seelog" - - migration_model "infini.sh/console/plugin/task_manager/model" - migration_util "infini.sh/console/plugin/task_manager/util" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -type processor struct { - Elasticsearch string - IndexName string - LogIndexName string - - scheduler migration_model.Scheduler -} - -func NewProcessor(elasticsearch, indexName, logIndexName string, scheduler migration_model.Scheduler) migration_model.Processor { - return &processor{ - Elasticsearch: elasticsearch, - IndexName: indexName, - LogIndexName: logIndexName, - scheduler: scheduler, - } -} - -func (p *processor) Process(t *task.Task) (err error) { - switch t.Status { - case task.StatusReady: - // schedule pipeline task & create pipeline - err = p.handleReadyPipelineTask(t) - case task.StatusRunning: - // check pipeline log - err = p.handleRunningPipelineTask(t) - case task.StatusPendingStop: - // stop pipeline - err = p.handlePendingStopPipelineTask(t) - } - return err -} - -func (p *processor) handleReadyPipelineTask(taskItem *task.Task) error { - switch taskItem.Metadata.Labels["pipeline_id"] { - case "es_scroll": - case "bulk_indexing": - case "dump_hash": - case "index_diff": - default: - return fmt.Errorf("task [%s] has unknown pipeline_id [%s]", taskItem.ID, taskItem.Metadata.Labels["pipeline_id"]) - } - - instance, err := p.cleanGatewayPipeline(taskItem) - if err != nil { - log.Errorf("failed to prepare instance before running pipeline, err: %v", err) - return nil - } - - cfg := migration_model.PipelineTaskConfig{} - err = migration_util.GetTaskConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to get task config, err: %v", err) - return err - } - - err = p.fillDynamicESConfig(taskItem, &cfg) - if err != nil { - log.Errorf("failed to update task config, err: %v", err) - return err - } - - cfg.Labels["retry_times"] = taskItem.RetryTimes - - // call instance api to create pipeline task - err = instance.CreatePipeline(util.MustToJSONBytes(cfg)) - if err != nil { - log.Errorf("create pipeline task [%s] failed, err: %+v", taskItem.ID, err) - return err - } - - switch taskItem.Metadata.Labels["pipeline_id"] { - case "es_scroll": - p.clearEsScrollLabels(taskItem.Metadata.Labels) - case "bulk_indexing": - p.clearBulkIndexLabels(taskItem.Metadata.Labels) - case "dump_hash": - p.clearDumpHashLabels(taskItem.Metadata.Labels) - case "index_diff": - p.clearIndexDiffLabels(taskItem.Metadata.Labels) - } - - taskItem.Status = task.StatusRunning - taskItem.StartTimeInMillis = time.Now().UnixMilli() - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("[%v] pipeline task [%s] started", taskItem.Metadata.Labels["pipeline_id"], taskItem.ID)) - - return nil -} - -func (p *processor) handleRunningPipelineTask(taskItem *task.Task) error { - switch taskItem.Metadata.Labels["pipeline_id"] { - case "es_scroll": - return p.handleRunningEsScrollPipelineTask(taskItem) - case "bulk_indexing": - return p.handleRunningBulkIndexingPipelineTask(taskItem) - case "dump_hash": - return p.handleRunningDumpHashPipelineTask(taskItem) - case "index_diff": - return p.handleRunningIndexDiffPipelineTask(taskItem) - default: - return fmt.Errorf("task [%s] has unknown pipeline_id [%s]", taskItem.ID, taskItem.Metadata.Labels["pipeline_id"]) - } - return nil -} - -func (p *processor) handlePendingStopPipelineTask(taskItem *task.Task) error { - switch taskItem.Metadata.Labels["pipeline_id"] { - case "es_scroll": - case "bulk_indexing": - case "dump_hash": - case "index_diff": - default: - return fmt.Errorf("task [%s] has unknown pipeline_id [%s]", taskItem.ID, taskItem.Metadata.Labels["pipeline_id"]) - } - - // we only check STOPPED log after the last task status update - hits, err := p.getPipelineLogs(taskItem, []string{"STOPPED"}, taskItem.Updated.UnixMilli()) - if err != nil { - log.Errorf("failed to get pipeline logs for task [%s], err: %v", taskItem.ID, err) - return nil - } - - stopped := len(hits) > 0 - - if stopped { - taskItem.Status = task.StatusStopped - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("[%v] task [%s] stopped", taskItem.Metadata.Labels["pipeline_id"], taskItem.ID)) - p.cleanGatewayPipeline(taskItem) - return nil - } - - instance, err := p.getPipelineExecutionInstance(taskItem) - if err != nil { - log.Errorf("failed to get execution instance for task [%s], err: %v", taskItem.ID, err) - return nil - } - - err = instance.StopPipelineWithTimeout(taskItem.ID, time.Second) - if err != nil { - if strings.Contains(err.Error(), "task not found") { - taskItem.Status = task.StatusStopped - p.saveTaskAndWriteLog(taskItem, nil, fmt.Sprintf("[%v] task [%s] not found on remote node, mark as stopped", taskItem.Metadata.Labels["pipeline_id"], taskItem.ID)) - p.cleanGatewayPipeline(taskItem) - return nil - } - log.Errorf("failed to stop pipeline, err: %v", err) - } - return nil -} - -func (p *processor) cleanGatewayPipeline(taskItem *task.Task) (instance *model.TaskWorker, err error) { - instance, err = p.getPipelineExecutionInstance(taskItem) - if err != nil { - log.Errorf("failed to get execution instance for task [%s], err: %v", taskItem.ID, err) - return - } - err = instance.DeletePipeline(taskItem.ID) - if err != nil && !strings.Contains(err.Error(), "task not found") { - log.Errorf("delete pipeline failed, err: %v", err) - return - } - - return instance, nil -} - -func (p *processor) getPipelineExecutionInstance(taskItem *task.Task) (*model.TaskWorker, error) { - instanceID, _ := util.ExtractString(taskItem.Metadata.Labels["execution_instance_id"]) - instance, err := p.scheduler.GetInstance(instanceID) - if err != nil { - return nil, err - } - return instance, nil -} - -func (p *processor) getPipelineLogs(taskItem *task.Task, status []string, timestampGte int64) ([]util.MapStr, error) { - query := util.MapStr{ - "size": 999, - "sort": []util.MapStr{ - { - "timestamp": util.MapStr{ - "order": "desc", - }, - }, - { - "payload.pipeline.logging.steps": util.MapStr{ - "order": "desc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.labels.task_id": taskItem.ID, - }, - }, - { - "terms": util.MapStr{ - "payload.pipeline.logging.status": status, - }, - }, - { - "range": util.MapStr{ - "metadata.labels.retry_times": util.MapStr{ - "gte": taskItem.RetryTimes, - }, - }, - }, - { - "range": util.MapStr{ - "timestamp": util.MapStr{ - "gte": timestampGte, - }, - }, - }, - }, - }, - }, - } - esClient := elastic.GetClient(p.Elasticsearch) - res, err := esClient.SearchWithRawQueryDSL(p.LogIndexName, util.MustToJSONBytes(query)) - if err != nil { - log.Errorf("search task log from es failed, err: %v", err) - return nil, err - } - var ret []util.MapStr - dups := map[string]struct{}{} - for _, hit := range res.Hits.Hits { - m := util.MapStr(hit.Source) - ctxID := migration_util.GetMapStringValue(m, "metadata.labels.context_id") - step := migration_util.GetMapIntValue(m, "payload.pipeline.logging.steps") - // NOTE: gateway <= 1.13.0 will not generate context_id, skip duplicate checks - if ctxID != "" { - dupKey := ctxID + "-" + strconv.Itoa(int(step)) - if _, ok := dups[dupKey]; ok { - continue - } - dups[dupKey] = struct{}{} - } - ret = append(ret, m) - } - return ret, nil -} - -func (p *processor) saveTaskAndWriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - esClient := elastic.GetClient(p.Elasticsearch) - _, err := esClient.Index(p.IndexName, "", taskItem.ID, taskItem, "") - if err != nil { - log.Errorf("failed to update task, err: %v", err) - } - if message != "" { - migration_util.WriteLog(taskItem, taskResult, message) - } -} - -// TODO: remove after implementing dynamic register elasticsearch configs at gateway -func (p *processor) fillDynamicESConfig(taskItem *task.Task, pipelineTaskConfig *migration_model.PipelineTaskConfig) error { - for _, p := range pipelineTaskConfig.Processor { - for k, v := range p { - v, ok := v.(map[string]interface{}) - if !ok { - return errors.New("invalid processor config") - } - processorConfig := util.MapStr(v) - if k == "bulk_indexing" || k == "es_scroll" || k == "dump_hash" { - elasticsearchID := migration_util.GetMapStringValue(processorConfig, "elasticsearch") - if elasticsearchID == "" { - return fmt.Errorf("invalid task config found for task [%s]", taskItem.ID) - } - esConfig := elastic.GetConfigNoPanic(elasticsearchID) - if esConfig == nil { - return fmt.Errorf("can't load elasticsearch config of [%s] for task task [%s]", elasticsearchID, taskItem.ID) - } - processorConfig["elasticsearch_config"] = util.MapStr{ - "name": elasticsearchID, - "enabled": true, - "endpoint": esConfig.Endpoint, - "basic_auth": esConfig.BasicAuth, - } - } - } - } - return nil -} diff --git a/plugin/task_manager/repeat.go b/plugin/task_manager/repeat.go deleted file mode 100644 index 66931cb3..00000000 --- a/plugin/task_manager/repeat.go +++ /dev/null @@ -1,130 +0,0 @@ -package task_manager - -import ( - "errors" - "fmt" - "time" - - log "github.com/cihub/seelog" - "infini.sh/console/plugin/task_manager/cluster_comparison" - "infini.sh/console/plugin/task_manager/cluster_migration" - migration_util "infini.sh/console/plugin/task_manager/util" - "infini.sh/framework/core/pipeline" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func (p *DispatcherProcessor) handleRepeatingTasks(ctx *pipeline.Context, taskType string) int { - tasks, err := p.getPendingExecutionTasks(taskType, p.config.TaskBatchSize) - if err != nil { - log.Errorf("failed to get pending [%s] tasks, err: %v", taskType, err) - return 0 - } - if len(tasks) == 0 { - return 0 - } - log.Debugf("handling pending [%s] tasks, count: %d", taskType, len(tasks)) - // refresh index after each batch - defer func() { - p.refreshTask() - }() - for i := range tasks { - if ctx.IsCanceled() { - return 0 - } - taskItem := &tasks[i] - err := p.handleTask(taskItem, p.handleRepeatingTask) - if err != nil { - log.Errorf("failed to handle task [%s]: [%v]", taskItem.ID, err) - - taskItem.Status = task.StatusError - tn := time.Now() - taskItem.CompletedTime = &tn - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: false, - Error: err.Error(), - }, fmt.Sprintf("failed to handle task [%s]", taskItem.ID)) - } - } - return len(tasks) -} - -func (p *DispatcherProcessor) getPendingExecutionTasks(taskType string, size int) ([]task.Task, error) { - queryDsl := util.MapStr{ - "size": size, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "asc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.type": taskType, - }, - }, - { - "term": util.MapStr{ - "metadata.labels.repeat_triggered": util.MapStr{ - "value": false, - }, - }, - }, - { - "range": util.MapStr{ - "metadata.labels.next_run_time": util.MapStr{ - "lte": time.Now().UnixMilli(), - }, - }, - }, - }, - }, - }, - } - return migration_util.GetTasks(queryDsl) -} - -// NOTE: we handle repeating in two iterations: -// - the first iteration will mark the task as ready to run -// - the second iteration will trigger the next repeat, and update the status accordingly -// This will make the second step automatically retryable -func (p *DispatcherProcessor) handleRepeatingTask(taskItem *task.Task) error { - if taskItem.Status == task.StatusInit { - taskItem.Status = task.StatusReady - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("task started automatically")) - return nil - } - repeatDone := migration_util.GetMapBoolValue(taskItem.Metadata.Labels, "repeat_done") - if repeatDone { - taskItem.Metadata.Labels["repeat_triggered"] = true - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("task repeat ended")) - return nil - } - var nextTask *task.Task - var err error - switch taskItem.Metadata.Type { - case "cluster_migration": - nextTask, err = cluster_migration.RepeatTask(taskItem) - case "cluster_comparison": - nextTask, err = cluster_comparison.RepeatTask(taskItem) - default: - return errors.New("invalid type") - } - if err != nil { - log.Errorf("failed to repeat task [%s], err: %v", taskItem.ID, err) - return nil - } - taskItem.Metadata.Labels["repeat_triggered"] = true - p.saveTaskAndWriteLog(taskItem, &task.TaskResult{ - Success: true, - }, fmt.Sprintf("next repeat task [%s] created", nextTask.ID)) - return nil -} diff --git a/plugin/task_manager/scheduler/scheduler.go b/plugin/task_manager/scheduler/scheduler.go deleted file mode 100644 index 2f064031..00000000 --- a/plugin/task_manager/scheduler/scheduler.go +++ /dev/null @@ -1,335 +0,0 @@ -package scheduler - -import ( - "errors" - "fmt" - "math" - "strings" - "sync" - "time" - - log "github.com/cihub/seelog" - - "infini.sh/console/model" - migration_model "infini.sh/console/plugin/task_manager/model" - - "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/pipeline" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -const initializeInterval = 10 * time.Second - -type scheduler struct { - Elasticsearch string - IndexName string - CheckInstanceAvailable bool - MaxTasksPerInstance int - - state map[string]DispatcherState - stateLock sync.Mutex -} - -type DispatcherState struct { - Total int - LastInitializedAt time.Time -} - -// NOTE: currently we assume task are scheduled sequentially, so GetInstance/GetPreferenceInstance doesn't need to handle locking -func NewScheduler(elasticsearch, indexName string, checkInstanceAvailable bool, maxTasksPerInstance int) (migration_model.Scheduler, error) { - scheduler := &scheduler{ - Elasticsearch: elasticsearch, - IndexName: indexName, - CheckInstanceAvailable: checkInstanceAvailable, - MaxTasksPerInstance: maxTasksPerInstance, - state: map[string]DispatcherState{}, - } - err := scheduler.RefreshInstanceJobsFromES() - if err != nil { - return nil, err - } - return scheduler, nil -} - -func (p *scheduler) GetPreferenceInstance(config migration_model.ExecutionConfig) (*model.TaskWorker, error) { - var ( - err error - minID string - minTotal = math.MaxInt - ) - - for _, node := range config.Nodes.Permit { - instanceTotal := p.getInstanceState(node.ID).Total - if instanceTotal < minTotal { - if p.CheckInstanceAvailable { - tempInst := model.TaskWorker{} - tempInst.ID = node.ID - _, err = orm.Get(&tempInst.Instance) - if err != nil { - log.Errorf("failed to get instance, err: %v", err) - continue - } - err = tempInst.TryConnectWithTimeout(time.Second) - if err != nil { - log.Debugf("instance [%s] is not available, caused by: %v", tempInst.ID, err) - continue - } - } - minID = node.ID - minTotal = instanceTotal - } - } - if minID == "" { - return nil, fmt.Errorf("no available instance") - } - - instance, err := p.GetInstance(minID) - if err != nil { - return nil, err - } - if p.getInstanceState(minID).Total >= p.MaxTasksPerInstance { - return nil, migration_model.ErrHitMax - } - return instance, nil -} - -func (p *scheduler) GetInstance(instanceID string) (*model.TaskWorker, error) { - if instanceID == "" { - return nil, errors.New("invalid instanceID") - } - instance := model.TaskWorker{} - instance.ID = instanceID - - _, err := orm.Get(&instance.Instance) - if err != nil { - log.Errorf("failed to get instance [%s] from orm, err: %v", instance.ID, err) - return nil, err - } - err = p.initializeInstance(&instance) - if err != nil { - log.Warnf("failed to initialized instance [%s], err: %v", instance.ID, err) - } - return &instance, nil -} - -func (p *scheduler) initializeInstance(instance *model.TaskWorker) error { - lastInitializedAt := p.getLastInitializedAt(instance.ID) - if time.Now().Sub(lastInitializedAt) < initializeInterval { - return nil - } - - status, err := instance.GetPipeline("pipeline_logging_merge") - if err != nil { - if strings.Contains(err.Error(), "pipeline not found") { - log.Infof("pipeline_logging_merge not found on instance [%s], initializing", instance.ID) - err := p.createPipelineLoggingMerge(instance) - if err != nil { - return err - } - } - } else if status.State == pipeline.STOPPED { - log.Infof("pipeline_logging_merge stopped on instance [%s], starting", instance.ID) - err = instance.StartPipeline("pipeline_logging_merge") - if err != nil { - return err - } - } - - status, err = instance.GetPipeline("ingest_pipeline_logging") - if err != nil { - if strings.Contains(err.Error(), "pipeline not found") { - log.Infof("ingest_pipeline_logging not found on instance [%s], initializing", instance.ID) - err := p.createIngestPipelineLogging(instance) - if err != nil { - return err - } - } - } else if status.State == pipeline.STOPPED { - log.Infof("ingest_pipeline_logging stopped on instance [%s], starting", instance.ID) - err = instance.StartPipeline("ingest_pipeline_logging") - if err != nil { - return err - } - } - - p.setLastInitializedAt(instance.ID, time.Now()) - return nil -} - -// TODO: now we're using the same configuraiton as the default gateway.yml -// user could change the following configurations manually: -// - input_queue (metrics.logging_queue) -// - elasticsearch (elasticsearch.name) -func (p *scheduler) createPipelineLoggingMerge(instance *model.TaskWorker) error { - cfg := &migration_model.PipelineTaskConfig{ - Name: "pipeline_logging_merge", - AutoStart: true, - KeepRunning: true, - Processor: []util.MapStr{ - util.MapStr{ - "indexing_merge": util.MapStr{ - "input_queue": "logging", - "idle_timeout_in_seconds": 1, - "elasticsearch": "logging-server", - "index_name": ".infini_logs", - "output_queue": util.MapStr{ - "name": "gateway-pipeline-logs", - "label": util.MapStr{ - "tag": "pipeline_logging", - }, - }, - "worker_size": 1, - "bulk_size_in_kb": 1, - }, - }, - }, - } - err := instance.CreatePipeline(util.MustToJSONBytes(cfg)) - if err != nil { - log.Errorf("create pipeline_logging_merge [%s] failed, err: %+v", instance.ID, err) - return err - } - return nil -} - -func (p *scheduler) createIngestPipelineLogging(instance *model.TaskWorker) error { - cfg := &migration_model.PipelineTaskConfig{ - Name: "ingest_pipeline_logging", - AutoStart: true, - KeepRunning: true, - Processor: []util.MapStr{ - util.MapStr{ - "bulk_indexing": util.MapStr{ - "bulk": util.MapStr{ - "compress": true, - "batch_size_in_mb": 1, - "batch_size_in_docs": 1, - }, - "consumer": util.MapStr{ - "fetch_max_messages": 100, - }, - "queues": util.MapStr{ - "type": "indexing_merge", - "tag": "pipeline_logging", - }, - }, - }, - }, - } - err := instance.CreatePipeline(util.MustToJSONBytes(cfg)) - if err != nil { - log.Errorf("create ingest_pipeline_logging [%s] failed, err: %+v", instance.ID, err) - return err - } - return nil -} - -func (p *scheduler) RefreshInstanceJobsFromES() error { - log.Debug("refreshing instance state from ES") - p.stateLock.Lock() - defer p.stateLock.Unlock() - - state, err := p.getInstanceTaskState() - if err != nil { - log.Errorf("failed to get instance task state, err: %v", err) - return err - } - p.state = state - - return nil -} - -func (p *scheduler) DecrInstanceJobs(instanceID string) { - p.stateLock.Lock() - defer p.stateLock.Unlock() - if st, ok := p.state[instanceID]; ok { - st.Total -= 1 - p.state[instanceID] = st - } -} - -func (p *scheduler) IncrInstanceJobs(instanceID string) { - p.stateLock.Lock() - defer p.stateLock.Unlock() - instanceState := p.state[instanceID] - instanceState.Total = instanceState.Total + 1 - p.state[instanceID] = instanceState -} - -func (p *scheduler) getLastInitializedAt(instanceID string) time.Time { - p.stateLock.Lock() - defer p.stateLock.Unlock() - if st, ok := p.state[instanceID]; ok { - return st.LastInitializedAt - } - return time.Time{} -} - -func (p *scheduler) setLastInitializedAt(instanceID string, t time.Time) { - p.stateLock.Lock() - defer p.stateLock.Unlock() - if st, ok := p.state[instanceID]; ok { - st.LastInitializedAt = t - p.state[instanceID] = st - } -} - -func (p *scheduler) getInstanceState(instanceID string) DispatcherState { - p.stateLock.Lock() - defer p.stateLock.Unlock() - - return p.state[instanceID] -} - -func (p *scheduler) getInstanceTaskState() (map[string]DispatcherState, error) { - query := util.MapStr{ - "size": 0, - "aggs": util.MapStr{ - "grp": util.MapStr{ - "terms": util.MapStr{ - "field": "metadata.labels.execution_instance_id", - "size": 1000, - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": "index_migration", - }, - }, - }, - { - "term": util.MapStr{ - "status": util.MapStr{ - "value": task.StatusRunning, - }, - }, - }, - }, - }, - }, - } - esClient := elastic.GetClient(p.Elasticsearch) - res, err := esClient.SearchWithRawQueryDSL(p.IndexName, util.MustToJSONBytes(query)) - if err != nil { - log.Errorf("search es failed, err: %v", err) - return nil, err - } - state := map[string]DispatcherState{} - for _, bk := range res.Aggregations["grp"].Buckets { - if key, ok := bk["key"].(string); ok { - if v, ok := bk["doc_count"].(float64); ok { - state[key] = DispatcherState{ - Total: int(v), - } - } - } - } - return state, nil -} diff --git a/plugin/task_manager/util/orm.go b/plugin/task_manager/util/orm.go deleted file mode 100644 index faa65d00..00000000 --- a/plugin/task_manager/util/orm.go +++ /dev/null @@ -1,270 +0,0 @@ -package util - -import ( - "fmt" - - log "github.com/cihub/seelog" - - "infini.sh/framework/core/orm" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func DeleteChildTasks(taskID string, taskType string) error { - q := util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskID, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": taskType, - }, - }, - }, - }, - }, - } - err := orm.DeleteBy(&task.Task{}, util.MustToJSONBytes(q)) - if err != nil { - return err - } - return nil -} - -func GetLastRepeatingChildTask(taskID string, taskType string) (*task.Task, *task.Task, error) { - queryDsl := util.MapStr{ - "size": 2, - "sort": []util.MapStr{ - { - "metadata.labels.next_run_time": util.MapStr{ - "order": "desc", - }, - }, - }, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.type": taskType, - }, - }, - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskID, - }, - }, - }, - }, - }, - }, - } - tasks, err := GetTasks(queryDsl) - if err != nil { - return nil, nil, err - } - if len(tasks) == 0 { - return nil, nil, nil - } - var lastRunChildTask *task.Task - if tasks[0].StartTimeInMillis > 0 { - lastRunChildTask = &tasks[0] - }else{ - if len(tasks) == 2 { - lastRunChildTask = &tasks[1] - } - } - - return &tasks[0], lastRunChildTask, nil -} - -func GetPendingChildTasks(taskID string, taskType string) ([]task.Task, error) { - return GetChildTasks(taskID, taskType, []string{task.StatusRunning, task.StatusPendingStop, task.StatusReady}) -} - -func CountRunningChildren(taskID string, taskType string) (int64, error) { - return CountTasks(util.MapStr{ - "query": util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "metadata.type": taskType, - }, - }, - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskID, - }, - }, - }, - { - "terms": util.MapStr{ - "status": []string{task.StatusRunning, task.StatusPendingStop, task.StatusReady}, - }, - }, - }, - }, - }, - }) -} - -func GetChildTasks(taskID string, taskType string, status []string) ([]task.Task, error) { - musts := []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskID, - }, - }, - }, - { - "term": util.MapStr{ - "metadata.type": taskType, - }, - }, - } - if len(status) > 0 { - musts = append(musts, util.MapStr{ - "terms": util.MapStr{ - "status": status, - }, - }) - } - queryDsl := util.MapStr{ - "size": 999, - "query": util.MapStr{ - "bool": util.MapStr{ - "must": musts, - }, - }, - } - return GetTasks(queryDsl) -} - -func CountTasks(query util.MapStr) (int64, error) { - return orm.Count(task.Task{}, util.MustToJSONBytes(query)) -} - -func GetTasks(query util.MapStr) ([]task.Task, error) { - err, res := orm.Search(task.Task{}, &orm.Query{ - RawQuery: util.MustToJSONBytes(query), - }) - if err != nil { - log.Errorf("query tasks from es failed, err: %v", err) - return nil, err - } - if res.Total == 0 { - return nil, nil - } - var tasks []task.Task - for _, row := range res.Result { - buf, err := util.ToJSONBytes(row) - if err != nil { - log.Errorf("marshal task json failed, err: %v", err) - return nil, err - } - tk := task.Task{} - err = util.FromJSONBytes(buf, &tk) - if err != nil { - log.Errorf("unmarshal task json failed, err: %v", err) - return nil, err - } - if tk.Metadata.Labels == nil { - continue - } - tasks = append(tasks, tk) - } - return tasks, nil -} - -// update status of subtask to pending stop -func UpdatePendingChildTasksToPendingStop(taskItem *task.Task, taskType string) error { - query := util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskItem.ID, - }, - }, - }, - { - "terms": util.MapStr{ - "status": []string{task.StatusRunning, task.StatusReady}, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": taskType, - }, - }, - }, - }, - }, - } - queryDsl := util.MapStr{ - "query": query, - "script": util.MapStr{ - "source": fmt.Sprintf("ctx._source['status'] = '%s'", task.StatusPendingStop), - }, - } - - err := orm.UpdateBy(taskItem, util.MustToJSONBytes(queryDsl)) - if err != nil { - return err - } - return nil -} - -// update status of subtask to ready -func UpdateStoppedChildTasksToReady(taskItem *task.Task, taskType string) error { - query := util.MapStr{ - "bool": util.MapStr{ - "must": []util.MapStr{ - { - "term": util.MapStr{ - "parent_id": util.MapStr{ - "value": taskItem.ID, - }, - }, - }, - { - "terms": util.MapStr{ - "status": []string{task.StatusError, task.StatusStopped, task.StatusComplete}, - }, - }, - { - "term": util.MapStr{ - "metadata.type": util.MapStr{ - "value": taskType, - }, - }, - }, - }, - }, - } - queryDsl := util.MapStr{ - "query": query, - "script": util.MapStr{ - "source": fmt.Sprintf("ctx._source['status'] = '%s'", task.StatusReady), - }, - } - - err := orm.UpdateBy(taskItem, util.MustToJSONBytes(queryDsl)) - if err != nil { - return err - } - return nil -} diff --git a/plugin/task_manager/util/pipeline.go b/plugin/task_manager/util/pipeline.go deleted file mode 100644 index 8c42e83d..00000000 --- a/plugin/task_manager/util/pipeline.go +++ /dev/null @@ -1,38 +0,0 @@ -package util - -import ( - migration_model "infini.sh/console/plugin/task_manager/model" - "infini.sh/framework/core/task" -) - -/* -These functions could return nil tasks -*/ - -func SplitIndexMigrationTasks(ptasks []task.Task) (scrollTask *task.Task, bulkTask *task.Task) { - for i, ptask := range ptasks { - if ptask.Metadata.Labels["pipeline_id"] == "bulk_indexing" { - bulkTask = &ptasks[i] - } else if ptask.Metadata.Labels["pipeline_id"] == "es_scroll" { - scrollTask = &ptasks[i] - } - } - return -} - -func SplitIndexComparisonTasks(ptasks []task.Task, cfg *migration_model.IndexComparisonTaskConfig) (sourceDumpTask *task.Task, targetDumpTask *task.Task, diffTask *task.Task) { - for i, ptask := range ptasks { - if ptask.Metadata.Labels["pipeline_id"] == "dump_hash" { - // TODO: we can't handle when compare the same cluster & same index - // catch it earlier when creating the task - if ptask.Metadata.Labels["cluster_id"] == cfg.Source.ClusterId && ptask.Metadata.Labels["index_name"] == cfg.Source.Indices { - sourceDumpTask = &ptasks[i] - } else { - targetDumpTask = &ptasks[i] - } - } else if ptask.Metadata.Labels["pipeline_id"] == "index_diff" { - diffTask = &ptasks[i] - } - } - return -} diff --git a/plugin/task_manager/util/repeat.go b/plugin/task_manager/util/repeat.go deleted file mode 100644 index 48f52ac2..00000000 --- a/plugin/task_manager/util/repeat.go +++ /dev/null @@ -1,79 +0,0 @@ -package util - -import ( - "time" - - migration_model "infini.sh/console/plugin/task_manager/model" - "infini.sh/framework/core/util" -) - -/* -is_repeat: task will repeat for more than 1 time -run_times: the total number of runs of a repeating task -repeat_done: task has reached the last repeat -next_run_time: the time this task will get picked by scheduler to start -repeat_triggered: the task has been picked by scheduler and started -*/ -func UpdateRepeatState(repeat *migration_model.Repeat, labels util.MapStr) error { - if labels == nil { - return nil - } - if repeat == nil { - labels["repeat_done"] = true - return nil - } - - if repeat.Interval >= time.Minute { - labels["is_repeat"] = true - } else { - labels["repeat_done"] = true - } - - runTimes := GetMapIntValue(labels, "run_times") - runTimes += 1 - labels["run_times"] = runTimes - - if repeat.TotalRun >= 1 && runTimes >= repeat.TotalRun { - labels["repeat_done"] = true - } - if _, ok := labels["next_run_time"]; !ok { - if repeat.NextRunTime != nil { - labels["next_run_time"] = repeat.NextRunTime.UnixMilli() - } - } else { - nextRunTime := GetMapIntValue(labels, "next_run_time") - labels["next_run_time"] = nextRunTime + repeat.Interval.Milliseconds() - } - labels["repeat_triggered"] = false - return nil -} - -func CopyRepeatState(oldLabels, newLabels util.MapStr) { - newLabels["run_times"] = oldLabels["run_times"] - newLabels["next_run_time"] = oldLabels["next_run_time"] -} - -func IsRepeating(repeat *migration_model.Repeat, labels map[string]interface{}) bool { - if repeat == nil { - return false - } - if repeat.Interval < time.Minute { - return false - } - if repeat.TotalRun < 1 { - return true - } - if repeat.TotalRun == 1 { - return false - } - nextRunTime := GetMapIntValue(labels, "next_run_time") - // not started yet - if nextRunTime == 0 { - return false - } - endTime := time.UnixMilli(nextRunTime).Add(time.Duration(repeat.TotalRun-1) * repeat.Interval) - if time.Now().Before(endTime) { - return true - } - return false -} diff --git a/plugin/task_manager/util/util.go b/plugin/task_manager/util/util.go deleted file mode 100644 index a6c91944..00000000 --- a/plugin/task_manager/util/util.go +++ /dev/null @@ -1,134 +0,0 @@ -package util - -import ( - log "github.com/cihub/seelog" - - "infini.sh/framework/core/event" - "infini.sh/framework/core/task" - "infini.sh/framework/core/util" -) - -func WriteLog(taskItem *task.Task, taskResult *task.TaskResult, message string) { - labels := util.MapStr{} - labels.Update(util.MapStr(taskItem.Metadata.Labels)) - labels["task_type"] = taskItem.Metadata.Type - labels["task_id"] = taskItem.ID - labels["parent_task_id"] = taskItem.ParentId - labels["retry_times"] = taskItem.RetryTimes - event.SaveLog(&event.Event{ - Metadata: event.EventMetadata{ - Category: "task", - Name: "logging", - Datatype: "event", - Labels: labels, - }, - Fields: util.MapStr{ - "task": util.MapStr{ - "logging": util.MapStr{ - "config": taskItem.ConfigString, - "status": taskItem.Status, - "message": message, - "result": taskResult, - }, - }, - }, - }) -} - -var runningTaskStatus = []string{task.StatusRunning, task.StatusReady} - -func IsRunningState(status string) bool { - return util.StringInArray(runningTaskStatus, status) -} - -var pendingTaskStatus = []string{task.StatusRunning, task.StatusReady, task.StatusPendingStop} - -func IsPendingState(status string) bool { - return util.StringInArray(pendingTaskStatus, status) -} - -func GetDirectParentId(parentIDs []string) string { - if len(parentIDs) == 0 { - return "" - } - return parentIDs[len(parentIDs)-1] -} - -func GetTaskConfig(task *task.Task, config interface{}) error { - if task.Config_ == nil { - return util.FromJSONBytes([]byte(task.ConfigString), config) - } - buf, err := util.ToJSONBytes(task.Config_) - if err != nil { - return err - } - return util.FromJSONBytes(buf, config) -} - -func GetMapIntValue(m util.MapStr, key string) int64 { - v, err := m.GetValue(key) - if err != nil { - return 0 - } - vv, err := util.ExtractInt(v) - if err != nil { - log.Errorf("got %s but failed to extract, err: %v", key, err) - return 0 - } - return vv -} - -func GetMapBoolValue(m util.MapStr, key string) bool { - v, err := m.GetValue(key) - if err != nil { - return false - } - vv, err := util.ExtractBool(v) - if err != nil { - log.Errorf("got %s but failed to extract, err: %v", key, err) - return false - } - return vv -} - -func GetMapStringValue(m util.MapStr, key string) string { - v, err := m.GetValue(key) - if err != nil { - return "" - } - vv, err := util.ExtractString(v) - if err != nil { - log.Errorf("got %s but failed to extract, err: %v", key, err) - return "" - } - return vv -} - -func GetMapStringSliceValue(m util.MapStr, key string) []string { - v, err := m.GetValue(key) - if err != nil { - return nil - } - if v == nil { - return nil - } - vv, ok := v.([]string) - if !ok { - vv, ok := v.(map[string]interface{}) - if !ok { - log.Errorf("got %s but failed to extract, type: %T", key, v) - return nil - } - ret := make([]string, 0, len(vv)) - for _, item := range vv { - itemV, err := util.ExtractString(item) - if err != nil { - log.Errorf("got %s but failed to extract, err: %v", key, err) - return nil - } - ret = append(ret, itemV) - } - return ret - } - return vv -}