diff --git a/Makefile b/Makefile index 979e6e32..8c787597 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ INFINI_BASE_FOLDER := $(OLDGOPATH)/src/infini.sh/ FRAMEWORK_FOLDER := $(INFINI_BASE_FOLDER)framework/ FRAMEWORK_REPO := ssh://git@git.infini.ltd:64221/infini/framework.git FRAMEWORK_BRANCH := master -FRAMEWORK_VENDOR_FOLDER := $(CURDIR)/../vendor/ +FRAMEWORK_VENDOR_FOLDER := $(CURDIR)/vendor/ FRAMEWORK_VENDOR_REPO := ssh://git@git.infini.ltd:64221/infini/framework-vendor.git FRAMEWORK_VENDOR_BRANCH := master diff --git a/api/index_management/rebuild.go b/api/index_management/rebuild.go new file mode 100644 index 00000000..82ef0dd2 --- /dev/null +++ b/api/index_management/rebuild.go @@ -0,0 +1,95 @@ +package index_management + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" + "infini.sh/search-center/model" +) + +func (handler APIHandler) ReindexAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + reindexItem := &model.InfiniReindex{} + resResult := map[string]interface{}{ + "errno": "0", + "errmsg": "", + "payload": nil, + } + + err := handler.DecodeJSON(req, reindexItem) + if err != nil { + resResult["errno"] = "E20001" + resResult["errmsg"] = err.Error() + handler.WriteJSON(w, resResult, http.StatusOK) + return + } + + fmt.Println(reindexItem) + + taskID, err := reindex(handler.Config.Elasticsearch, reindexItem) + if err != nil { + resResult["errno"] = "E20002" + resResult["errmsg"] = err.Error() + handler.WriteJSON(w, resResult, http.StatusOK) + return + } + resResult["payload"] = taskID + handler.WriteJSON(w, resResult, http.StatusOK) +} + +func reindex(esName string, body *model.InfiniReindex) (string, error) { + client := elastic.GetClient(esName) + esConfig := elastic.GetConfig(esName) + url := fmt.Sprintf("%s/_reindex?wait_for_completion=false", esConfig.Endpoint) + source := map[string]interface{}{ + "index": body.Source.Index, + } + if body.Source.MaxDocs > 0 { + source["max_docs"] = body.Source.MaxDocs + } + if body.Source.Query != nil { + source["query"] = body.Source.Query + } + if body.Source.Sort != "" { + source["sort"] = body.Source.Sort + } + if body.Source.Source != "" { + source["_source"] = body.Source.Source + } + dest := map[string]string{ + "index": body.Dest.Index, + } + if body.Dest.Pipeline != "" { + dest["pipeline"] = body.Dest.Pipeline + } + esBody := map[string]interface{}{ + "source": source, + "dest": dest, + } + buf, _ := json.Marshal(esBody) + fmt.Println(string(buf)) + reindexRes, err := client.Request("POST", url, buf) + if err != nil { + return "", err + } + resBody := struct { + Task string `json:"task"` + }{} + err = json.Unmarshal(reindexRes.Body, &resBody) + if err != nil { + return "", err + } + body.ID = util.GetUUID() + body.TaskId = resBody.Task + body.CreatedAt = time.Now() + + _, err = client.Index("infinireindex", body.ID, body) + if err != nil { + return "", err + } + return body.ID, nil +} diff --git a/api/init.go b/api/init.go index 7709bcb8..581162cb 100644 --- a/api/init.go +++ b/api/init.go @@ -11,13 +11,16 @@ func Init(cfg *config.AppConfig) { handler := index_management.APIHandler{ Config: cfg, } + var pathPrefix = "/_search-center/" //ui.HandleUIMethod(api.POST, "/api/get_indices",index_management.API1) - ui.HandleUIMethod(api.GET, "/api/dict/_search", handler.GetDictListAction) - ui.HandleUIMethod(api.POST, "/api/dict/_create", handler.CreateDictItemAction) + ui.HandleUIMethod(api.GET, pathPrefix+"dict/_search", handler.GetDictListAction) + ui.HandleUIMethod(api.POST, pathPrefix+"dict/_create", handler.CreateDictItemAction) //ui.HandleUIMethod(api.GET, "/api/dict/:id",handler.GetDictItemAction) - ui.HandleUIMethod(api.DELETE, "/api/dict/:id", handler.DeleteDictItemAction) + ui.HandleUIMethod(api.DELETE, pathPrefix+"dict/:id", handler.DeleteDictItemAction) //ui.HandleUIMethod(api.DELETE, "/api/dict/", handler.DeleteDictItemAction2) - ui.HandleUIMethod(api.POST, "/api/dict/_update", handler.UpdateDictItemAction) - ui.HandleUIMethod(api.POST, "/api/doc/:index", handler.HandleDocumentAction) - ui.HandleUIMethod(api.GET, "/api/indices/_cat", handler.HandleGetIndicesAction) + ui.HandleUIMethod(api.POST, pathPrefix+"dict/_update", handler.UpdateDictItemAction) + ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index", handler.HandleDocumentAction) + ui.HandleUIMethod(api.GET, pathPrefix+"indices/_cat", handler.HandleGetIndicesAction) + + ui.HandleUIMethod(api.POST, pathPrefix+"rebuild/_create", handler.ReindexAction) } diff --git a/main.go b/main.go index ec2d2cb2..8437ca2a 100644 --- a/main.go +++ b/main.go @@ -70,6 +70,7 @@ func main() { }, func() { orm.RegisterSchema(model.Dict{}) + orm.RegisterSchema(model.InfiniReindex{}) }) } diff --git a/model/reindex.go b/model/reindex.go new file mode 100644 index 00000000..70d2b316 --- /dev/null +++ b/model/reindex.go @@ -0,0 +1,24 @@ +package model + +import "time" + +type InfiniReindex struct { + ID string `json:"id" elastic_meta:"_id"` + Name string `json:"name" elastic_mapping:"name:{type:text}"` + Desc string `json:"desc" elastic_mapping:"desc:{type:text}"` + TaskId string `json:"task_id" elastic_mapping:"task_id:{type:keyword}"` + Source struct { + Index string `json:"index"` + MaxDocs int `json:"max_docs"` + Query map[string]interface{} `json:"query"` + Sort string `json:"sort"` + Source string `json:"_source"` + } `json:"source" elastic_mapping:"source:{type:object}"` + Dest struct { + Index string `json:"index"` + Pipeline string `json:"pipeline"` + } `json:"dest" elastic_mapping:"dest:{type:object}"` + + CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"` + Status string `json:"status" elastic_mapping:"status:{type:keyword}"` +} diff --git a/ui.go b/ui.go index 146e2427..8452e87f 100644 --- a/ui.go +++ b/ui.go @@ -2,9 +2,10 @@ package main import ( "fmt" - public "infini.sh/search-center/.public" "net/http" + public "infini.sh/search-center/.public" + log "github.com/cihub/seelog" "infini.sh/framework/core/api" "infini.sh/framework/core/ui" @@ -21,7 +22,7 @@ type UI struct { func (h UI) InitUI() { - vfs.RegisterFS(public.StaticFS{StaticFolder: h.Config.UILocalPath, TrimLeftPath: h.Config.UILocalPath , CheckLocalFirst: h.Config.UILocalEnabled, SkipVFS: !h.Config.UIVFSEnabled}) + vfs.RegisterFS(public.StaticFS{StaticFolder: h.Config.UILocalPath, TrimLeftPath: h.Config.UILocalPath, CheckLocalFirst: h.Config.UILocalEnabled, SkipVFS: !h.Config.UIVFSEnabled}) ui.HandleUI("/", vfs.FileServer(vfs.VFS())) diff --git a/web/config/config.js b/web/config/config.js index b8fb6de6..726d0f15 100644 --- a/web/config/config.js +++ b/web/config/config.js @@ -68,7 +68,7 @@ export default { // }, // }, proxy: { - '/api/': { + '/_search-center/': { target: 'http://localhost:2900', changeOrigin: true, // pathRewrite: { '^/server': '' }, diff --git a/web/mock/datamanagement/data/doc.js b/web/mock/datamanagement/data/doc.js new file mode 100644 index 00000000..712979d5 --- /dev/null +++ b/web/mock/datamanagement/data/doc.js @@ -0,0 +1,160 @@ +export const queryData = { + "errmsg": "", + "errno": "0", + "payload": { + "data": [ + { + "_index": "infini-test", + "address": "hunan changsha", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "liugq@qq.com", + "hobbies": "[\"basketball\",\"pingpan\"]", + "id": "jc6_jXYBKoaaPbVfj_8W", + "name": "liugq国家" + }, + { + "_index": "infini-test", + "address": "hunan changsha", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": "[\"test5\"]", + "id": "bvhm18dath2d6oa9046g", + "name": "hello4" + }, + { + "_index": "infini-test", + "address": "hunan changsha", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": "[\"test2\"]", + "id": "bvhlv6dath2d6oa9045g", + "name": "test 词典" + }, + { + "_index": "infini-test", + "address": "hunan changsha", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": "[\"test1\"]", + "id": "bvhltpdath2d6oa90450", + "name": "liugqy" + }, + { + "_index": "infini-test", + "address": "hunan zhuzhou", + "created_at": "2020-12-23T03:59:57.620Z", + "email": "cincky@qq.com", + "hobbies": "[\"basketball\",\"badminton1\"]", + "id": "js7EjXYBKoaaPbVfvf-c", + "name": "cincky" + }, + { + "_index": "infini-test", + "address": "湖北武汉2", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": [ + "test3" + ], + "id": "bvi5ellath2e0ukbq5e0", + "name": "武汉test" + }, + { + "_index": "infini-test", + "address": "hunan changsha", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": [ + "test3" + ], + "id": "bvia41lath2eneoeeij0", + "name": "铁路测试词典" + }, + { + "_index": "infini-test", + "address": "beijing", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": [ + "basketball1", + "badminton" + ], + "id": "bvia4ctath2eneoeeijg", + "name": "北京" + }, + { + "_index": "infini-test", + "address": "湖北武汉", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": [ + "test4" + ], + "id": "bvi5omtath2e0ukbq5eg", + "name": "武汉2" + }, + { + "_index": "infini-test", + "address": "hunan changsha1", + "created_at": "2020-12-23T03:57:57.620Z", + "email": "786027438@qq.com", + "hobbies": [ + "test3" + ], + "id": "bvhm0d5ath2d6oa90460", + "name": "hello2" + } + ], + "mappings": { + "infini-test": { + "mappings": { + "dynamic_templates": [ + { + "strings": { + "mapping": { + "ignore_above": 256, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "address": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "age": { + "type": "integer" + }, + "created_at": { + "type": "date" + }, + "email": { + "type": "keyword" + }, + "hobbies": { + "type": "text" + }, + "id": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "type": "text" + } + } + } + } + }, + "total": { + "relation": "eq", + "value": 11 + } + } +} \ No newline at end of file diff --git a/web/mock/datamanagement/document.js b/web/mock/datamanagement/document.js new file mode 100644 index 00000000..ec8b2ab6 --- /dev/null +++ b/web/mock/datamanagement/document.js @@ -0,0 +1,48 @@ +import {queryData} from './data/doc'; + +function getUUID(len){ + len = len || 20; + let chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; + var maxPos = chars.length; +　var uuid = ''; +　for (let i = 0; i < len; i++) { + uuid += chars.charAt(Math.floor(Math.random() * maxPos)); + } + return uuid; +} + +export default { + 'post /_search-center/doc/:index': function(req, res){ + switch(req.body.action){ + case 'SAVE': + res.send({ + errno: "0", + errmsg: "" + }); + break; + case 'ADD': + res.send({ + errno: "0", + errmsg: "", + payload: { + ...req.body.payload, + id: getUUID(), + } + }); + break; + case 'DELETE': + res.send({ + errno: "0" + }); + break; + default: + res.send(queryData) + } + }, + 'get /_search-center/indices/_cat': function(req, res){ + res.send({ + errno: "0", + payload: ["infini-test"], + }); + } +} \ No newline at end of file diff --git a/web/package.json b/web/package.json index 6332987c..b8567055 100644 --- a/web/package.json +++ b/web/package.json @@ -42,6 +42,7 @@ "react-dom": "^16.5.1", "react-fittext": "^1.0.0", "react-highlight-words": "^0.16.0", + "react-infinite-scroller": "^1.2.4", "react-json-prettify": "^0.2.0", "react-json-view": "^1.19.1", "react-router-dom": "^4.3.1", diff --git a/web/src/components/GlobalHeader/DropdownSelect.js b/web/src/components/GlobalHeader/DropdownSelect.js new file mode 100644 index 00000000..94e36ff7 --- /dev/null +++ b/web/src/components/GlobalHeader/DropdownSelect.js @@ -0,0 +1,117 @@ +import { Button, Dropdown, List, Spin, message, Icon } from 'antd'; +import React from 'react'; +import InfiniteScroll from 'react-infinite-scroller'; +import styles from './DropdownSelect.less'; + +class DropdownSelect extends React.Component{ + state={ + value: this.props.defaultValue, + data: this.props.data || [], + loading: false, + hasMore: true, + } + + handleItemClick = (item)=>{ + let preValue = this.state.value; + this.setState({ + value: item, + },()=>{ + let onChange = this.props.onChange; + if(preValue != item && onChange && typeof onChange == 'function'){ + onChange(item) + } + }) + } + + componentDidMount(){ + let data = []; + for(let i = 1; i<=28; i++){ + data.push('cluster'+i) + } + this.setState({ + data: data, + }) + } + fetchData = ()=>{ + let me = this; + return new Promise(resolve => { + setTimeout(() => { + let start = me.state.data.length; + let data =[] + for(let i = start + 1; i { + let { data } = this.state; + this.setState({ + loading: true, + }) + if (data.length > 50) { + message.warning('No more data'); + this.setState({ + hasMore: false, + loading: false, + }); + return; + } + this.fetchData().then((newdata)=>{ + data = data.concat(newdata); + this.setState({ + data, + loading: false, + }); + }); + } + + + render(){ + let me = this; + const menu = (
+
+ + ( + + + + )} + > + {this.state.loading && this.state.hasMore && ( +
+ +
+ )} + + +
+ {!this.state.loading && this.state.hasMore && ( +
+ pull load more +
+ )} +
); + return( + + + + ) + } + +} + +export default DropdownSelect; \ No newline at end of file diff --git a/web/src/components/GlobalHeader/DropdownSelect.less b/web/src/components/GlobalHeader/DropdownSelect.less new file mode 100644 index 00000000..7a2d6805 --- /dev/null +++ b/web/src/components/GlobalHeader/DropdownSelect.less @@ -0,0 +1,42 @@ +.btn-ds, .btn-ds:hover{ + min-width: 150px; + padding: 0; + border: none; + border-bottom: 1px solid #333; + text-align: left; + transition: all .3s cubic-bezier(.645,.045,.355,1); + border-radius: 0; + margin-left: 15px; + position: relative; + bottom: 8px; + span{ + color: #333; + } +} + +.dropmenu{ + box-shadow: 0 0 15px 0 rgba(0, 0, 0, .15); + padding: 20px; + width: 500px; + background: #fff; + .item{ + display: block; + float: left; + } + .btnitem{ + border-radius: 0px; + width: 100px; + } +} + +.infiniteContainer { + overflow: auto; + overflow-x: hidden; + height: 300px; +} +.loadingContainer { + position: absolute; + bottom: 40px; + width: 100%; + text-align: center; +} \ No newline at end of file diff --git a/web/src/components/GlobalHeader/index.js b/web/src/components/GlobalHeader/index.js index 156f1b7d..666f7172 100644 --- a/web/src/components/GlobalHeader/index.js +++ b/web/src/components/GlobalHeader/index.js @@ -4,6 +4,7 @@ import Link from 'umi/link'; import Debounce from 'lodash-decorators/debounce'; import styles from './index.less'; import RightContent from './RightContent'; +import DropdownSelect from './DropdownSelect' export default class GlobalHeader extends PureComponent { componentWillUnmount() { @@ -36,6 +37,9 @@ export default class GlobalHeader extends PureComponent { type={collapsed ? 'menu-unfold' : 'menu-fold'} onClick={this.toggle} /> + {}} + data={['cluster1', 'cluster2','cluster3', 'cluster4','cluster5', 'cluster6']}/> ); diff --git a/web/src/components/infini/InputSelect.js b/web/src/components/infini/InputSelect.js new file mode 100644 index 00000000..db1b1d3c --- /dev/null +++ b/web/src/components/infini/InputSelect.js @@ -0,0 +1,57 @@ +import React from 'react'; +import {Input, Menu, Dropdown} from 'antd'; + +class InputSelect extends React.Component{ + componentDidMount(){ + } + constructor(props){ + // console.log('new: '+ props.id) + super(props); + this.state = { + value: props.defaultValue || props.value, + originData: props.data || [], + } + } + + onClick = ({ key }) => { + this.setState({ + value: key, + }) + this.triggerChange(key) + } + triggerChange = (val)=>{ + let {onChange} = this.props; + if(onChange && typeof onChange == 'function'){ + onChange(val) + } + } + handleChange = (ev) => { + let val = ev.target.value; + let filterData = this.state.originData.slice(); + if(val != ""){ + filterData = filterData.filter(v=>v.value.includes(val)) + } + this.setState({ + value: val, + data: filterData + }) + this.triggerChange(val); + } + render(){ + let {id, data, onChange, value, ...restProps} = this.props; + let filterData = this.state.data || data || []; + return ( + + {filterData.map(op =>( + {op.label} + ))} + + } trigger={['focus']}> + + + ) + } +} + +export default InputSelect; \ No newline at end of file diff --git a/web/src/pages/DataManagement/Document.js b/web/src/pages/DataManagement/Document.js index 39eb6a3e..af2c62bd 100644 --- a/web/src/pages/DataManagement/Document.js +++ b/web/src/pages/DataManagement/Document.js @@ -3,10 +3,11 @@ import { formatMessage, FormattedMessage } from 'umi/locale'; import router from 'umi/router'; import { connect } from 'dva'; import { Col, Form, Row,Select, Input, Card,Icon, Table, InputNumber, Popconfirm, - Divider,Button,Tooltip, Cascader, Modal, DatePicker, Dropdown,Menu, message } from 'antd'; + Divider,Button,Tooltip, Modal, DatePicker, message } from 'antd'; import Editor, {monaco} from '@monaco-editor/react'; import moment from 'moment'; import {createDependencyProposals} from './autocomplete'; +import InputSelect from '@/components/infini/InputSelect'; function findParentIdentifier(textUntilPosition){ let chars = textUntilPosition; @@ -364,54 +365,6 @@ class EditableCell extends React.Component { } } -class InputSelect extends React.Component{ - state = { - value: this.props.defaultValue - } - onClick = ({ key }) => { - this.inputEl.setState({ - value: key - }) - this.setState({ - value: key, - }) - this.triggerChange(key) - } - triggerChange = (val)=>{ - let {onChange} = this.props; - if(onChange && typeof onChange == 'function'){ - onChange(val) - } - } - handleChange = (ev) => { - let val = ev.target.value; - let filterData = this.props.data.slice(); - if(val != ""){ - filterData = filterData.filter(v=>v.value.includes(val)) - } - this.setState({ - value: val, - data: filterData - }) - this.triggerChange(val); - } - render(){ - let {data, onChange, ...restProps} = this.props; - let filterData = this.state.data || data; - return ( - - {filterData.map(op =>( - {op.label} - ))} - - } trigger={['focus']}> - {this.inputEl=el}} {...restProps} onChange={this.handleChange}/> - - ) - } -} - @connect(({document})=>({ document })) @@ -440,14 +393,20 @@ class Doucment extends React.Component { componentDidMount(){ const {location, dispatch } = this.props; //console.log(match, location); - let index = location.query.index || 'infini-test'; + let index = location.query.index; let cluster = location.query.cluster || 'single-es'; + if(!cluster){ + return + } dispatch({ type: 'document/fetchIndices', payload: { cluster, } }).then(()=>{ + if(!index){ + return + } this.fetchData({ pageSize: 10, pageIndex: 1, @@ -559,7 +518,7 @@ class Doucment extends React.Component { }) const clusters = ["single-es"]; let {cluster, index}= this.props.document; - cluster = cluster || this.props.location.query.cluster; + cluster = cluster || this.props.location.query.cluster || 'single-es'; index = index || this.props.location.query.index; return (
@@ -620,14 +579,13 @@ class Doucment extends React.Component {
query example:
-   "bool":{
-     "must": {
-       "match":{
-         "FIELD": "VALUE"
-       }
-     }
-   }
}` }}> - + 
{JSON.stringify({
+                              must: {
+                                match: {
+                                  FIELD: "VALUE"
+                                }
+                              }
+                            }, null, 2)}
diff --git a/web/src/pages/DataManagement/Rebuild.js b/web/src/pages/DataManagement/Rebuild.js index f6efb503..ccde8741 100644 --- a/web/src/pages/DataManagement/Rebuild.js +++ b/web/src/pages/DataManagement/Rebuild.js @@ -1,20 +1,43 @@ import React, { Component } from 'react'; import { connect } from 'dva'; import PageHeaderWrapper from '@/components/PageHeaderWrapper'; -import {Steps, Card, Form, Select, Input,Button, Divider,message,Spin, Row, Col,Result} from 'antd'; +import {Steps, Card, Form, Select, Input,Button, Divider,message, InputNumber} from 'antd'; +import InputSelect from '@/components/infini/InputSelect'; const {Step} = Steps; const {Option} = Select; const {TextArea} = Input; @Form.create() -@connect() +@connect(({document}) => ({ + document +})) class Rebuild extends Component { state = { currentStep: 0, - configData: {}, + configData: { + source:{}, + dest:{}, + }, } - renderSteps(currentStep){ + componentDidMount(){ + const {dispatch} = this.props; + dispatch({ + type:'document/fetchIndices', + payload:{ + cluster: 'sinlge-es' + } + }) + } + renderSteps = (currentStep) => { + let {clusterIndices} = this.props.document; + clusterIndices = clusterIndices || []; + let indices = clusterIndices.map((item)=>{ + return { + label: item, + value: item, + } + }); var stepDom = ''; const { getFieldDecorator } = this.props.form; const formItemLayout = { @@ -28,6 +51,9 @@ class Rebuild extends Component { sm: { span: 16 }, md:{span:20}, }, + style: { + marginBottom: 10, + } }; const tailFormItemLayout = { wrapperCol: { @@ -49,35 +75,23 @@ class Rebuild extends Component { case 0: stepDom = (
- - {getFieldDecorator('source_index', { - initialValue: this.state.configData.source_index, - rules: [{ required: true, message: '请选择要重建的索引' }], + + {getFieldDecorator('name', { + initialValue: this.state.configData.name, + rules: [{ required: true, message: 'please input a task name' }], })( - , + )} - - {getFieldDecorator('creterial', { + + {getFieldDecorator('desc', { initialValue: this.state.configData.creterial, rules: [ - {required: true, }, ], })(