From 8d66f9f22b993136d63dd341513ed8dc285c4bae Mon Sep 17 00:00:00 2001 From: silenceqi Date: Fri, 22 Jan 2021 21:17:23 +0800 Subject: [PATCH] modify document management for es 2.x --- api/index_management/cluster.go | 17 ++ api/index_management/document.go | 14 +- api/index_management/indices.go | 6 - api/index_management/rebuild.go | 10 +- api/init.go | 2 + web/mock/datamanagement/document.js | 50 ++--- web/mock/datamanagement/rebuild.js | 6 +- web/src/layouts/BasicLayout.js | 6 + web/src/models/cluster.js | 26 +++ web/src/pages/DataManagement/Document.js | 176 ++++++++++++------ web/src/pages/DataManagement/Index.js | 2 +- .../pages/DataManagement/models/document.js | 9 + web/src/pages/DataManagement/models/index.js | 2 +- web/src/services/cluster.js | 8 + web/src/services/doc.js | 18 +- web/src/utils/elasticsearch.js | 102 +++++++++- 16 files changed, 342 insertions(+), 112 deletions(-) create mode 100644 api/index_management/cluster.go create mode 100644 web/src/models/cluster.js create mode 100644 web/src/services/cluster.js diff --git a/api/index_management/cluster.go b/api/index_management/cluster.go new file mode 100644 index 00000000..bfe7b506 --- /dev/null +++ b/api/index_management/cluster.go @@ -0,0 +1,17 @@ +package index_management + +import ( + "net/http" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" +) + +func (handler APIHandler) GetClusterVersion(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + client := elastic.GetClient(handler.Config.Elasticsearch) + ver := client.GetMajorVersion() + resBody := newResponseBody() + resBody["payload"] = map[string]int{ + "major": ver, + } + handler.WriteJSON(w, resBody, http.StatusOK) +} \ No newline at end of file diff --git a/api/index_management/document.go b/api/index_management/document.go index 31b632e0..e891874f 100644 --- a/api/index_management/document.go +++ b/api/index_management/document.go @@ -36,7 +36,8 @@ func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *ht if strings.Trim(id, "/") == "" { id = util.GetUUID() } - _, err = client.Index(indexName, id, reqBody) + docType := handler.GetParameter(req, "_type") + _, err = client.Index(indexName, docType, id, reqBody) if err != nil { resResult["status"] = false resResult["error"] = err @@ -61,7 +62,8 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req } indexName := ps.ByName("index") id := ps.ByName("id") - resp, err := client.Get(indexName, id) + typ := handler.GetParameter(req, "_type") + resp, err := client.Get(indexName,typ, id) if err != nil { resResult["status"] = false resResult["error"] = err.Error() @@ -70,9 +72,12 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req } source := resp.Source for k, v := range reqBody { + if k == "id" { + continue + } source[k] = v } - _, err = client.Index(indexName, id, source) + _, err = client.Index(indexName, typ, id, source) if err != nil { resResult["status"] = false resResult["error"] = err.Error() @@ -88,7 +93,8 @@ func (handler APIHandler) HandleDeleteDocumentAction(w http.ResponseWriter, req resResult := newResponseBody() indexName := ps.ByName("index") id := ps.ByName("id") - _, err := client.Delete(indexName, id) + typ := handler.GetParameter(req, "_type") + _, err := client.Delete(indexName, typ, id) if err != nil { resResult["error"] = err.Error() resResult["status"] = false diff --git a/api/index_management/indices.go b/api/index_management/indices.go index 50970447..75508841 100644 --- a/api/index_management/indices.go +++ b/api/index_management/indices.go @@ -1,7 +1,6 @@ package index_management import ( - "fmt" "net/http" "strings" @@ -111,11 +110,6 @@ func (handler APIHandler) HandleDeleteIndexAction(w http.ResponseWriter, req *ht } func (handler APIHandler) HandleCreateIndexAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - defer func() { - if err := recover(); err != nil { - fmt.Println(err) - } - }() client := elastic.GetClient(handler.Config.Elasticsearch) indexName := ps.ByName("index") resBody := newResponseBody() diff --git a/api/index_management/rebuild.go b/api/index_management/rebuild.go index c79a5c3f..e7463f83 100644 --- a/api/index_management/rebuild.go +++ b/api/index_management/rebuild.go @@ -31,8 +31,8 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R } //fmt.Println(reindexItem) - - ID, err := reindex(handler.Config.Elasticsearch, reindexItem) + typ := handler.GetParameter(req, "_type") + ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ) if err != nil { resResult["error"] = err resResult["status"] = false @@ -43,7 +43,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R handler.WriteJSON(w, resResult, http.StatusOK) } -func reindex(esName string, body *model.Reindex) (string, error) { +func reindex(esName string, body *model.Reindex, typ string) (string, error) { client := elastic.GetClient(esName) source := map[string]interface{}{ "index": body.Source.Index, @@ -77,7 +77,7 @@ func reindex(esName string, body *model.Reindex) (string, error) { body.Status = model.ReindexStatusRunning body.CreatedAt = time.Now() - _, err = client.Index(orm.GetIndexName(body), body.ID, body) + _, err = client.Index(orm.GetIndexName(body), typ, body.ID, body) if err != nil { return "", err } @@ -171,7 +171,7 @@ func SyncRebuildResult(esName string) error { } source["status"] = status source["task_source"] = doc.Source - _, err := client.Index(orm.GetIndexName(model.Reindex{}), esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source) + _, err := client.Index(orm.GetIndexName(model.Reindex{}), "", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source) return err } return nil diff --git a/api/init.go b/api/init.go index 7e489256..abfae4ee 100644 --- a/api/init.go +++ b/api/init.go @@ -36,6 +36,8 @@ func Init(cfg *config.AppConfig) { ui.HandleUIMethod(api.DELETE, pathPrefix+"index/:index", handler.HandleDeleteIndexAction) ui.HandleUIMethod(api.POST, pathPrefix+"index/:index", handler.HandleCreateIndexAction) + ui.HandleUIMethod(api.GET, pathPrefix+"cluster/:cluster/version", handler.GetClusterVersion) + task.RegisterScheduleTask(task.ScheduleTask{ Description: "sync reindex task result to index infinireindex", Task: func() { diff --git a/web/mock/datamanagement/document.js b/web/mock/datamanagement/document.js index 05c62c62..7b2aa66d 100644 --- a/web/mock/datamanagement/document.js +++ b/web/mock/datamanagement/document.js @@ -12,29 +12,29 @@ function getUUID(len){ } export default { - 'post /_search-center/doc/:index/_search': function(req, res){ - res.send(queryData) - }, - 'post /_search-center/doc/:index/_create': function(req, res){ - res.send({ - status: true, - payload: { - ...req.body.payload, - id: getUUID(), - } - }); - }, - 'put /_search-center/doc/:index/:id': function(req, res){ - res.send({ - status: true, - payload: req.body - }); - }, - - 'delete /_search-center/doc/:index/:id': function(req, res){ - res.send({ - status: true, - payload: null, - }); - } + // 'post /_search-center/doc/:index/_search': function(req, res){ + // res.send(queryData) + // }, + // 'post /_search-center/doc/:index/_create': function(req, res){ + // res.send({ + // status: true, + // payload: { + // ...req.body.payload, + // id: getUUID(), + // } + // }); + // }, + // 'put /_search-center/doc/:index/:id': function(req, res){ + // res.send({ + // status: true, + // payload: req.body + // }); + // }, + // + // 'delete /_search-center/doc/:index/:id': function(req, res){ + // res.send({ + // status: true, + // payload: null, + // }); + // } } \ No newline at end of file diff --git a/web/mock/datamanagement/rebuild.js b/web/mock/datamanagement/rebuild.js index 20c2a05d..6c27bfe3 100644 --- a/web/mock/datamanagement/rebuild.js +++ b/web/mock/datamanagement/rebuild.js @@ -527,7 +527,7 @@ let data = { }; export default { - 'get /_search-center/rebuild/_search': function(req, res){ - res.send(data) - } + // 'get /_search-center/rebuild/_search': function(req, res){ + // res.send(data) + // } } \ No newline at end of file diff --git a/web/src/layouts/BasicLayout.js b/web/src/layouts/BasicLayout.js index 4f56066f..b1b3902b 100644 --- a/web/src/layouts/BasicLayout.js +++ b/web/src/layouts/BasicLayout.js @@ -102,6 +102,12 @@ class BasicLayout extends React.PureComponent { dispatch({ type: 'setting/getSetting', }); + dispatch({ + type: 'cluster/fetchClusterVersion', + payload: { + cluster: 'single-es' + } + }); this.renderRef = requestAnimationFrame(() => { this.setState({ rendering: false, diff --git a/web/src/models/cluster.js b/web/src/models/cluster.js new file mode 100644 index 00000000..855f020c --- /dev/null +++ b/web/src/models/cluster.js @@ -0,0 +1,26 @@ +import { getClusterVersion } from '@/services/cluster'; + +export default { + namespace: 'cluster', + state: { + }, + + effects: { + *fetchClusterVersion({ payload }, { call, put }) { + let res = yield call(getClusterVersion, payload); + yield put({ + type: 'saveData', + payload: res.payload + }) + } + }, + + reducers: { + saveData(state, {payload}){ + return { + ...state, + ...payload, + } + } + }, +}; diff --git a/web/src/pages/DataManagement/Document.js b/web/src/pages/DataManagement/Document.js index 6a9f4e6a..8180ef0e 100644 --- a/web/src/pages/DataManagement/Document.js +++ b/web/src/pages/DataManagement/Document.js @@ -3,12 +3,12 @@ import { formatMessage, FormattedMessage } from 'umi/locale'; import router from 'umi/router'; import { connect } from 'dva'; import { Col, Form, Row,Select, Input, Card,Icon, Table, InputNumber, Popconfirm, - Divider,Button,Tooltip, Modal, DatePicker, message } from 'antd'; + Divider,Button,Tooltip, Modal, DatePicker, message,Cascader } from 'antd'; import Editor, {monaco} from '@monaco-editor/react'; import moment from 'moment'; import {createDependencyProposals} from './autocomplete'; import InputSelect from '@/components/infini/InputSelect'; -import {getFields} from '@/utils/elasticsearch'; +import {getFields,getESAPI} from '@/utils/elasticsearch'; function findParentIdentifier(textUntilPosition){ let chars = textUntilPosition; @@ -60,36 +60,44 @@ function findParentIdentifier(textUntilPosition){ // }, // }, // }); -monaco.init().then((mi)=>{ - mi.languages.onLanguage("json", ()=>{ - mi.languages.registerCompletionItemProvider('json', { - triggerCharacters: ['"'], - provideCompletionItems: function(model, position, ctx) { - var textUntilPosition = model.getValueInRange({startLineNumber: 1, startColumn: 1, endLineNumber: position.lineNumber, endColumn: position.column}); - - if(textUntilPosition.indexOf('{') < 0){ - return { suggestions: [] }; - } - - let key = findParentIdentifier(textUntilPosition); - var word = model.getWordUntilPosition(position); - - var range = { - startLineNumber: position.lineNumber, - endLineNumber: position.lineNumber, - startColumn: word.startColumn, - endColumn: word.endColumn - }; - - //console.log(ctx, range,textUntilPosition) - return { - suggestions: createDependencyProposals(key, range, mi, ctx.triggerCharacter) - }; - } - }); - }) +var langDisposer = null; +function initEditor() { + monaco.init().then((mi) => { + //mi.languages.onLanguage("json", () => { + langDisposer = mi.languages.registerCompletionItemProvider('json', { + triggerCharacters: ['"'], + provideCompletionItems: function (model, position, ctx) { + var textUntilPosition = model.getValueInRange({ + startLineNumber: 1, + startColumn: 1, + endLineNumber: position.lineNumber, + endColumn: position.column + }); -}) + if (textUntilPosition.indexOf('{') < 0) { + return {suggestions: []}; + } + + let key = findParentIdentifier(textUntilPosition); + var word = model.getWordUntilPosition(position); + + var range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn + }; + + //console.log(ctx, range,textUntilPosition) + return { + suggestions: createDependencyProposals(key, range, mi, ctx.triggerCharacter) + }; + } + }); + // }) + + }) +} const {Option} = Select; @@ -202,7 +210,14 @@ class EditableCell extends React.Component { // console.log(record, doclist.mappings) // return // } - const {properties} = doclist.mappings[record._index].mappings; + let properties = null; + let _type = record._type || doclist._type; + if(typeof _type !== 'undefined' && _type !== '' && _type !== '_doc'){ + properties = doclist.mappings[record._index].mappings[_type].properties; + }else{ + properties = doclist.mappings[record._index].mappings.properties; + } + if(!properties[key]){ return ''; } @@ -236,6 +251,7 @@ class EditableCell extends React.Component { type: 'document/saveDocItem', payload: { index: doclist._index, + _type: doclist._type, data: { id: key, ...row, @@ -247,6 +263,7 @@ class EditableCell extends React.Component { type: 'document/addDocItem', payload: { index: doclist._index, + _type: doclist._type, data: row, } }) @@ -258,7 +275,7 @@ class EditableCell extends React.Component { const {dispatch} = this.props; dispatch({ type: 'document/saveData', - payload: { editingKey: record.id, _index: record._index } + payload: { editingKey: record.id, _index: record._index, _type: record._type } }); } @@ -268,6 +285,7 @@ class EditableCell extends React.Component { type: 'document/deleteDocItem', payload: { index: record._index, + _type: record._type, data: { id: record.id, } @@ -336,6 +354,10 @@ class EditableCell extends React.Component { cell: EditableCell, }, }; + let total = doclist.total || 0; + if(total.value){ + total = total.value; + } return ( `Total ${total} items`, @@ -362,19 +384,20 @@ class EditableCell extends React.Component { } } -@connect(({document})=>({ - document +@connect(({document,cluster})=>({ + document, + cluster, })) @Form.create() class Doucment extends React.Component { - state={ + state={ bodyDisplay: 'none', } // constructor(props){ // super(props); // this.filterGetter = createRef(); // } - + fetchData = (params) => { const {dispatch} = this.props; return dispatch({ @@ -386,8 +409,13 @@ class Doucment extends React.Component { handleEditorDidMount = (_valueGetter) =>{ this.filterGetter = _valueGetter; } - + componentWillUnmount() { + if(langDisposer != null) { + langDisposer.dispose(); + } + } componentDidMount(){ + initEditor() const {location, dispatch } = this.props; //console.log(match, location); let index = location.query.index; @@ -421,19 +449,23 @@ class Doucment extends React.Component { } handleNewClick = ()=>{ - const {dispatch, document} = this.props; + const {dispatch, document,cluster} = this.props; if(document.isAddNew){ //!document.data || document.data.length == 0 return; } - let {indices, mappings} = document; - if(!indices){ + let {mappings, indices} = document; + if(indices.length === 0) { + indices = Object.keys(mappings); + } + if(indices.length === 0){ return } let _index = indices[0]; - if(indices && indices.length > 1){ + let _type = ''; + if(indices.length > 0){ //console.log(this.indexSelEl); - let vals = this.indexSelEl.rcSelect.state.value; - if(vals.length == 0){ + let vals = this.indexSelEl.state.value; + if(vals.length === 0){ Modal.error({ title: '系统提示', content: '请选择新建文档目标索引', @@ -441,13 +473,20 @@ class Doucment extends React.Component { return }else{ _index = vals[0]; + if(vals.length>1){ + _type = vals[1]; + } } } - let properties = mappings[_index].mappings.properties; + let properties = getESAPI(cluster.major).getProperties({ + index: _index, + mappings: mappings, + typ: _type, + }); //mappings[_index].mappings.properties; let keys = Object.keys(properties) - let newDoc = {id:"", _index}; + let newDoc = {id:"", _index,_type}; for(let key of keys){ - if(properties[key].type == 'date'){ + if(properties[key].type === 'date'){ newDoc[key] = null }else{ newDoc[key] = "" @@ -459,7 +498,8 @@ class Doucment extends React.Component { docItem: newDoc, extra: { isAddNew: true, - _index + _index, + _type, } }, }) @@ -495,17 +535,43 @@ class Doucment extends React.Component { } renderNew = ()=>{ - const {indices} = this.props.document; + let {indices, mappings} = this.props.document; // if((indices && indices.length > 1)){ // return; // } + const {major} = this.props.cluster; + if(indices && indices.length >= 0){ + indices = getESAPI(major).extractIndicesFromMappings(mappings).filter(item=>{ + if(indices.length > 0){ + return indices.indexOf(item.index) > -1; + } + return true; + }).map(item=>{ + + let newItem= { + label: item.index, + value: item.index, + }; + if(item.types){ + newItem.children = item.types.map(typ=>{ + return { + label: typ, + value: typ, + } + }) + } + return newItem; + }) + } return (
- {(indices && indices.length > 1) ? () : ''} + {(indices) ? ({this.indexSelEl=el}} options={indices} style={{width: 200, marginRight:5}} placeholder="please select a index"> + ) : ''} + {/*{(indices) ? () : ''}*/}
) diff --git a/web/src/pages/DataManagement/Index.js b/web/src/pages/DataManagement/Index.js index 1a5ee556..44930e8b 100644 --- a/web/src/pages/DataManagement/Index.js +++ b/web/src/pages/DataManagement/Index.js @@ -332,7 +332,7 @@ class Index extends PureComponent {
{ + for (let fi in filterMappings[key].mappings[typ].properties) { + fields.indexOf(fi) === - 1 && fields.push(fi); + } + }) } } @@ -29,22 +37,98 @@ export function getFields(index, mappings){ export function formatESSearchResult(esResp) { const total = esResp.hits.total - if(total.value == 0){ + if(total == null || total.value == 0){ return { total: total, data: [], }; } let dataArr = []; - for(let hit of esResp.hits.hits) { - if(!hit._source.id){ - hit._source["id"] = hit._id + if(esResp.hits.hits) { + for (let hit of esResp.hits.hits) { + if (!hit._source.id) { + hit._source["id"] = hit._id + } + hit._source["_index"] = hit._index + if(hit["_type"]){ + hit._source["_type"] = hit["_type"] + } + dataArr.push(hit._source) } - hit._source["_index"] = hit._index - dataArr.push(hit._source) } return { total: total, data: dataArr, } +} + +const ESAPIV0 = { + getProperties(params){ + const {index, mappings} = params; + if(typeof mappings[index] === 'undefined'){ + return {}; + } + return mappings[index].mappings.properties; + }, + extractIndicesFromMappings(mappings){ + if(!mappings){ + return []; + } + return Object.keys(mappings).map(index=>{return{ + index: index + }}); + } +} + +const ESAPIV2 = { + ...ESAPIV0, + getProperties(params){ + const {index, mappings, typ} = params; + if(typeof mappings[index] === 'undefined'){ + return {}; + } + let targetMappings = mappings[index].mappings; + if(typeof targetMappings[typ] === 'undefined'){ + return {} + } + return targetMappings[typ].properties; + }, + extractIndicesFromMappings(mappings){ + let indices = []; + for(let index in mappings){ + indices.push({ + index: index, + types: Object.keys(mappings[index].mappings).map(typ => { return typ}) + }); + } + return indices; + } +} + + +const ESAPIV5 = { + ...ESAPIV2 +} + +const ESAPIV6 = { + ...ESAPIV5, + typeName: "_doc", + getProperties(params){ + params.typ = this.typeName; + return ESAPIV5.getProperties(params); + } +} + +const ESAPI = { + "0": ESAPIV0, + "2": ESAPIV2, + "5": ESAPIV5, + "6": ESAPIV6, +} + +export function getESAPI(ver){ + if(typeof ESAPI[ver] != "undefined"){ + return ESAPI[ver]; + } + return ESAPI["0"]; } \ No newline at end of file