floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify document management for es 2.x

This commit is contained in:
silenceqi 2021-01-22 21:17:23 +08:00
parent e1ad5515a1
commit 8d66f9f22b
16 changed files with 342 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
api/index_management/cluster.go Normal file
View File

@ -0,0 +1,17 @@
package index_management
import (
"net/http"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/elastic"
)
func (handler APIHandler) GetClusterVersion(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
ver := client.GetMajorVersion()
resBody := newResponseBody()
resBody["payload"] = map[string]int{
"major": ver,
}
handler.WriteJSON(w, resBody, http.StatusOK)
}

14
api/index_management/document.go
View File

@ -36,7 +36,8 @@ func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *ht
if strings.Trim(id, "/") == "" {
id = util.GetUUID()
}
_, err = client.Index(indexName, id, reqBody)
docType := handler.GetParameter(req, "_type")
_, err = client.Index(indexName, docType, id, reqBody)
if err != nil {
resResult["status"] = false
resResult["error"] = err
@ -61,7 +62,8 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req
}
indexName := ps.ByName("index")
id := ps.ByName("id")
resp, err := client.Get(indexName, id)
typ := handler.GetParameter(req, "_type")
resp, err := client.Get(indexName,typ, id)
if err != nil {
resResult["status"] = false
resResult["error"] = err.Error()
@ -70,9 +72,12 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req
}
source := resp.Source
for k, v := range reqBody {
if k == "id" {
continue
}
source[k] = v
}
_, err = client.Index(indexName, id, source)
_, err = client.Index(indexName, typ, id, source)
if err != nil {
resResult["status"] = false
resResult["error"] = err.Error()
@ -88,7 +93,8 @@ func (handler APIHandler) HandleDeleteDocumentAction(w http.ResponseWriter, req
resResult := newResponseBody()
indexName := ps.ByName("index")
id := ps.ByName("id")
_, err := client.Delete(indexName, id)
typ := handler.GetParameter(req, "_type")
_, err := client.Delete(indexName, typ, id)
if err != nil {
resResult["error"] = err.Error()
resResult["status"] = false

6
api/index_management/indices.go
View File

@ -1,7 +1,6 @@
package index_management
import (
"fmt"
"net/http"
"strings"
@ -111,11 +110,6 @@ func (handler APIHandler) HandleDeleteIndexAction(w http.ResponseWriter, req *ht
}
func (handler APIHandler) HandleCreateIndexAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
}
}()
client := elastic.GetClient(handler.Config.Elasticsearch)
indexName := ps.ByName("index")
resBody := newResponseBody()

10
api/index_management/rebuild.go
View File

@ -31,8 +31,8 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
}
//fmt.Println(reindexItem)
ID, err := reindex(handler.Config.Elasticsearch, reindexItem)
typ := handler.GetParameter(req, "_type")
ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ)
if err != nil {
resResult["error"] = err
resResult["status"] = false
@ -43,7 +43,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
handler.WriteJSON(w, resResult, http.StatusOK)
}
func reindex(esName string, body *model.Reindex) (string, error) {
func reindex(esName string, body *model.Reindex, typ string) (string, error) {
client := elastic.GetClient(esName)
source := map[string]interface{}{
"index": body.Source.Index,
@ -77,7 +77,7 @@ func reindex(esName string, body *model.Reindex) (string, error) {
body.Status = model.ReindexStatusRunning
body.CreatedAt = time.Now()
_, err = client.Index(orm.GetIndexName(body), body.ID, body)
_, err = client.Index(orm.GetIndexName(body), typ, body.ID, body)
if err != nil {
return "", err
}
@ -171,7 +171,7 @@ func SyncRebuildResult(esName string) error {
}
source["status"] = status
source["task_source"] = doc.Source
_, err := client.Index(orm.GetIndexName(model.Reindex{}), esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
_, err := client.Index(orm.GetIndexName(model.Reindex{}), "", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
return err
}
return nil

2
api/init.go
View File

@ -36,6 +36,8 @@ func Init(cfg *config.AppConfig) {
ui.HandleUIMethod(api.DELETE, pathPrefix+"index/:index", handler.HandleDeleteIndexAction)
ui.HandleUIMethod(api.POST, pathPrefix+"index/:index", handler.HandleCreateIndexAction)
ui.HandleUIMethod(api.GET, pathPrefix+"cluster/:cluster/version", handler.GetClusterVersion)
task.RegisterScheduleTask(task.ScheduleTask{
Description: "sync reindex task result to index infinireindex",
Task: func() {

50
web/mock/datamanagement/document.js
View File

@ -12,29 +12,29 @@ function getUUID(len){
}
export default {
'post /_search-center/doc/:index/_search': function(req, res){
res.send(queryData)
},
'post /_search-center/doc/:index/_create': function(req, res){
res.send({
status: true,
payload: {
...req.body.payload,
id: getUUID(),
}
});
},
'put /_search-center/doc/:index/:id': function(req, res){
res.send({
status: true,
payload: req.body
});
},
'delete /_search-center/doc/:index/:id': function(req, res){
res.send({
status: true,
payload: null,
});
}
// 'post /_search-center/doc/:index/_search': function(req, res){
// res.send(queryData)
// },
// 'post /_search-center/doc/:index/_create': function(req, res){
// res.send({
// status: true,
// payload: {
// ...req.body.payload,
// id: getUUID(),
// }
// });
// },
// 'put /_search-center/doc/:index/:id': function(req, res){
// res.send({
// status: true,
// payload: req.body
// });
// },
//
// 'delete /_search-center/doc/:index/:id': function(req, res){
// res.send({
// status: true,
// payload: null,
// });
// }
}

6
web/mock/datamanagement/rebuild.js
View File

@ -527,7 +527,7 @@ let data = {
};
export default {
'get /_search-center/rebuild/_search': function(req, res){
res.send(data)
}
// 'get /_search-center/rebuild/_search': function(req, res){
// res.send(data)
// }
}

6
web/src/layouts/BasicLayout.js
View File

@ -102,6 +102,12 @@ class BasicLayout extends React.PureComponent {
dispatch({
type: 'setting/getSetting',
});
dispatch({
type: 'cluster/fetchClusterVersion',
payload: {
cluster: 'single-es'
}
});
this.renderRef = requestAnimationFrame(() => {
this.setState({
rendering: false,

26
web/src/models/cluster.js Normal file
View File

@ -0,0 +1,26 @@
import { getClusterVersion } from '@/services/cluster';
export default {
namespace: 'cluster',
state: {
},
effects: {
*fetchClusterVersion({ payload }, { call, put }) {
let res = yield call(getClusterVersion, payload);
yield put({
type: 'saveData',
payload: res.payload
})
}
},
reducers: {
saveData(state, {payload}){
return {
...state,
...payload,
}
}
},
};

176
web/src/pages/DataManagement/Document.js
View File

@ -3,12 +3,12 @@ import { formatMessage, FormattedMessage } from 'umi/locale';
import router from 'umi/router';
import { connect } from 'dva';
import { Col, Form, Row,Select, Input, Card,Icon, Table, InputNumber, Popconfirm,
Divider,Button,Tooltip, Modal, DatePicker, message } from 'antd';
Divider,Button,Tooltip, Modal, DatePicker, message,Cascader } from 'antd';
import Editor, {monaco} from '@monaco-editor/react';
import moment from 'moment';
import {createDependencyProposals} from './autocomplete';
import InputSelect from '@/components/infini/InputSelect';
import {getFields} from '@/utils/elasticsearch';
import {getFields,getESAPI} from '@/utils/elasticsearch';
function findParentIdentifier(textUntilPosition){
let chars = textUntilPosition;
@ -60,36 +60,44 @@ function findParentIdentifier(textUntilPosition){
// },
// },
// });
monaco.init().then((mi)=>{
mi.languages.onLanguage("json", ()=>{
mi.languages.registerCompletionItemProvider('json', {
triggerCharacters: ['"'],
provideCompletionItems: function(model, position, ctx) {
var textUntilPosition = model.getValueInRange({startLineNumber: 1, startColumn: 1, endLineNumber: position.lineNumber, endColumn: position.column});
if(textUntilPosition.indexOf('{') < 0){
return { suggestions: [] };
}
let key = findParentIdentifier(textUntilPosition);
var word = model.getWordUntilPosition(position);
var range = {
startLineNumber: position.lineNumber,
endLineNumber: position.lineNumber,
startColumn: word.startColumn,
endColumn: word.endColumn
};
//console.log(ctx, range,textUntilPosition)
return {
suggestions: createDependencyProposals(key, range, mi, ctx.triggerCharacter)
};
}
});
})
var langDisposer = null;
function initEditor() {
monaco.init().then((mi) => {
//mi.languages.onLanguage("json", () => {
langDisposer = mi.languages.registerCompletionItemProvider('json', {
triggerCharacters: ['"'],
provideCompletionItems: function (model, position, ctx) {
var textUntilPosition = model.getValueInRange({
startLineNumber: 1,
startColumn: 1,
endLineNumber: position.lineNumber,
endColumn: position.column
});
})
if (textUntilPosition.indexOf('{') < 0) {
return {suggestions: []};
}
let key = findParentIdentifier(textUntilPosition);
var word = model.getWordUntilPosition(position);
var range = {
startLineNumber: position.lineNumber,
endLineNumber: position.lineNumber,
startColumn: word.startColumn,
endColumn: word.endColumn
};
//console.log(ctx, range,textUntilPosition)
return {
suggestions: createDependencyProposals(key, range, mi, ctx.triggerCharacter)
};
}
});
// })
})
}
const {Option} = Select;
@ -202,7 +210,14 @@ class EditableCell extends React.Component {
// console.log(record, doclist.mappings)
// return
// }
const {properties} = doclist.mappings[record._index].mappings;
let properties = null;
let _type = record._type || doclist._type;
if(typeof _type !== 'undefined' && _type !== '' && _type !== '_doc'){
properties = doclist.mappings[record._index].mappings[_type].properties;
}else{
properties = doclist.mappings[record._index].mappings.properties;
}
if(!properties[key]){
return '';
}
@ -236,6 +251,7 @@ class EditableCell extends React.Component {
type: 'document/saveDocItem',
payload: {
index: doclist._index,
_type: doclist._type,
data: {
id: key,
...row,
@ -247,6 +263,7 @@ class EditableCell extends React.Component {
type: 'document/addDocItem',
payload: {
index: doclist._index,
_type: doclist._type,
data: row,
}
})
@ -258,7 +275,7 @@ class EditableCell extends React.Component {
const {dispatch} = this.props;
dispatch({
type: 'document/saveData',
payload: { editingKey: record.id, _index: record._index }
payload: { editingKey: record.id, _index: record._index, _type: record._type }
});
}
@ -268,6 +285,7 @@ class EditableCell extends React.Component {
type: 'document/deleteDocItem',
payload: {
index: record._index,
_type: record._type,
data: {
id: record.id,
}
@ -336,6 +354,10 @@ class EditableCell extends React.Component {
cell: EditableCell,
},
};
let total = doclist.total || 0;
if(total.value){
total = total.value;
}
return (
<EditableContext.Provider value={this.props.form}>
<Table
@ -350,7 +372,7 @@ class EditableCell extends React.Component {
rowClassName="editable-row"
pagination={{
showSizeChanger: true,
total: doclist.total? doclist.total.value: 0,
total: total,
pageSize: doclist.pageSize,
current: doclist.pageIndex,
showTotal: (total, range) => `Total ${total} items`,
@ -362,19 +384,20 @@ class EditableCell extends React.Component {
}
}
@connect(({document})=>({
document
@connect(({document,cluster})=>({
document,
cluster,
}))
@Form.create()
class Doucment extends React.Component {
state={
state={
bodyDisplay: 'none',
}
// constructor(props){
// super(props);
// this.filterGetter = createRef();
// }
fetchData = (params) => {
const {dispatch} = this.props;
return dispatch({
@ -386,8 +409,13 @@ class Doucment extends React.Component {
handleEditorDidMount = (_valueGetter) =>{
this.filterGetter = _valueGetter;
}
componentWillUnmount() {
if(langDisposer != null) {
langDisposer.dispose();
}
}
componentDidMount(){
initEditor()
const {location, dispatch } = this.props;
//console.log(match, location);
let index = location.query.index;
@ -421,19 +449,23 @@ class Doucment extends React.Component {
}
handleNewClick = ()=>{
const {dispatch, document} = this.props;
const {dispatch, document,cluster} = this.props;
if(document.isAddNew){ //!document.data || document.data.length == 0
return;
}
let {indices, mappings} = document;
if(!indices){
let {mappings, indices} = document;
if(indices.length === 0) {
indices = Object.keys(mappings);
}
if(indices.length === 0){
return
}
let _index = indices[0];
if(indices && indices.length > 1){
let _type = '';
if(indices.length > 0){
//console.log(this.indexSelEl);
let vals = this.indexSelEl.rcSelect.state.value;
if(vals.length == 0){
let vals = this.indexSelEl.state.value;
if(vals.length === 0){
Modal.error({
title: '系统提示',
content: '请选择新建文档目标索引',
@ -441,13 +473,20 @@ class Doucment extends React.Component {
return
}else{
_index = vals[0];
if(vals.length>1){
_type = vals[1];
}
}
}
let properties = mappings[_index].mappings.properties;
let properties = getESAPI(cluster.major).getProperties({
index: _index,
mappings: mappings,
typ: _type,
}); //mappings[_index].mappings.properties;
let keys = Object.keys(properties)
let newDoc = {id:"", _index};
let newDoc = {id:"", _index,_type};
for(let key of keys){
if(properties[key].type == 'date'){
if(properties[key].type === 'date'){
newDoc[key] = null
}else{
newDoc[key] = ""
@ -459,7 +498,8 @@ class Doucment extends React.Component {
docItem: newDoc,
extra: {
isAddNew: true,
_index
_index,
_type,
}
},
})
@ -495,17 +535,43 @@ class Doucment extends React.Component {
}
renderNew = ()=>{
const {indices} = this.props.document;
let {indices, mappings} = this.props.document;
// if((indices && indices.length > 1)){
// return;
// }
const {major} = this.props.cluster;
if(indices && indices.length >= 0){
indices = getESAPI(major).extractIndicesFromMappings(mappings).filter(item=>{
if(indices.length > 0){
return indices.indexOf(item.index) > -1;
}
return true;
}).map(item=>{
let newItem= {
label: item.index,
value: item.index,
};
if(item.types){
newItem.children = item.types.map(typ=>{
return {
label: typ,
value: typ,
}
})
}
return newItem;
})
}
return (
<div>
{(indices && indices.length > 1) ? (<Select ref={el=>{this.indexSelEl=el}} style={{width: 200, marginRight:5}} placeholder="please select a index">
{indices.map(item=>{
return (<Select.Option key={item} label={item}>{item}</Select.Option>)
})}
</Select>) : ''}
{(indices) ? (<Cascader ref={el=>{this.indexSelEl=el}} options={indices} style={{width: 200, marginRight:5}} placeholder="please select a index">
</Cascader>) : ''}
{/*{(indices) ? (<Select ref={el=>{this.indexSelEl=el}} style={{width: 200, marginRight:5}} placeholder="please select a index">*/}
{/* {indices.map(item=>{*/}
{/* return (<Select.Option key={item} label={item}>{item}</Select.Option>)*/}
{/* })}*/}
{/*</Select>) : ''}*/}
<Button type="primary" icon="plus" onClick={this.handleNewClick}>{formatMessage({ id: 'form.button.new' })}</Button>
</div>
)

2
web/src/pages/DataManagement/Index.js
View File

@ -332,7 +332,7 @@ class Index extends PureComponent {
</div>
<Table bordered
dataSource={indices}
rowKey='id'
rowKey='index'
pagination={
{pageSize: 10,}
}

9
web/src/pages/DataManagement/models/document.js
View File

@ -95,6 +95,14 @@ export default {
let res = yield call(saveDoc, payload);
if(res.status === false){
message.warn("保存数据失败")
yield put({
type: 'saveData',
payload: {
editingKey: '',
isLoading: false,
_index: ''
}
})
return
}
encodeObjectField(doc);
@ -152,6 +160,7 @@ export default {
}
encodeObjectField(res.payload);
res.payload['_index'] = payload.index;
res.payload['_type'] = payload._type;
yield put({
type: '_addNew',
payload: {

2
web/src/pages/DataManagement/models/index.js
View File

@ -20,7 +20,7 @@ export default {
type: 'saveData',
payload: {
clusterIndices: resp.payload,
cluster: payload.cluster,
// cluster: payload.cluster,
}
})
},

8
web/src/services/cluster.js Normal file
View File

@ -0,0 +1,8 @@
import request from '@/utils/request';
import {pathPrefix} from './common';
export async function getClusterVersion(params) {
return request(`${pathPrefix}/cluster/${params.cluster}/version`, {
method: 'GET'
});
}

18
web/src/services/doc.js
View File

@ -13,14 +13,22 @@ export async function getDocList(params) {
}
export async function saveDoc(params) {
return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
let url = `${pathPrefix}/doc/${params.index}/${params.data.id}`;
if(params._type){
url += `?_type=${params._type}`;
}
return request(url, {
method: 'PUT',
body: params.data,
});
}
export async function deleteDoc(params) {
return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
let url =`${pathPrefix}/doc/${params.index}/${params.data.id}`;
if(params._type){
url += `?_type=${params._type}`;
}
return request(url, {
method: 'DELETE',
});
}
@ -28,7 +36,11 @@ export async function deleteDoc(params) {
export async function addDoc(params) {
let id = params.data.id || '';
delete(params.data, 'id');
return request(`${pathPrefix}/doc/${params.index}/_create`, {
let url = `${pathPrefix}/doc/${params.index}/_create`;
if(params._type){
url += `?_type=${params._type}`;
}
return request(url, {
method: 'POST',
body: params.data,
});

102
web/src/utils/elasticsearch.js
View File

@ -16,11 +16,19 @@ export function getFields(index, mappings){
}
filterMappings[index] = mappings[index];
}
let fields = [];
for(let key in filterMappings){
for(let fi in filterMappings[key].mappings.properties){
fields.push(fi);
if(filterMappings[key].mappings.properties) {
for (let fi in filterMappings[key].mappings.properties) {
fields.push(fi);
}
}else{
Object.keys(filterMappings[key].mappings).forEach((typ)=>{
for (let fi in filterMappings[key].mappings[typ].properties) {
fields.indexOf(fi) === - 1 && fields.push(fi);
}
})
}
}
@ -29,22 +37,98 @@ export function getFields(index, mappings){
export function formatESSearchResult(esResp) {
const total = esResp.hits.total
if(total.value == 0){
if(total == null || total.value == 0){
return {
total: total,
data: [],
};
}
let dataArr = [];
for(let hit of esResp.hits.hits) {
if(!hit._source.id){
hit._source["id"] = hit._id
if(esResp.hits.hits) {
for (let hit of esResp.hits.hits) {
if (!hit._source.id) {
hit._source["id"] = hit._id
}
hit._source["_index"] = hit._index
if(hit["_type"]){
hit._source["_type"] = hit["_type"]
}
dataArr.push(hit._source)
}
hit._source["_index"] = hit._index
dataArr.push(hit._source)
}
return {
total: total,
data: dataArr,
}
}
const ESAPIV0 = {
getProperties(params){
const {index, mappings} = params;
if(typeof mappings[index] === 'undefined'){
return {};
}
return mappings[index].mappings.properties;
},
extractIndicesFromMappings(mappings){
if(!mappings){
return [];
}
return Object.keys(mappings).map(index=>{return{
index: index
}});
}
}
const ESAPIV2 = {
...ESAPIV0,
getProperties(params){
const {index, mappings, typ} = params;
if(typeof mappings[index] === 'undefined'){
return {};
}
let targetMappings = mappings[index].mappings;
if(typeof targetMappings[typ] === 'undefined'){
return {}
}
return targetMappings[typ].properties;
},
extractIndicesFromMappings(mappings){
let indices = [];
for(let index in mappings){
indices.push({
index: index,
types: Object.keys(mappings[index].mappings).map(typ => { return typ})
});
}
return indices;
}
}
const ESAPIV5 = {
...ESAPIV2
}
const ESAPIV6 = {
...ESAPIV5,
typeName: "_doc",
getProperties(params){
params.typ = this.typeName;
return ESAPIV5.getProperties(params);
}
}
const ESAPI = {
"0": ESAPIV0,
"2": ESAPIV2,
"5": ESAPIV5,
"6": ESAPIV6,
}
export function getESAPI(ver){
if(typeof ESAPI[ver] != "undefined"){
return ESAPI[ver];
}
return ESAPI["0"];
}