modify document management for es 2.x

api/index_management/cluster.go

@@ -0,0 +1,17 @@
+package index_management
+
+import (
+	"net/http"
+	httprouter "infini.sh/framework/core/api/router"
+	"infini.sh/framework/core/elastic"
+)
+
+func (handler APIHandler) GetClusterVersion(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	client := elastic.GetClient(handler.Config.Elasticsearch)
+	ver := client.GetMajorVersion()
+	resBody := newResponseBody()
+	resBody["payload"] = map[string]int{
+		"major": ver,
+	}
+	handler.WriteJSON(w, resBody, http.StatusOK)
+}

api/index_management/document.go

@@ -36,7 +36,8 @@ func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *ht
 	if strings.Trim(id, "/") == "" {
 		id = util.GetUUID()
 	}
-	_, err = client.Index(indexName, id, reqBody)
+	docType := handler.GetParameter(req, "_type")
+	_, err = client.Index(indexName, docType, id, reqBody)
 	if err != nil {
 		resResult["status"] = false
 		resResult["error"] = err
@@ -61,7 +62,8 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req
 	}
 	indexName := ps.ByName("index")
 	id := ps.ByName("id")
-	resp, err := client.Get(indexName, id)
+	typ := handler.GetParameter(req, "_type")
+	resp, err := client.Get(indexName,typ, id)
 	if err != nil {
 		resResult["status"] = false
 		resResult["error"] = err.Error()
@@ -70,9 +72,12 @@ func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req
 	}
 	source := resp.Source
 	for k, v := range reqBody {
+		if k == "id" {
+			continue
+		}
 		source[k] = v
 	}
-	_, err = client.Index(indexName, id, source)
+	_, err = client.Index(indexName, typ, id, source)
 	if err != nil {
 		resResult["status"] = false
 		resResult["error"] = err.Error()
@@ -88,7 +93,8 @@ func (handler APIHandler) HandleDeleteDocumentAction(w http.ResponseWriter, req
 	resResult := newResponseBody()
 	indexName := ps.ByName("index")
 	id := ps.ByName("id")
-	_, err := client.Delete(indexName, id)
+	typ := handler.GetParameter(req, "_type")
+	_, err := client.Delete(indexName, typ, id)
 	if err != nil {
 		resResult["error"] = err.Error()
 		resResult["status"] = false

api/index_management/indices.go

@@ -1,7 +1,6 @@
 package index_management
 
 import (
-	"fmt"
 	"net/http"
 	"strings"
 
@@ -111,11 +110,6 @@ func (handler APIHandler) HandleDeleteIndexAction(w http.ResponseWriter, req *ht
 }
 
 func (handler APIHandler) HandleCreateIndexAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-	defer func() {
-		if err := recover(); err != nil {
-			fmt.Println(err)
-		}
-	}()
 	client := elastic.GetClient(handler.Config.Elasticsearch)
 	indexName := ps.ByName("index")
 	resBody := newResponseBody()

api/index_management/rebuild.go

@@ -31,8 +31,8 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
 	}
 
 	//fmt.Println(reindexItem)
-
+	typ := handler.GetParameter(req, "_type")
-	ID, err := reindex(handler.Config.Elasticsearch, reindexItem)
+	ID, err := reindex(handler.Config.Elasticsearch, reindexItem, typ)
 	if err != nil {
 		resResult["error"] = err
 		resResult["status"] = false
@@ -43,7 +43,7 @@ func (handler APIHandler) HandleReindexAction(w http.ResponseWriter, req *http.R
 	handler.WriteJSON(w, resResult, http.StatusOK)
 }
 
-func reindex(esName string, body *model.Reindex) (string, error) {
+func reindex(esName string, body *model.Reindex, typ string) (string, error) {
 	client := elastic.GetClient(esName)
 	source := map[string]interface{}{
 		"index": body.Source.Index,
@@ -77,7 +77,7 @@ func reindex(esName string, body *model.Reindex) (string, error) {
 	body.Status = model.ReindexStatusRunning
 	body.CreatedAt = time.Now()
 
-	_, err = client.Index(orm.GetIndexName(body), body.ID, body)
+	_, err = client.Index(orm.GetIndexName(body), typ, body.ID, body)
 	if err != nil {
 		return "", err
 	}
@@ -171,7 +171,7 @@ func SyncRebuildResult(esName string) error {
 		}
 		source["status"] = status
 		source["task_source"] = doc.Source
-		_, err := client.Index(orm.GetIndexName(model.Reindex{}), esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
+		_, err := client.Index(orm.GetIndexName(model.Reindex{}), "", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
 		return err
 	}
 	return nil

api/init.go

@@ -36,6 +36,8 @@ func Init(cfg *config.AppConfig) {
 	ui.HandleUIMethod(api.DELETE, pathPrefix+"index/:index", handler.HandleDeleteIndexAction)
 	ui.HandleUIMethod(api.POST, pathPrefix+"index/:index", handler.HandleCreateIndexAction)
 
+	ui.HandleUIMethod(api.GET, pathPrefix+"cluster/:cluster/version", handler.GetClusterVersion)
+
 	task.RegisterScheduleTask(task.ScheduleTask{
 		Description: "sync reindex task result to index infinireindex",
 		Task: func() {

web/mock/datamanagement/document.js

@@ -12,29 +12,29 @@ function getUUID(len){
 }
 
 export default {
-  'post /_search-center/doc/:index/_search': function(req, res){
+  // 'post /_search-center/doc/:index/_search': function(req, res){
-    res.send(queryData)
+  //   res.send(queryData)
-  },
+  // },
-  'post /_search-center/doc/:index/_create': function(req, res){
+  // 'post /_search-center/doc/:index/_create': function(req, res){
-    res.send({
+  //   res.send({
-      status: true,
+  //     status: true,
-      payload: {
+  //     payload: {
-        ...req.body.payload,
+  //       ...req.body.payload,
-        id: getUUID(),
+  //       id: getUUID(),
-      }
+  //     }
-    });
+  //   });
-  },
+  // },
-  'put /_search-center/doc/:index/:id': function(req, res){
+  // 'put /_search-center/doc/:index/:id': function(req, res){
-    res.send({
+  //   res.send({
-      status: true,
+  //     status: true,
-      payload: req.body
+  //     payload: req.body
-    });
+  //   });
-  },
+  // },
-
+  //
-  'delete /_search-center/doc/:index/:id': function(req, res){
+  // 'delete /_search-center/doc/:index/:id': function(req, res){
-    res.send({
+  //   res.send({
-      status: true,
+  //     status: true,
-      payload: null,
+  //     payload: null,
-    });
+  //   });
-  }
+  // }
 }

web/mock/datamanagement/rebuild.js

@@ -527,7 +527,7 @@ let data = {
 };
 
 export default {
-  'get /_search-center/rebuild/_search': function(req, res){
+  // 'get /_search-center/rebuild/_search': function(req, res){
-    res.send(data)
+  //   res.send(data)
-  }
+  // }
 }

web/src/layouts/BasicLayout.js

@@ -102,6 +102,12 @@ class BasicLayout extends React.PureComponent {
     dispatch({
       type: 'setting/getSetting',
     });
+    dispatch({
+      type: 'cluster/fetchClusterVersion',
+      payload: {
+        cluster: 'single-es'
+      }
+    });
     this.renderRef = requestAnimationFrame(() => {
       this.setState({
         rendering: false,

web/src/models/cluster.js

@@ -0,0 +1,26 @@
+import { getClusterVersion } from '@/services/cluster';
+
+export default {
+    namespace: 'cluster',
+    state: {
+    },
+
+    effects: {
+        *fetchClusterVersion({ payload }, { call, put }) {
+            let res = yield call(getClusterVersion, payload);
+            yield put({
+                type: 'saveData',
+                payload: res.payload
+            })
+        }
+    },
+
+    reducers: {
+        saveData(state, {payload}){
+            return {
+                ...state,
+                ...payload,
+            }
+        }
+    },
+};

web/src/pages/DataManagement/Document.js

@@ -3,12 +3,12 @@ import { formatMessage, FormattedMessage } from 'umi/locale';
 import router from 'umi/router';
 import { connect } from 'dva';
 import { Col, Form, Row,Select, Input, Card,Icon, Table, InputNumber, Popconfirm,
-   Divider,Button,Tooltip, Modal, DatePicker, message } from 'antd';
+   Divider,Button,Tooltip, Modal, DatePicker, message,Cascader } from 'antd';
 import Editor, {monaco} from '@monaco-editor/react';
 import moment from 'moment';
 import {createDependencyProposals} from './autocomplete';
 import InputSelect from '@/components/infini/InputSelect';
-import {getFields} from '@/utils/elasticsearch';
+import {getFields,getESAPI} from '@/utils/elasticsearch';
 
 function findParentIdentifier(textUntilPosition){
   let chars = textUntilPosition;
@@ -60,15 +60,22 @@ function findParentIdentifier(textUntilPosition){
 //     },
 //   },
 // });
-monaco.init().then((mi)=>{
+var langDisposer = null;
-  mi.languages.onLanguage("json", ()=>{
+function initEditor() {
-    mi.languages.registerCompletionItemProvider('json', {
+    monaco.init().then((mi) => {
+        //mi.languages.onLanguage("json", () => {
+            langDisposer = mi.languages.registerCompletionItemProvider('json', {
                 triggerCharacters: ['"'],
-      provideCompletionItems: function(model, position, ctx) { 
+                provideCompletionItems: function (model, position, ctx) {
-          var textUntilPosition = model.getValueInRange({startLineNumber: 1, startColumn: 1, endLineNumber: position.lineNumber, endColumn: position.column});
+                    var textUntilPosition = model.getValueInRange({
+                        startLineNumber: 1,
+                        startColumn: 1,
+                        endLineNumber: position.lineNumber,
+                        endColumn: position.column
+                    });
 
-          if(textUntilPosition.indexOf('{') < 0){
+                    if (textUntilPosition.indexOf('{') < 0) {
-            return { suggestions: [] };
+                        return {suggestions: []};
                     }
 
                     let key = findParentIdentifier(textUntilPosition);
@@ -87,9 +94,10 @@ monaco.init().then((mi)=>{
                     };
                 }
             });
-  })
+       // })
 
-})
+    })
+}
 
 const {Option} = Select;
 
@@ -202,7 +210,14 @@ class EditableCell extends React.Component {
       //   console.log(record, doclist.mappings)
       //   return
       // }
-      const {properties} = doclist.mappings[record._index].mappings;
+      let properties = null;
+      let _type = record._type || doclist._type;
+      if(typeof _type !== 'undefined' && _type !== '' && _type !== '_doc'){
+        properties = doclist.mappings[record._index].mappings[_type].properties;
+      }else{
+        properties = doclist.mappings[record._index].mappings.properties;
+      }
+
       if(!properties[key]){
         return '';
       }
@@ -236,6 +251,7 @@ class EditableCell extends React.Component {
             type: 'document/saveDocItem',
             payload: {
               index: doclist._index,
+              _type: doclist._type,
               data: {
                 id: key,
                 ...row,
@@ -247,6 +263,7 @@ class EditableCell extends React.Component {
             type: 'document/addDocItem',
             payload: {
               index: doclist._index,
+              _type: doclist._type,
               data: row,
             }
           })
@@ -258,7 +275,7 @@ class EditableCell extends React.Component {
       const {dispatch} = this.props;
       dispatch({
         type: 'document/saveData',
-        payload: { editingKey: record.id, _index: record._index }
+        payload: { editingKey: record.id, _index: record._index, _type: record._type }
       });
     }
 
@@ -268,6 +285,7 @@ class EditableCell extends React.Component {
         type: 'document/deleteDocItem',
         payload: {
           index: record._index,
+          _type: record._type,
           data: {
             id: record.id,
           }
@@ -336,6 +354,10 @@ class EditableCell extends React.Component {
           cell: EditableCell,
         },
       };
+      let total = doclist.total || 0;
+      if(total.value){
+        total = total.value;
+      }
       return (
         <EditableContext.Provider value={this.props.form}>
           <Table
@@ -350,7 +372,7 @@ class EditableCell extends React.Component {
             rowClassName="editable-row"
             pagination={{
               showSizeChanger: true,
-              total: doclist.total?  doclist.total.value: 0,
+              total: total,
               pageSize: doclist.pageSize,
               current: doclist.pageIndex,
               showTotal: (total, range) => `Total ${total} items`,
@@ -362,8 +384,9 @@ class EditableCell extends React.Component {
     }
   }
 
-@connect(({document})=>({
+@connect(({document,cluster})=>({
-  document
+  document,
+  cluster,
 }))
 @Form.create()
 class Doucment extends React.Component {
@@ -386,8 +409,13 @@ class Doucment extends React.Component {
   handleEditorDidMount = (_valueGetter) =>{
     this.filterGetter = _valueGetter;
   }
-  
+  componentWillUnmount() {
+      if(langDisposer != null) {
+          langDisposer.dispose();
+      }
+  }
   componentDidMount(){
+      initEditor()
     const {location, dispatch } = this.props;
     //console.log(match, location);
     let index = location.query.index;
@@ -421,19 +449,23 @@ class Doucment extends React.Component {
   }
   
   handleNewClick = ()=>{
-    const {dispatch, document} = this.props;
+    const {dispatch, document,cluster} = this.props;
     if(document.isAddNew){ //!document.data || document.data.length == 0 
       return;
     }
-    let {indices, mappings} = document;
+    let {mappings, indices} = document;
-    if(!indices){
+    if(indices.length === 0) {
+      indices = Object.keys(mappings);
+    }
+    if(indices.length === 0){
         return
     }
     let _index = indices[0];
-    if(indices && indices.length > 1){
+    let _type = '';
+    if(indices.length > 0){
       //console.log(this.indexSelEl);
-      let vals = this.indexSelEl.rcSelect.state.value;
+      let vals = this.indexSelEl.state.value;
-      if(vals.length == 0){
+      if(vals.length === 0){
         Modal.error({
           title: '系统提示',
           content: '请选择新建文档目标索引',
@@ -441,13 +473,20 @@ class Doucment extends React.Component {
         return
       }else{
         _index = vals[0];
+        if(vals.length>1){
+          _type = vals[1];
         }
       }
-    let properties = mappings[_index].mappings.properties;
+    }
+    let properties = getESAPI(cluster.major).getProperties({
+      index: _index,
+      mappings: mappings,
+      typ: _type,
+    }); //mappings[_index].mappings.properties;
     let keys = Object.keys(properties)
-    let newDoc = {id:"", _index};
+    let newDoc = {id:"", _index,_type};
     for(let key of keys){
-      if(properties[key].type == 'date'){
+      if(properties[key].type === 'date'){
         newDoc[key] = null
       }else{
         newDoc[key] = ""
@@ -459,7 +498,8 @@ class Doucment extends React.Component {
         docItem: newDoc,
         extra: {
           isAddNew: true,
-          _index
+          _index,
+          _type,
         }
       },
     })
@@ -495,17 +535,43 @@ class Doucment extends React.Component {
   }
 
   renderNew = ()=>{
-    const {indices} = this.props.document;
+    let {indices, mappings} = this.props.document;
     // if((indices && indices.length > 1)){
     //   return;
     // }
+    const {major} = this.props.cluster;
+    if(indices && indices.length >= 0){
+      indices = getESAPI(major).extractIndicesFromMappings(mappings).filter(item=>{
+        if(indices.length > 0){
+          return indices.indexOf(item.index) > -1;
+        }
+        return true;
+      }).map(item=>{
+
+        let newItem= {
+          label: item.index,
+          value: item.index,
+        };
+        if(item.types){
+          newItem.children = item.types.map(typ=>{
+            return {
+              label: typ,
+              value: typ,
+            }
+          })
+        }
+        return newItem;
+      })
+    }
     return (
       <div>
-        {(indices && indices.length > 1) ? (<Select ref={el=>{this.indexSelEl=el}} style={{width: 200, marginRight:5}} placeholder="please select a index">
+        {(indices) ? (<Cascader ref={el=>{this.indexSelEl=el}} options={indices} style={{width: 200, marginRight:5}} placeholder="please select a index">
-          {indices.map(item=>{
+        </Cascader>) : ''}
-            return (<Select.Option key={item} label={item}>{item}</Select.Option>)
+        {/*{(indices) ? (<Select ref={el=>{this.indexSelEl=el}} style={{width: 200, marginRight:5}} placeholder="please select a index">*/}
-          })}
+        {/*  {indices.map(item=>{*/}
-        </Select>) : ''}
+        {/*    return (<Select.Option key={item} label={item}>{item}</Select.Option>)*/}
+        {/*  })}*/}
+        {/*</Select>) : ''}*/}
         <Button type="primary" icon="plus" onClick={this.handleNewClick}>{formatMessage({ id: 'form.button.new' })}</Button>
       </div>
     )

web/src/pages/DataManagement/Index.js

@@ -332,7 +332,7 @@ class Index extends PureComponent {
             </div>
             <Table bordered
               dataSource={indices}
-              rowKey='id'
+              rowKey='index'
               pagination={
                 {pageSize: 10,}
               }

web/src/pages/DataManagement/models/document.js

@@ -95,6 +95,14 @@ export default {
       let res = yield call(saveDoc, payload);
       if(res.status === false){
         message.warn("保存数据失败")
+        yield put({
+          type: 'saveData',
+          payload: {
+            editingKey: '',
+            isLoading: false,
+            _index: ''
+          }
+        })
         return
       }
       encodeObjectField(doc);
@@ -152,6 +160,7 @@ export default {
       }
       encodeObjectField(res.payload);
       res.payload['_index'] = payload.index;
+      res.payload['_type'] = payload._type;
       yield put({
         type: '_addNew',
         payload: {

web/src/pages/DataManagement/models/index.js

@@ -20,7 +20,7 @@ export default {
                 type: 'saveData',
                 payload: {
                     clusterIndices: resp.payload,
-                    cluster: payload.cluster,
+                  //  cluster: payload.cluster,
                 }
             })
         },

web/src/services/cluster.js

@@ -0,0 +1,8 @@
+import request from '@/utils/request';
+import {pathPrefix} from './common';
+
+export async function getClusterVersion(params) {
+    return request(`${pathPrefix}/cluster/${params.cluster}/version`, {
+        method: 'GET'
+    });
+}

web/src/services/doc.js

@@ -13,14 +13,22 @@ export async function getDocList(params) {
 }
 
 export async function saveDoc(params) {
-  return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
+  let url =  `${pathPrefix}/doc/${params.index}/${params.data.id}`;
+  if(params._type){
+    url += `?_type=${params._type}`;
+  }
+  return request(url, {
     method: 'PUT',
     body: params.data,
   });
 }
 
 export async function deleteDoc(params) {
-  return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
+  let url =`${pathPrefix}/doc/${params.index}/${params.data.id}`;
+  if(params._type){
+    url += `?_type=${params._type}`;
+  }
+  return request(url, {
     method: 'DELETE',
   });
 }
@@ -28,7 +36,11 @@ export async function deleteDoc(params) {
 export async function addDoc(params) {
   let id = params.data.id || '';
   delete(params.data, 'id');
-  return request(`${pathPrefix}/doc/${params.index}/_create`, {
+  let url = `${pathPrefix}/doc/${params.index}/_create`;
+  if(params._type){
+    url += `?_type=${params._type}`;
+  }
+  return request(url, {
     method: 'POST',
     body: params.data,
   });

web/src/utils/elasticsearch.js

@@ -19,9 +19,17 @@ export function getFields(index, mappings){
 
   let fields = [];
   for(let key in filterMappings){
-    for(let fi in filterMappings[key].mappings.properties){
+    if(filterMappings[key].mappings.properties) {
+      for (let fi in filterMappings[key].mappings.properties) {
         fields.push(fi);
       }
+    }else{
+      Object.keys(filterMappings[key].mappings).forEach((typ)=>{
+        for (let fi in filterMappings[key].mappings[typ].properties) {
+          fields.indexOf(fi) === - 1 && fields.push(fi);
+        }
+      })
+    }
   }
 
   return fields;
@@ -29,22 +37,98 @@ export function getFields(index, mappings){
 
 export function formatESSearchResult(esResp) {
   const total = esResp.hits.total
-  if(total.value == 0){
+  if(total == null || total.value == 0){
     return {
       total: total,
       data: [],
     };
   }
   let dataArr = [];
-  for(let hit of esResp.hits.hits) {
+  if(esResp.hits.hits) {
-    if(!hit._source.id){
+    for (let hit of esResp.hits.hits) {
+      if (!hit._source.id) {
         hit._source["id"] = hit._id
       }
       hit._source["_index"] = hit._index
+      if(hit["_type"]){
+        hit._source["_type"] = hit["_type"]
+      }
       dataArr.push(hit._source)
     }
+  }
   return {
     total: total,
     data: dataArr,
   }
 }
+
+const ESAPIV0 = {
+  getProperties(params){
+    const {index, mappings} = params;
+    if(typeof mappings[index] === 'undefined'){
+      return {};
+    }
+    return mappings[index].mappings.properties;
+  },
+  extractIndicesFromMappings(mappings){
+    if(!mappings){
+      return [];
+    }
+    return Object.keys(mappings).map(index=>{return{
+      index: index
+    }});
+  }
+}
+
+const ESAPIV2 = {
+  ...ESAPIV0,
+  getProperties(params){
+    const {index, mappings, typ} = params;
+    if(typeof mappings[index] === 'undefined'){
+      return {};
+    }
+    let targetMappings = mappings[index].mappings;
+    if(typeof targetMappings[typ] === 'undefined'){
+      return {}
+    }
+    return targetMappings[typ].properties;
+  },
+  extractIndicesFromMappings(mappings){
+    let indices = [];
+    for(let index in mappings){
+      indices.push({
+        index: index,
+        types: Object.keys(mappings[index].mappings).map(typ => { return typ})
+      });
+    }
+    return indices;
+  }
+}
+
+
+const ESAPIV5 = {
+  ...ESAPIV2
+}
+
+const ESAPIV6 = {
+  ...ESAPIV5,
+  typeName: "_doc",
+  getProperties(params){
+    params.typ = this.typeName;
+    return ESAPIV5.getProperties(params);
+  }
+}
+
+const ESAPI = {
+  "0": ESAPIV0,
+  "2": ESAPIV2,
+  "5": ESAPIV5,
+  "6": ESAPIV6,
+}
+
+export function getESAPI(ver){
+  if(typeof ESAPI[ver] != "undefined"){
+    return ESAPI[ver];
+  }
+  return ESAPI["0"];
+}