floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite index name

This commit is contained in:
silenceqi 2021-01-09 17:01:17 +08:00
parent 7f8c3051e2
commit 843237db60
18 changed files with 788 additions and 46165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/index_management/document.go
View File

@ -34,7 +34,7 @@ func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *ht
indexName := ps.ByName("index")
id := ps.ByName("id")
if strings.Trim(id, "/") == "" {
util.GetUUID()
id = util.GetUUID()
}
_, err = client.Index(indexName, id, reqBody)
if err != nil {

30
api/index_management/index.go
View File

@ -33,7 +33,7 @@ func (handler APIHandler) GetDictListAction(w http.ResponseWriter, req *http.Req
if len(tags) > 3 {
tags = tags[0:3]
}
rel, err := model2.GetDictList(from, size, name, tags)
rel, err := model2.GetDictList(from, size, name, tags, handler.Config.Elasticsearch)
if err != nil {
resp["error"] = err
resp["status"] = false
@ -115,30 +115,4 @@ func (handler APIHandler) UpdateDictItemAction(w http.ResponseWriter, req *http.
resp["payload"] = dict
handler.WriteJSON(w, resp, http.StatusOK)
}
// TaskAction handle task creation and return task list which support parameter: `from`, `size` and `host`, eg:
//curl -XGET http://127.0.0.1:8001/task?from=100&size=10&host=elasticsearch.cn
func (handler APIHandler) TaskAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
//fr := handler.GetParameter(req, "from")
//si := handler.GetParameter(req, "size")
//host := handler.GetParameter(req, "host")
//status := handler.GetIntOrDefault(req, "status", -1)
//
//from, err := strconv.Atoi(fr)
//if err != nil {
// from = 0
//}
//size, err := strconv.Atoi(si)
//if err != nil {
// size = 10
//}
//
//orm.Search()
//total, tasks, err := model.GetTaskList(from, size, host, status)
//if err != nil {
// handler.Error(w, err)
//} else {
// handler.WriteJSONListResult(w, total, tasks, http.StatusOK)
}
}

7
api/index_management/indices.go
View File

@ -26,7 +26,7 @@ func (handler APIHandler) HandleGetMappingsAction(w http.ResponseWriter, req *ht
}
if copyAll {
for key, _ := range *idxs {
if strings.HasPrefix(key, ".") {
if strings.HasPrefix(key, ".") || strings.HasPrefix(key, "infini-") {
delete(*idxs, key)
}
}
@ -40,6 +40,11 @@ func (handler APIHandler) HandleGetMappingsAction(w http.ResponseWriter, req *ht
func (handler APIHandler) HandleGetIndicesAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
catIndices, err := client.GetIndices()
for key, _ := range *catIndices {
if strings.HasPrefix(key,".") || strings.HasPrefix(key, "infini-"){
delete(*catIndices, key)
}
}
resBody := newResponseBody()
if err != nil {
resBody["status"] = false

9
api/index_management/rebuild.go
View File

@ -3,6 +3,7 @@ package index_management
import (
"encoding/json"
"fmt"
"infini.sh/framework/core/orm"
"net/http"
"strings"
"time"
@ -76,7 +77,7 @@ func reindex(esName string, body *model.Reindex) (string, error) {
body.Status = model.ReindexStatusRunning
body.CreatedAt = time.Now()
_, err = client.Index("infinireindex", body.ID, body)
_, err = client.Index(orm.GetIndexName(body), body.ID, body)
if err != nil {
return "", err
}
@ -139,7 +140,7 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req
func SyncRebuildResult(esName string) error {
client := elastic.GetClient(esName)
esBody := fmt.Sprintf(`{"query":{"match":{"status": "%s"}}}`, model.ReindexStatusRunning)
esRes, err := client.SearchWithRawQueryDSL("infinireindex", []byte(esBody))
esRes, err := client.SearchWithRawQueryDSL(orm.GetIndexName(model.Reindex{}), []byte(esBody))
if err != nil {
return err
}
@ -170,7 +171,7 @@ func SyncRebuildResult(esName string) error {
}
source["status"] = status
source["task_source"] = doc.Source
_, err := client.Index("infinireindex", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
_, err := client.Index(orm.GetIndexName(model.Reindex{}), esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
return err
}
return nil
@ -197,7 +198,7 @@ func buildTermsQuery(fieldName string, terms []string) string {
func deleteTasksByIds(esName string, terms []string) error {
client := elastic.GetClient(esName)
esBody := buildTermsQuery("_id", terms)
deleteRes, err := client.DeleteByQuery("infinireindex", []byte(esBody))
deleteRes, err := client.DeleteByQuery(orm.GetIndexName(model.Reindex{}), []byte(esBody))
if err != nil {
return err
}

4
main.go
View File

@ -69,8 +69,8 @@ func main() {
module.Start()
}, func() {
orm.RegisterSchema(model.Dict{})
orm.RegisterSchema(model.Reindex{})
orm.RegisterSchemaWithIndexName(model.Dict{}, "infini-dict")
orm.RegisterSchemaWithIndexName(model.Reindex{}, "infini-reindex")
})
}

19
model/dict.go
View File

@ -2,6 +2,7 @@ package model
import (
"fmt"
"infini.sh/framework/core/elastic"
"strings"
"time"
@ -18,7 +19,7 @@ type Dict struct {
UpdatedAt time.Time `json:"updated_at" elastic_mapping:"updated_at:{type:date}"`
}
func GetDictList(from, size int, name string, tags []string) (orm.Result, error) {
func GetDictList(from, size int, name string, tags []string, esName string) (*elastic.SearchResponse, error) {
//sort := []orm.Sort{}
//sort = append(sort, orm.Sort{Field: "created_at", SortType: orm.DESC})
var (
@ -56,14 +57,10 @@ func GetDictList(from, size int, name string, tags []string) (orm.Result, error)
}
query = fmt.Sprintf(query, must, should, minShould)
rq := fmt.Sprintf(`{"from":%d, "size":%d, "sort": %s, "query": %s}`, from, size, sort, query)
//fmt.Println(rq)
q := &orm.Query{
//From: from,
//Size: size,
//Sort: &sort,
RawQuery: []byte(rq),
}
//var dictList = []Dict{}
err, sr := orm.Search(Dict{}, nil, q)
return sr, err
//q := &orm.Query{
// RawQuery: []byte(rq),
//}
//err, sr := orm.Search(Dict{}, nil, q)
client := elastic.GetClient(esName)
return client.SearchWithRawQueryDSL(orm.GetIndexName(Dict{}), []byte(rq))
}

3
model/reindex.go
View File

@ -2,6 +2,7 @@ package model
import (
"fmt"
"infini.sh/framework/core/orm"
"strings"
"time"
@ -59,5 +60,5 @@ func GetRebuildList(esName string, from, size int, name string) (*elastic.Search
query = fmt.Sprintf(query, must)
rq := fmt.Sprintf(`{"from":%d, "size":%d, "sort": %s, "query": %s}`, from, size, sort, query)
client := elastic.GetClient(esName)
return client.SearchWithRawQueryDSL("infinireindex", []byte(rq))
return client.SearchWithRawQueryDSL(orm.GetIndexName(Reindex{}), []byte(rq))
}

50
web/mock/datamanagement/data/doc.js
View File

@ -5,12 +5,12 @@ export const queryData = {
"hits": {
"total": {
"relation": "eq",
"value": 11
"value": 12
},
"max_score": 1,
"hits": [
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "jc6_jXYBKoaaPbVfj_8W",
"_source": {
@ -23,7 +23,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvhm18dath2d6oa9046g",
"_source": {
@ -35,7 +35,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvhlv6dath2d6oa9045g",
"_source": {
@ -47,7 +47,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvhltpdath2d6oa90450",
"_source": {
@ -59,20 +59,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_type": "_doc",
"_id": "js7EjXYBKoaaPbVfvf-c",
"_source": {
"address": "hunan zhuzhou",
"created_at": "2020-12-23T03:59:57.620Z",
"email": "cincky@qq.com",
"hobbies": "[\"basketball\",\"badminton1\"]",
"id": "js7EjXYBKoaaPbVfvf-c",
"name": "cincky"
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvi5ellath2e0ukbq5e0",
"_source": {
@ -87,7 +74,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvia41lath2eneoeeij0",
"_source": {
@ -101,7 +88,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvi5omtath2e0ukbq5eg",
"_source": {
@ -116,7 +103,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvhlsptath2d6oa9044g",
"_source": {
@ -131,7 +118,7 @@ export const queryData = {
}
},
{
"_index": "infini-test",
"_index": "test-custom",
"_type": "_doc",
"_id": "bvhm0d5ath2d6oa90460",
"_source": {
@ -145,6 +132,23 @@ export const queryData = {
"id": "bvhm0d5ath2d6oa90460",
"name": "hello2"
}
},
{
"_index": "test-custom",
"_type": "_doc",
"_id": "bvia4ctath2eneoeeijg",
"_source": {
"address": "beijing",
"age": 31,
"created_at": "2020-12-23T03:57:57.620Z",
"email": "786027438@qq.com",
"hobbies": [
"basketball1",
"badminton"
],
"id": "bvia4ctath2eneoeeijg",
"name": "北京"
}
}
]
}

2
web/mock/datamanagement/document.js
View File

@ -12,7 +12,7 @@ function getUUID(len){
}
export default {
'get /_search-center/doc/:index/_search': function(req, res){
'post /_search-center/doc/:index/_search': function(req, res){
res.send(queryData)
},
'post /_search-center/doc/:index/_create': function(req, res){

46625
web/mock/datamanagement/indices.js
View File

File diff suppressed because it is too large Load Diff

136
web/mock/search/dict.js
View File

File diff suppressed because one or more lines are too long

6
web/src/pages/DataManagement/Document.js
View File

@ -513,10 +513,10 @@ class Doucment extends React.Component {
//console.log(this.props.document);
let clusterIndices = this.props.document.clusterIndices || [];
clusterIndices = clusterIndices.filter(index => !index.index.startsWith('.')).map((index) =>{
clusterIndices = Object.keys(clusterIndices).map((index) =>{
return {
label: index.index,
value: index.index,
label: index,
value: index,
};
})
const clusters = ["single-es"];

2
web/src/pages/DataManagement/models/document.js
View File

@ -1,6 +1,6 @@
import {getDocList, saveDoc, deleteDoc, addDoc} from '@/services/doc';
import {getMappings, getIndices} from '@/services/indices';
import {formatESSearchResult} from '@/utils/utils';
import {formatESSearchResult} from '@/utils/elasticsearch';
import { message } from 'antd';
function encodeObjectField(doc){

2
web/src/pages/DataManagement/models/rebuildlist.js
View File

@ -1,6 +1,6 @@
import {getRebuildList,reindex, deleteRebuild} from '@/services/rebuild';
import { message } from 'antd';
import {formatESSearchResult} from '@/utils/utils';
import {formatESSearchResult} from '@/utils/elasticsearch';
const delay = (ms) => new Promise((resolve) => {

9
web/src/pages/SearchManage/models/dict.js
View File

@ -1,5 +1,6 @@
import { message } from 'antd';
import { getDictList, addDict, deleteDict,updateDict } from '@/services/search';
import {formatESSearchResult} from '@/utils/elasticsearch';
const utf8 = {
encode: function (string) {
@ -67,7 +68,9 @@ export default {
message.error(resp.error)
return
}
resp.payload.Result = resp.payload.Result.map((item)=>{
resp.payload = formatESSearchResult(resp.payload);
console.log(resp.payload);
resp.payload.data = resp.payload.data.map((item)=>{
item.content = utf8.decode(atob(item.content))
return item;
})
@ -78,8 +81,8 @@ export default {
yield put({
type: 'saveData',
payload: {
dictList: resp.payload.Result,
total: resp.payload.Total,
dictList: resp.payload.data,
total: resp.payload.total.value,
search: search,
},
});

3
web/src/services/doc.js
View File

@ -18,14 +18,13 @@ export async function saveDoc(params) {
export async function deleteDoc(params) {
return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
method: 'DELETE',
body: ['testid1', 'testid2'],
});
}
export async function addDoc(params) {
let id = params.data.id || '';
delete(params.data, 'id');
return request(`${pathPrefix}/doc/${params.index}/${id}`, {
return request(`${pathPrefix}/doc/${params.index}/_create`, {
method: 'POST',
body: params.data,
});

22
web/src/utils/elasticsearch.js
View File

@ -26,3 +26,25 @@ export function getFields(index, mappings){
return fields;
}
export function formatESSearchResult(esResp) {
const total = esResp.hits.total
if(total.value == 0){
return {
total: total,
data: [],
};
}
let dataArr = [];
for(let hit of esResp.hits.hits) {
if(!hit._source.id){
hit._source["id"] = hit._id
}
hit._source["_index"] = hit._index
dataArr.push(hit._source)
}
return {
total: total,
data: dataArr,
}
}

22
web/src/utils/utils.js
View File

@ -180,26 +180,4 @@ export function formatWan(val) {
export function isAntdPro() {
return window.location.hostname === 'preview.pro.ant.design';
}
export function formatESSearchResult(esResp) {
const total = esResp.hits.total
if(esResp.hits.hits.length == 0){
return {
total: total,
data: [],
};
}
let dataArr = [];
for(let hit of esResp.hits.hits) {
if(!hit._source.id){
hit._source["id"] = hit._id
}
hit._source["_index"] = hit._index
dataArr.push(hit._source)
}
return {
total: total,
data: dataArr,
}
}