modify rebuild and doc

This commit is contained in:
silenceqi 2021-01-06 22:53:25 +08:00
parent 33c78d2278
commit f63a70126d
17 changed files with 46608 additions and 232 deletions

View File

@ -12,26 +12,19 @@ import (
)
type docReqBody struct {
Index string `json:"index"`
Action string `json:"action"`
Payload map[string]interface{} `json:"payload"`
PageIndex int `json:"pageIndex"`
PageSize int `json:"pageSize"`
Filter string `json:"filter"`
Cluster string `json:"cluster"`
Keyword string `json:"keyword"`
Sort string `json:"sort"`
SortDirection string `json:"sort_direction"`
PageIndex int `json:"pageIndex"`
PageSize int `json:"pageSize"`
Filter string `json:"filter"`
Cluster string `json:"cluster"`
Keyword string `json:"keyword"`
Sort string `json:"sort"`
SortDirection string `json:"sort_direction"`
}
func (handler APIHandler) HandleDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
reqBody := docReqBody{}
resResult := map[string]interface{}{
"errno": "0",
"errmsg": "",
"payload": nil,
}
reqBody := map[string]interface{}{}
resResult := newResponseBody()
err := handler.DecodeJSON(req, &reqBody)
if err != nil {
resResult["errno"] = "E10001"
@ -40,106 +33,129 @@ func (handler APIHandler) HandleDocumentAction(w http.ResponseWriter, req *http.
return
}
indexName := ps.ByName("index")
var id string
if val, ok := reqBody.Payload["id"]; ok {
id = val.(string)
id := util.GetUUID()
_, err = client.Index(indexName, id, reqBody)
if err != nil {
resResult["errno"] = "E10002"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
if _, ok := reqBody.Payload["_index"]; ok {
delete(reqBody.Payload, "_index")
reqBody["id"] = id
resResult["payload"] = reqBody
handler.WriteJSON(w, resResult, http.StatusOK)
}
func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
reqBody := map[string]interface{}{}
resResult := newResponseBody()
err := handler.DecodeJSON(req, &reqBody)
if err != nil {
resResult["errno"] = "E10001"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
switch reqBody.Action {
case "ADD":
id = util.GetUUID()
//security problem
_, err := client.Index(indexName, id, reqBody.Payload)
if err != nil {
resResult["errno"] = "E10002"
resResult["errmsg"] = err.Error()
break
}
reqBody.Payload["id"] = id
resResult["payload"] = reqBody.Payload
case "SAVE":
if id == "" {
resResult["errno"] = "E10003"
resResult["errmsg"] = "empty id"
break
}
resp, err := client.Get(indexName, id)
if err != nil {
resResult["errno"] = "E10004"
resResult["errmsg"] = err.Error()
break
}
source := resp.Source
for k, v := range reqBody.Payload {
source[k] = v
}
_, err = client.Index(indexName, id, source)
if err != nil {
resResult["errno"] = "E10005"
resResult["errmsg"] = err.Error()
break
}
case "DELETE":
if id == "" {
panic("empty id")
}
_, err = client.Delete(indexName, id)
if err != nil {
resResult["errmsg"] = err.Error()
resResult["errno"] = "E10006"
break
}
default:
var (
pageSize = 10
pageIndex = 1
sort = ""
)
if reqBody.PageSize > 0 {
pageSize = reqBody.PageSize
}
if reqBody.PageIndex > 0 {
pageIndex = reqBody.PageIndex
}
from := (pageIndex - 1) * pageSize
filter := `{"match_all": {}}`
if reqBody.Keyword != "" {
filter = fmt.Sprintf(`{"query_string":{"query":"%s"}}`, reqBody.Keyword)
}
if reqBody.Filter != "" {
filter = reqBody.Filter
}
if sortField := strings.Trim(reqBody.Sort, " "); sortField != "" {
sortDirection := reqBody.SortDirection
if sortDirection != "desc" {
sortDirection = "asc"
}
sort = fmt.Sprintf(`"%s":{"order":"%s"}`, sortField, sortDirection)
}
query := fmt.Sprintf(`{"from":%d, "size": %d, "query": %s, "sort": [{%s}]}`, from, pageSize, filter, sort)
//fmt.Println(indexName, query)
var reqBytes = []byte(query)
resp, err := client.SearchWithRawQueryDSL(indexName, reqBytes)
if err != nil {
resResult["errno"] = "E10007"
resResult["errmsg"] = err.Error()
break
}
result := formatESSearchResult(resp)
_, _, idxs, err := client.GetMapping(false, indexName)
if err != nil {
resResult["errno"] = "E10008"
resResult["errmsg"] = err.Error()
break
}
result["mappings"] = idxs
resResult["payload"] = result
indexName := ps.ByName("index")
id := ps.ByName("id")
resp, err := client.Get(indexName, id)
if err != nil {
resResult["errno"] = "E10004"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
source := resp.Source
for k, v := range reqBody {
source[k] = v
}
_, err = client.Index(indexName, id, source)
if err != nil {
resResult["errno"] = "E10005"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
resResult["payload"] = reqBody
handler.WriteJSON(w, resResult, http.StatusOK)
}
func (handler APIHandler) HandleDeleteDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
resResult := newResponseBody()
indexName := ps.ByName("index")
id := ps.ByName("id")
_, err := client.Delete(indexName, id)
if err != nil {
resResult["errmsg"] = err.Error()
resResult["errno"] = "E10006"
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
resResult["payload"] = true
handler.WriteJSON(w, resResult, http.StatusOK)
}
func (handler APIHandler) HandleSearchDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
client := elastic.GetClient(handler.Config.Elasticsearch)
reqBody := docReqBody{}
resResult := newResponseBody()
err := handler.DecodeJSON(req, &reqBody)
if err != nil {
resResult["errno"] = "E10001"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
indexName := ps.ByName("index")
var (
pageSize = 10
pageIndex = 1
sort = ""
)
if reqBody.PageSize > 0 {
pageSize = reqBody.PageSize
}
if reqBody.PageIndex > 0 {
pageIndex = reqBody.PageIndex
}
from := (pageIndex - 1) * pageSize
filter := `{"match_all": {}}`
if reqBody.Keyword != "" {
filter = fmt.Sprintf(`{"query_string":{"query":"%s"}}`, reqBody.Keyword)
}
if reqBody.Filter != "" {
filter = reqBody.Filter
}
if sortField := strings.Trim(reqBody.Sort, " "); sortField != "" {
sortDirection := reqBody.SortDirection
if sortDirection != "desc" {
sortDirection = "asc"
}
sort = fmt.Sprintf(`"%s":{"order":"%s"}`, sortField, sortDirection)
}
query := fmt.Sprintf(`{"from":%d, "size": %d, "query": %s, "sort": [{%s}]}`, from, pageSize, filter, sort)
//fmt.Println(indexName, query)
var reqBytes = []byte(query)
resp, err := client.SearchWithRawQueryDSL(indexName, reqBytes)
if err != nil {
resResult["errno"] = "E10007"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
result := formatESSearchResult(resp)
_, _, idxs, err := client.GetMapping(false, indexName)
if err != nil {
resResult["errno"] = "E10008"
resResult["errmsg"] = err.Error()
handler.WriteJSON(w, resResult, http.StatusOK)
return
}
result["mappings"] = idxs
resResult["payload"] = result
handler.WriteJSON(w, resResult, http.StatusOK)
}

View File

@ -28,7 +28,7 @@ func (handler APIHandler) ReindexAction(w http.ResponseWriter, req *http.Request
return
}
fmt.Println(reindexItem)
//fmt.Println(reindexItem)
taskID, err := reindex(handler.Config.Elasticsearch, reindexItem)
if err != nil {
@ -79,6 +79,7 @@ func reindex(esName string, body *model.InfiniReindex) (string, error) {
}
body.ID = util.GetUUID()
body.TaskId = resBody.Task
body.Status = model.ReindexStatusRunning
body.CreatedAt = time.Now()
_, err = client.Index("infinireindex", body.ID, body)
@ -136,60 +137,56 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req
handler.WriteJSON(w, resBody, http.StatusOK)
return
}
var ids = []string{}
idMap := map[string]int{}
for idx, doc := range esResp.Hits.Hits {
taskId := doc.Source["task_id"].(string)
ids = append(ids, taskId)
idMap[taskId] = idx
}
taskResp, err := getTasksByTerms(esName, ids)
err = SyncRebuildResult(esName)
if err != nil {
resBody["errno"] = "E20004"
resBody["errmsg"] = err.Error()
handler.WriteJSON(w, resBody, http.StatusOK)
return
}
var (
completed bool
status string
esErrStr string
tookTime int
)
for _, doc := range taskResp.Hits.Hits {
status = "RUNNING"
tookTime = 0
esErrStr = ""
completed = doc.Source["completed"].(bool)
source := esResp.Hits.Hits[idMap[doc.ID.(string)]].Source
if esErr, ok := doc.Source["error"]; ok {
status = "FAILED"
if errMap, ok := esErr.(map[string]interface{}); ok {
esErrStr = errMap["reason"].(string)
}
} else {
if resMap, ok := doc.Source["response"].(map[string]interface{}); ok {
tookTime = int(resMap["took"].(float64))
}
status = "SUCCESS"
}
if !completed {
status = "RUNNING"
}
source["status"] = status
source["error"] = esErrStr
source["took_time"] = tookTime
}
resBody["payload"] = formatESSearchResult(esResp)
handler.WriteJSON(w, resBody, http.StatusOK)
}
func getTasksByTerms(esName string, terms []string) (*elastic.SearchResponse, error) {
if len(terms) == 0 {
return nil, nil
}
func SyncRebuildResult(esName string) error {
client := elastic.GetClient(esName)
esBody := buildTermsQuery(terms)
return client.SearchWithRawQueryDSL(".tasks", []byte(esBody))
esBody := `{"query":{"match":{"status": "RUNNING"}}}`
esRes, err := client.SearchWithRawQueryDSL("infinireindex", []byte(esBody))
if err != nil {
return err
}
var ids = []string{}
idMap := map[string]int{}
for idx, doc := range esRes.Hits.Hits {
taskId := doc.Source["task_id"].(string)
ids = append(ids, taskId)
idMap[taskId] = idx
}
if len(ids) == 0 {
return nil
}
taskResp, err := client.SearchTasksByIds(ids)
if err != nil {
return err
}
var (
status model.ReindexStatus
)
for _, doc := range taskResp.Hits.Hits {
status = model.ReindexStatusRunning
source := esRes.Hits.Hits[idMap[doc.ID.(string)]].Source
if _, ok := doc.Source["error"]; ok {
status = model.ReindexStatusFailed
} else {
status = model.ReindexStatusSuccess
}
source["status"] = status
source["task_source"] = doc.Source
_, err := client.Index("infinireindex", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
return err
}
return nil
}
func buildTermsQuery(terms []string) string {

View File

@ -1,7 +1,10 @@
package api
import (
"log"
"infini.sh/framework/core/api"
"infini.sh/framework/core/task"
"infini.sh/framework/core/ui"
"infini.sh/search-center/api/index_management"
"infini.sh/search-center/config"
@ -19,11 +22,24 @@ func Init(cfg *config.AppConfig) {
ui.HandleUIMethod(api.DELETE, pathPrefix+"dict/:id", handler.DeleteDictItemAction)
//ui.HandleUIMethod(api.DELETE, "/api/dict/", handler.DeleteDictItemAction2)
ui.HandleUIMethod(api.POST, pathPrefix+"dict/_update", handler.UpdateDictItemAction)
ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index", handler.HandleDocumentAction)
ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index/_search", handler.HandleSearchDocumentAction)
ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index/_create", handler.HandleAddDocumentAction)
ui.HandleUIMethod(api.PUT, pathPrefix+"doc/:index/:id", handler.HandleUpdateDocumentAction)
ui.HandleUIMethod(api.DELETE, pathPrefix+"doc/:index/:id", handler.HandleDeleteDocumentAction)
ui.HandleUIMethod(api.GET, pathPrefix+"indices/_cat", handler.HandleGetIndicesAction)
ui.HandleUIMethod(api.POST, pathPrefix+"rebuild/_create", handler.ReindexAction)
ui.HandleUIMethod(api.GET, pathPrefix+"rebuild/list", handler.HandleGetRebuildListAction)
ui.HandleUIMethod(api.POST, pathPrefix+"rebuild/_delete", handler.HandleDeleteRebuildAction)
ui.HandleUIMethod(api.GET, pathPrefix+"indices/_mappings/:index", handler.HandleGetMappingsAction)
task.RegisterScheduleTask(task.ScheduleTask{
Description: "sync reindex task result to index infinireindex",
Task: func() {
err := index_management.SyncRebuildResult(cfg.Elasticsearch)
if err != nil {
log.Println(err)
}
},
})
}

View File

@ -8,6 +8,14 @@ import (
"infini.sh/framework/core/elastic"
)
type ReindexStatus string
const (
ReindexStatusRunning ReindexStatus = "RUNNING"
ReindexStatusSuccess ReindexStatus = "SUCCEED"
ReindexStatusFailed ReindexStatus = "FAILED"
)
type InfiniReindex struct {
ID string `json:"id" elastic_meta:"_id"`
Name string `json:"name" elastic_mapping:"name:{type:text}"`
@ -24,8 +32,8 @@ type InfiniReindex struct {
Pipeline string `json:"pipeline"`
} `json:"dest" elastic_mapping:"dest:{type:object}"`
CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"`
Status string `json:"status" elastic_mapping:"status:{type:keyword}"`
CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"`
Status ReindexStatus `json:"status" elastic_mapping:"status:{type:keyword}"`
}
func GetRebuildList(esName string, from, size int, name string) (*elastic.SearchResponse, error) {

View File

@ -120,12 +120,12 @@ export default [
]
}, {
path: '/data/rebuild',
name: 'rebuild',
component: './DataManagement/Rebuild',
},{
path: '/data/rebuild/list',
name: 'rebuildlist',
component: './DataManagement/RebuildList',
},{
path: '/data/rebuild/new',
name: 'rebuild',
component: './DataManagement/Rebuild',
hideInMenu: true,
}, {
path: '/data/import',

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,432 @@
let data = {
"errmsg": "",
"errno": "0",
"payload": {
"data": [
{
"_index": "infinireindex",
"created_at": "2021-01-06T15:55:31.426604+08:00",
"desc": "test source index not exists",
"dest": {
"index": "infini-test8",
"pipeline": ""
},
"id": "bvqmpstath24tgfo107g",
"name": "test failed",
"source": {
"_source": [],
"index": "infini-testx",
"query": null
},
"status": "FAILED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:824925",
"task_source": {
"completed": true,
"error": {
"index": "infini-testx",
"index_uuid": "_na_",
"reason": "no such index [infini-testx]",
"resource.id": "infini-testx",
"resource.type": "index_or_alias",
"type": "index_not_found_exception"
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-testx] to [infini-test8][_doc]",
"headers": {},
"id": 824925,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 172714,
"start_time_in_millis": 1609919731425,
"status": {
"batches": 0,
"created": 0,
"deleted": 0,
"noops": 0,
"requests_per_second": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 0,
"updated": 0,
"version_conflicts": 0
},
"type": "transport"
}
}
},
{
"_index": "infinireindex",
"created_at": "2021-01-06T15:48:46.585112+08:00",
"desc": "test query param again",
"dest": {
"index": "infini-test8",
"pipeline": ""
},
"id": "bvqmmnlath24tgfo1070",
"name": "test query one",
"source": {
"_source": [],
"index": "infini-test",
"query": {
"match": {
"name": "cincky"
}
}
},
"status": "SUCCEED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:822567",
"task_source": {
"completed": true,
"response": {
"batches": 1,
"created": 0,
"deleted": 0,
"failures": [],
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled": "0s",
"throttled_millis": 0,
"throttled_until": "0s",
"throttled_until_millis": 0,
"timed_out": false,
"took": 44,
"total": 1,
"updated": 1,
"version_conflicts": 0
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-test] to [infini-test8][_doc]",
"headers": {},
"id": 822567,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 44277146,
"start_time_in_millis": 1609919326584,
"status": {
"batches": 1,
"created": 0,
"deleted": 0,
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 1,
"updated": 1,
"version_conflicts": 0
},
"type": "transport"
}
}
},
{
"_index": "infinireindex",
"created_at": "2021-01-06T15:46:04.745132+08:00",
"desc": "test query param",
"dest": {
"index": "infini-test8",
"pipeline": ""
},
"id": "bvqmlf5ath24tgfo106g",
"name": "test query",
"source": {
"_source": [],
"index": "infini-test1",
"query": {
"match": {
"name": "test"
}
}
},
"status": "SUCCEED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:821548",
"task_source": {
"completed": true,
"response": {
"batches": 0,
"created": 0,
"deleted": 0,
"failures": [],
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled": "0s",
"throttled_millis": 0,
"throttled_until": "0s",
"throttled_until_millis": 0,
"timed_out": false,
"took": 0,
"total": 0,
"updated": 0,
"version_conflicts": 0
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-test1] to [infini-test8][_doc]",
"headers": {},
"id": 821548,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 982379,
"start_time_in_millis": 1609919164744,
"status": {
"batches": 0,
"created": 0,
"deleted": 0,
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 0,
"updated": 0,
"version_conflicts": 0
},
"type": "transport"
}
}
},
{
"_index": "infinireindex",
"created_at": "2021-01-06T15:44:10.535851+08:00",
"desc": "test source param",
"dest": {
"index": "infini-test8",
"pipeline": ""
},
"id": "bvqmkilath24tgfo1060",
"name": "test source",
"source": {
"_source": [
"email",
"hobbies",
"created_at",
"name"
],
"index": "infini-test1",
"query": null
},
"status": "SUCCEED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:820833",
"task_source": {
"completed": true,
"response": {
"batches": 1,
"created": 0,
"deleted": 0,
"failures": [],
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled": "0s",
"throttled_millis": 0,
"throttled_until": "0s",
"throttled_until_millis": 0,
"timed_out": false,
"took": 53,
"total": 4,
"updated": 4,
"version_conflicts": 0
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-test1] to [infini-test8][_doc]",
"headers": {},
"id": 820833,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 53101238,
"start_time_in_millis": 1609919050535,
"status": {
"batches": 1,
"created": 0,
"deleted": 0,
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 4,
"updated": 4,
"version_conflicts": 0
},
"type": "transport"
}
}
},
{
"_index": "infinireindex",
"created_at": "2021-01-06T15:41:08.159721+08:00",
"desc": "test pipeline param",
"dest": {
"index": "infini-test8",
"pipeline": "test"
},
"id": "bvqmj55ath24tgfo105g",
"name": "test pipeline",
"source": {
"_source": [],
"index": "infini-test1",
"query": null
},
"status": "SUCCEED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:819744",
"task_source": {
"completed": true,
"response": {
"batches": 1,
"created": 0,
"deleted": 0,
"failures": [],
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled": "0s",
"throttled_millis": 0,
"throttled_until": "0s",
"throttled_until_millis": 0,
"timed_out": false,
"took": 251,
"total": 4,
"updated": 4,
"version_conflicts": 0
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-test1] to [infini-test8][_doc]",
"headers": {},
"id": 819744,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 251872120,
"start_time_in_millis": 1609918868159,
"status": {
"batches": 1,
"created": 0,
"deleted": 0,
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 4,
"updated": 4,
"version_conflicts": 0
},
"type": "transport"
}
}
},
{
"_index": "infinireindex",
"created_at": "2021-01-05T23:25:24.751473+08:00",
"desc": "test new rebuild api",
"dest": {
"index": "infini-test8",
"pipeline": ""
},
"id": "bvq89p5ath243f63hngg",
"name": "test new rebuild api",
"source": {
"_source": [],
"index": "infini-test",
"query": null
},
"status": "SUCCEED",
"task_id": "F0D6OfeVSzuMhf5528ANTw:707730",
"task_source": {
"completed": true,
"response": {
"batches": 1,
"created": 11,
"deleted": 0,
"failures": [],
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled": "0s",
"throttled_millis": 0,
"throttled_until": "0s",
"throttled_until_millis": 0,
"timed_out": false,
"took": 117,
"total": 11,
"updated": 0,
"version_conflicts": 0
},
"task": {
"action": "indices:data/write/reindex",
"cancellable": true,
"description": "reindex from [infini-test] to [infini-test8][_doc]",
"headers": {},
"id": 707730,
"node": "F0D6OfeVSzuMhf5528ANTw",
"running_time_in_nanos": 118081190,
"start_time_in_millis": 1609860324750,
"status": {
"batches": 1,
"created": 11,
"deleted": 0,
"noops": 0,
"requests_per_second": -1,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"throttled_until_millis": 0,
"total": 11,
"updated": 0,
"version_conflicts": 0
},
"type": "transport"
}
}
}
],
"total": {
"relation": "eq",
"value": 6
}
}
};
export default {
'get /_search-center/rebuild/list': function(req, res){
res.send(data)
}
}

View File

@ -96,7 +96,8 @@ export default {
'menu.data.pipes': 'PIPELINE',
'menu.data.index': 'INDEX',
'menu.data.snapshot': 'SNAPSHOT',
'menu.data.rebuild': 'REBUILD',
'menu.data.rebuildlist': 'REBUILD',
'menu.data.rebuild': 'NEW REBUILD',
'menu.data.export': 'IMPORT&EXPORT',
'menu.data.query': 'DOCUMENT',

View File

@ -104,7 +104,8 @@ export default {
'menu.data.pipes': '数据加工',
'menu.data.index': '索引管理',
'menu.data.snapshot': '快照管理',
'menu.data.rebuild': '重建管理',
'menu.data.rebuildlist': '重建管理',
'menu.data.rebuild': '新建重建',
'menu.data.export': '导入导出',
'menu.data.query': '文档管理',

View File

@ -22,9 +22,7 @@ const operationTabList = [
];
@connect(({logstash,loading }) => ({
data: logstash.logstash,
loading: loading.models.logstash,
submitting: loading.effects['logstash/submitLogstashConfig'],
}))
@Form.create()
@ -70,7 +68,7 @@ class Import extends Component {
render() {
const { operationkey } = this.state;
const { submitting, data, loading } = this.props;
const { submitting, data } = this.props;
const {
form: { getFieldDecorator, getFieldValue },
} = this.props;
@ -114,7 +112,7 @@ class Import extends Component {
<Form onSubmit={this.handleSubmit} hideRequiredMark style={{ marginTop: 8 }}>
<FormItem {...formItemLayout} label="选择索引">
{getFieldDecorator('index', {
initialValue: data.index,
initialValue: '',
rules: [
{
required: true,
@ -131,7 +129,7 @@ class Import extends Component {
</FormItem>
<FormItem {...formItemLayout} label="_bulk数据">
{getFieldDecorator('bulk', {
initialValue: data.bulk,
initialValue: '',
rules: [
{
required: true,

View File

@ -46,7 +46,10 @@ class Rebuild extends Component {
}
}
}else{
filterMappings[index] = mappings[index] || {};
if(!mappings[index]){
return [];
}
filterMappings[index] = mappings[index];
}
let fields = [];
@ -279,32 +282,32 @@ class Rebuild extends Component {
<Button key="back" onClick={()=>this.backward(currentStep)}>
上一步
</Button>,
<Button key="cancel" style={{margin:'auto 2em'}} onClick={() =>{}}>
取消
</Button>,
<Button key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
// <Button key="cancel" style={{margin:'auto 2em'}} onClick={() =>{}}>
// 取消
// </Button>,
<Button key="forward" style={{ marginLeft: '2em' }} type="primary" onClick={() => this.handleNext(currentStep)}>
下一步
</Button>,
];
}
if (currentStep === 2) {
return [
<Button key="back" style={{ float: 'left' }} onClick={()=>this.backward(currentStep)}>
<Button key="back" onClick={()=>this.backward(currentStep)}>
上一步
</Button>,
<Button key="cancel" style={{margin:'auto 2em'}} onClick={() => {}}>
取消
</Button>,
<Button key="submit" type="primary" onClick={() =>this.handleNext(currentStep)}>
// <Button key="cancel" style={{margin:'auto 2em'}} onClick={() => {}}>
// 取消
// </Button>,
<Button key="submit" style={{ marginLeft: '2em' }} type="primary" onClick={() =>this.handleNext(currentStep)}>
完成
</Button>,
];
}
return [
<Button key="cancel" onClick={() => {}}>
取消
</Button>,
<Button style={{marginLeft:'2em'}} key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
// <Button key="cancel" onClick={() => {}}>
// 取消
// </Button>,
<Button key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
下一步
</Button>,
];

View File

@ -2,6 +2,7 @@ import { Card, Table, Form, Row, Input, Col, Button, Divider, Tooltip,Popconfirm
import Link from 'umi/link';
import React from 'react';
import {connect} from 'dva'
import moment from 'moment';
@connect(({rebuildlist}) => ({
rebuildlist,
@ -22,41 +23,52 @@ class RebuildList extends React.Component {
})
}
columns = [{
title: 'id',
dataIndex: 'id',
key: 'id',
},{
title: 'rebuild name',
dataIndex: 'name',
key: 'name',
},{
title: 'description',
dataIndex: 'desc',
key: 'desc',
},{
title: 'status',
dataIndex: 'status',
key: 'status',
render: (text, record) => (
<span style={{color: text== 'SUCCESS' ? 'green': (text=='FAILED' ? 'red': 'blue')}}>
{text == 'FAILED'? <Tooltip placeholder="top" title={record.error}>{text}</Tooltip> : text}
<span style={{color: text== 'SUCCEED' ? 'green': (text=='FAILED' ? 'red': 'blue')}}>
{text == 'FAILED'? <Tooltip placeholder="top" title={record.task_source.error.reason}>{text}</Tooltip> : text}
</span>
),
},{
title: 'took_time',
dataIndex: 'took_time',
key: 'took_time',
render: (_, record)=>(record.task_source &&record.task_source.response ?record.task_source.response.took: -1)
},{
title: 'total',
dataIndex: 'total',
key: 'total',
render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.total: 0)
},{
title: 'updated',
dataIndex: 'updated',
key: 'updated',
render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.updated: 0)
},{
title: 'created',
dataIndex: 'created',
key: 'created',
render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.created: 0)
},{
title: 'created_at',
dataIndex: 'created_at',
key: 'created_at',
render: (text) =>(<span>{moment(text).format('YYYY-MM-DD HH:mm:ss')}</span>)
},{
title: 'Operation',
render: (text, record) => (
<div>
<Popconfirm title="Sure to delete?" onConfirm={() => this.handleDeleteClick(record)}><a key="delete">Delete</a>
</Popconfirm>
{record.status=='FAILED' ? <span><Divider type="vertical" /><a key="redo" onClick={()=>this.handleRedoClick(record)}>Redo</a></span>
{record.status=='FAILED' ? <span><Divider type="vertical" />
<Popconfirm title="Sure to redo?" onConfirm={() => this.handleRedoClick(record)}><a key="redo">Redo</a></Popconfirm>
</span>
: ''}
</div>
@ -100,6 +112,15 @@ class RebuildList extends React.Component {
name: nameVal,
})
}
handleRefreshClick = ()=>{
const {rebuildlist} = this.props;
this.fetchData({
pageIndex: rebuildlist.pageIndex,
pageSize: rebuildlist.pageSize,
name: rebuildlist.name,
})
}
render(){
const {getFieldDecorator} = this.props.form;
@ -120,7 +141,7 @@ class RebuildList extends React.Component {
</Col>
<Col md={8} sm={8}>
<div style={{paddingTop:4}}>
<Button type="primary" onClick={this.handleSearch}>
<Button type="primary" icon="search" onClick={this.handleSearch}>
Search
</Button>
</div>
@ -130,13 +151,19 @@ class RebuildList extends React.Component {
<Divider style={{marginBottom:0}} />
<Card
bodyStyle={{padding:0}}
extra={<Link to="/data/rebuild"> <Button type="primary" icon="plus">New</Button></Link>}
extra={<div><Button onClick={this.handleRefreshClick} icon="redo">Refresh</Button><Link to="/data/rebuild/new"> <Button type="primary" icon="plus">New</Button></Link></div>}
bordered={false}>
<Table columns={this.columns}
loading={rebuildlist.isLoading}
bordered
rowKey="id"
expandedRowRender={record => <div>
<Row>
<Col span={24}>
<span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>description: </span>
<span>{record.desc}</span>
</Col>
</Row>
<Row>
<Col span={12}><span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>source</span><pre>{JSON.stringify(record.source, null, 2)}</pre></Col>
<Col span={12}><span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>dest</span><pre>{JSON.stringify(record.dest, null, 2)}</pre></Col>

View File

@ -32,7 +32,7 @@ class Indices extends Component {
},
{
key: 'doc',
tab: '索引文档管理',
tab: '索引管理',
},
{
key: 'template',

View File

@ -115,6 +115,9 @@ export default {
}
});
let res = yield call(deleteDoc, payload);
if(typeof res == 'string'){
res = JSON.parse(res);
}
if(res.errno != "0"){
message.warn("删除数据失败")
return
@ -152,6 +155,7 @@ export default {
docItem: res.payload,
extra: {
isLoading: false,
isAddNew: false,
}
}
})

View File

@ -15,7 +15,6 @@ export default {
effects:{
*addTask({payload}, {call, put}){
let resp = yield call(reindex, payload);
console.log(resp);
if(resp.errno != "0"){
message.warn("rebuild failed")
return
@ -34,7 +33,6 @@ export default {
},
*fetchMappings({payload}, {call, put}){
let resp = yield call(getMappings, payload);
console.log(resp);
if(resp.errno != "0"){
message.warn("get mappings failed")
return

View File

@ -88,14 +88,17 @@ export default {
message.error("delete failed")
return
}
yield call(delay, 1000);
let {pageIndex, pageSize, name} = select(state=>state.rebuildlist);
let {data, total} = yield select(state=>state.rebuildlist);
let newData = data.filter(item=> !payload.includes(item.id));
yield put({
type: 'fetchRebuildList',
type: 'saveData',
payload: {
pageIndex: pageIndex,
pageSize: pageSize,
name: name
data: newData,
total: {
...total,
value: total.value - payload.length,
},
isLoading: false,
}
})
}

View File

@ -2,7 +2,7 @@ import request from '@/utils/request';
import {pathPrefix} from './common';
export async function getDocList(params) {
return request(`${pathPrefix}/doc/${params.index}`, {
return request(`${pathPrefix}/doc/${params.index}/_search`, {
method: 'POST',
body: {
action: 'SEARCH',
@ -12,35 +12,23 @@ export async function getDocList(params) {
}
export async function saveDoc(params) {
return request(`${pathPrefix}/doc/${params.index}`, {
method: 'POST',
body: {
payload: params.data,
index: params.index,
action: 'SAVE',
},
return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
method: 'PUT',
body: params.data,
});
}
export async function deleteDoc(params) {
return request(`${pathPrefix}/doc/${params.index}`, {
method: 'POST',
body: {
index: params.index,
action: 'DELETE',
payload: params.data,
},
return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
method: 'DELETE',
});
}
export async function addDoc(params) {
return request(`${pathPrefix}/doc/${params.index}`, {
delete(params.data, 'id');
return request(`${pathPrefix}/doc/${params.index}/_create`, {
method: 'POST',
body: {
payload: params.data,
index: params.index,
action: 'ADD',
},
body: params.data,
});
}