modify rebuild and doc

api/index_management/document.go

@@ -12,26 +12,19 @@ import (
 )
 
 type docReqBody struct {
-	Index         string                 `json:"index"`
+	PageIndex     int    `json:"pageIndex"`
-	Action        string                 `json:"action"`
+	PageSize      int    `json:"pageSize"`
-	Payload       map[string]interface{} `json:"payload"`
+	Filter        string `json:"filter"`
-	PageIndex     int                    `json:"pageIndex"`
+	Cluster       string `json:"cluster"`
-	PageSize      int                    `json:"pageSize"`
+	Keyword       string `json:"keyword"`
-	Filter        string                 `json:"filter"`
+	Sort          string `json:"sort"`
-	Cluster       string                 `json:"cluster"`
+	SortDirection string `json:"sort_direction"`
-	Keyword       string                 `json:"keyword"`
-	Sort          string                 `json:"sort"`
-	SortDirection string                 `json:"sort_direction"`
 }
 
-func (handler APIHandler) HandleDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+func (handler APIHandler) HandleAddDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 	client := elastic.GetClient(handler.Config.Elasticsearch)
-	reqBody := docReqBody{}
+	reqBody := map[string]interface{}{}
-	resResult := map[string]interface{}{
+	resResult := newResponseBody()
-		"errno":   "0",
-		"errmsg":  "",
-		"payload": nil,
-	}
 	err := handler.DecodeJSON(req, &reqBody)
 	if err != nil {
 		resResult["errno"] = "E10001"
@@ -40,106 +33,129 @@ func (handler APIHandler) HandleDocumentAction(w http.ResponseWriter, req *http.
 		return
 	}
 	indexName := ps.ByName("index")
-	var id string
+	id := util.GetUUID()
-	if val, ok := reqBody.Payload["id"]; ok {
+	_, err = client.Index(indexName, id, reqBody)
-		id = val.(string)
+	if err != nil {
+		resResult["errno"] = "E10002"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
 	}
-	if _, ok := reqBody.Payload["_index"]; ok {
+	reqBody["id"] = id
-		delete(reqBody.Payload, "_index")
+	resResult["payload"] = reqBody
+	handler.WriteJSON(w, resResult, http.StatusOK)
+}
+
+func (handler APIHandler) HandleUpdateDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	client := elastic.GetClient(handler.Config.Elasticsearch)
+	reqBody := map[string]interface{}{}
+	resResult := newResponseBody()
+	err := handler.DecodeJSON(req, &reqBody)
+	if err != nil {
+		resResult["errno"] = "E10001"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
 	}
-	switch reqBody.Action {
+	indexName := ps.ByName("index")
-	case "ADD":
+	id := ps.ByName("id")
-		id = util.GetUUID()
+	resp, err := client.Get(indexName, id)
-		//security problem
+	if err != nil {
-		_, err := client.Index(indexName, id, reqBody.Payload)
+		resResult["errno"] = "E10004"
-		if err != nil {
+		resResult["errmsg"] = err.Error()
-			resResult["errno"] = "E10002"
+		handler.WriteJSON(w, resResult, http.StatusOK)
-			resResult["errmsg"] = err.Error()
+		return
-			break
-		}
-		reqBody.Payload["id"] = id
-		resResult["payload"] = reqBody.Payload
-	case "SAVE":
-		if id == "" {
-			resResult["errno"] = "E10003"
-			resResult["errmsg"] = "empty id"
-			break
-		}
-		resp, err := client.Get(indexName, id)
-		if err != nil {
-			resResult["errno"] = "E10004"
-			resResult["errmsg"] = err.Error()
-			break
-		}
-		source := resp.Source
-		for k, v := range reqBody.Payload {
-			source[k] = v
-		}
-		_, err = client.Index(indexName, id, source)
-		if err != nil {
-			resResult["errno"] = "E10005"
-			resResult["errmsg"] = err.Error()
-			break
-		}
-
-	case "DELETE":
-		if id == "" {
-			panic("empty id")
-		}
-		_, err = client.Delete(indexName, id)
-		if err != nil {
-			resResult["errmsg"] = err.Error()
-			resResult["errno"] = "E10006"
-			break
-		}
-	default:
-		var (
-			pageSize  = 10
-			pageIndex = 1
-			sort      = ""
-		)
-		if reqBody.PageSize > 0 {
-			pageSize = reqBody.PageSize
-		}
-		if reqBody.PageIndex > 0 {
-			pageIndex = reqBody.PageIndex
-		}
-		from := (pageIndex - 1) * pageSize
-		filter := `{"match_all": {}}`
-		if reqBody.Keyword != "" {
-			filter = fmt.Sprintf(`{"query_string":{"query":"%s"}}`, reqBody.Keyword)
-		}
-		if reqBody.Filter != "" {
-			filter = reqBody.Filter
-		}
-		if sortField := strings.Trim(reqBody.Sort, " "); sortField != "" {
-			sortDirection := reqBody.SortDirection
-			if sortDirection != "desc" {
-				sortDirection = "asc"
-			}
-			sort = fmt.Sprintf(`"%s":{"order":"%s"}`, sortField, sortDirection)
-		}
-		query := fmt.Sprintf(`{"from":%d, "size": %d, "query": %s, "sort": [{%s}]}`, from, pageSize, filter, sort)
-		//fmt.Println(indexName, query)
-		var reqBytes = []byte(query)
-		resp, err := client.SearchWithRawQueryDSL(indexName, reqBytes)
-		if err != nil {
-			resResult["errno"] = "E10007"
-			resResult["errmsg"] = err.Error()
-			break
-		}
-		result := formatESSearchResult(resp)
-
-		_, _, idxs, err := client.GetMapping(false, indexName)
-		if err != nil {
-			resResult["errno"] = "E10008"
-			resResult["errmsg"] = err.Error()
-			break
-		}
-		result["mappings"] = idxs
-		resResult["payload"] = result
-
 	}
+	source := resp.Source
+	for k, v := range reqBody {
+		source[k] = v
+	}
+	_, err = client.Index(indexName, id, source)
+	if err != nil {
+		resResult["errno"] = "E10005"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
+	}
+	resResult["payload"] = reqBody
+	handler.WriteJSON(w, resResult, http.StatusOK)
+}
+
+func (handler APIHandler) HandleDeleteDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	client := elastic.GetClient(handler.Config.Elasticsearch)
+	resResult := newResponseBody()
+	indexName := ps.ByName("index")
+	id := ps.ByName("id")
+	_, err := client.Delete(indexName, id)
+	if err != nil {
+		resResult["errmsg"] = err.Error()
+		resResult["errno"] = "E10006"
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
+	}
+	resResult["payload"] = true
+	handler.WriteJSON(w, resResult, http.StatusOK)
+}
+func (handler APIHandler) HandleSearchDocumentAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+	client := elastic.GetClient(handler.Config.Elasticsearch)
+	reqBody := docReqBody{}
+	resResult := newResponseBody()
+	err := handler.DecodeJSON(req, &reqBody)
+	if err != nil {
+		resResult["errno"] = "E10001"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
+	}
+	indexName := ps.ByName("index")
+	var (
+		pageSize  = 10
+		pageIndex = 1
+		sort      = ""
+	)
+	if reqBody.PageSize > 0 {
+		pageSize = reqBody.PageSize
+	}
+	if reqBody.PageIndex > 0 {
+		pageIndex = reqBody.PageIndex
+	}
+	from := (pageIndex - 1) * pageSize
+	filter := `{"match_all": {}}`
+	if reqBody.Keyword != "" {
+		filter = fmt.Sprintf(`{"query_string":{"query":"%s"}}`, reqBody.Keyword)
+	}
+	if reqBody.Filter != "" {
+		filter = reqBody.Filter
+	}
+	if sortField := strings.Trim(reqBody.Sort, " "); sortField != "" {
+		sortDirection := reqBody.SortDirection
+		if sortDirection != "desc" {
+			sortDirection = "asc"
+		}
+		sort = fmt.Sprintf(`"%s":{"order":"%s"}`, sortField, sortDirection)
+	}
+	query := fmt.Sprintf(`{"from":%d, "size": %d, "query": %s, "sort": [{%s}]}`, from, pageSize, filter, sort)
+	//fmt.Println(indexName, query)
+	var reqBytes = []byte(query)
+	resp, err := client.SearchWithRawQueryDSL(indexName, reqBytes)
+	if err != nil {
+		resResult["errno"] = "E10007"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
+	}
+	result := formatESSearchResult(resp)
+
+	_, _, idxs, err := client.GetMapping(false, indexName)
+	if err != nil {
+		resResult["errno"] = "E10008"
+		resResult["errmsg"] = err.Error()
+		handler.WriteJSON(w, resResult, http.StatusOK)
+		return
+	}
+	result["mappings"] = idxs
+	resResult["payload"] = result
+
 	handler.WriteJSON(w, resResult, http.StatusOK)
 }
 

api/index_management/rebuild.go

@@ -28,7 +28,7 @@ func (handler APIHandler) ReindexAction(w http.ResponseWriter, req *http.Request
 		return
 	}
 
-	fmt.Println(reindexItem)
+	//fmt.Println(reindexItem)
 
 	taskID, err := reindex(handler.Config.Elasticsearch, reindexItem)
 	if err != nil {
@@ -79,6 +79,7 @@ func reindex(esName string, body *model.InfiniReindex) (string, error) {
 	}
 	body.ID = util.GetUUID()
 	body.TaskId = resBody.Task
+	body.Status = model.ReindexStatusRunning
 	body.CreatedAt = time.Now()
 
 	_, err = client.Index("infinireindex", body.ID, body)
@@ -136,60 +137,56 @@ func (handler APIHandler) HandleGetRebuildListAction(w http.ResponseWriter, req
 		handler.WriteJSON(w, resBody, http.StatusOK)
 		return
 	}
-
+	err = SyncRebuildResult(esName)
-	var ids = []string{}
-	idMap := map[string]int{}
-	for idx, doc := range esResp.Hits.Hits {
-		taskId := doc.Source["task_id"].(string)
-		ids = append(ids, taskId)
-		idMap[taskId] = idx
-	}
-	taskResp, err := getTasksByTerms(esName, ids)
 	if err != nil {
 		resBody["errno"] = "E20004"
 		resBody["errmsg"] = err.Error()
+		handler.WriteJSON(w, resBody, http.StatusOK)
+		return
 	}
-	var (
+
-		completed bool
-		status    string
-		esErrStr  string
-		tookTime  int
-	)
-	for _, doc := range taskResp.Hits.Hits {
-		status = "RUNNING"
-		tookTime = 0
-		esErrStr = ""
-		completed = doc.Source["completed"].(bool)
-		source := esResp.Hits.Hits[idMap[doc.ID.(string)]].Source
-		if esErr, ok := doc.Source["error"]; ok {
-			status = "FAILED"
-			if errMap, ok := esErr.(map[string]interface{}); ok {
-				esErrStr = errMap["reason"].(string)
-			}
-		} else {
-			if resMap, ok := doc.Source["response"].(map[string]interface{}); ok {
-				tookTime = int(resMap["took"].(float64))
-			}
-			status = "SUCCESS"
-		}
-		if !completed {
-			status = "RUNNING"
-		}
-		source["status"] = status
-		source["error"] = esErrStr
-		source["took_time"] = tookTime
-	}
 	resBody["payload"] = formatESSearchResult(esResp)
 	handler.WriteJSON(w, resBody, http.StatusOK)
 }
 
-func getTasksByTerms(esName string, terms []string) (*elastic.SearchResponse, error) {
+func SyncRebuildResult(esName string) error {
-	if len(terms) == 0 {
-		return nil, nil
-	}
 	client := elastic.GetClient(esName)
-	esBody := buildTermsQuery(terms)
+	esBody := `{"query":{"match":{"status": "RUNNING"}}}`
-	return client.SearchWithRawQueryDSL(".tasks", []byte(esBody))
+	esRes, err := client.SearchWithRawQueryDSL("infinireindex", []byte(esBody))
+	if err != nil {
+		return err
+	}
+	var ids = []string{}
+	idMap := map[string]int{}
+	for idx, doc := range esRes.Hits.Hits {
+		taskId := doc.Source["task_id"].(string)
+		ids = append(ids, taskId)
+		idMap[taskId] = idx
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+	taskResp, err := client.SearchTasksByIds(ids)
+	if err != nil {
+		return err
+	}
+	var (
+		status model.ReindexStatus
+	)
+	for _, doc := range taskResp.Hits.Hits {
+		status = model.ReindexStatusRunning
+		source := esRes.Hits.Hits[idMap[doc.ID.(string)]].Source
+		if _, ok := doc.Source["error"]; ok {
+			status = model.ReindexStatusFailed
+		} else {
+			status = model.ReindexStatusSuccess
+		}
+		source["status"] = status
+		source["task_source"] = doc.Source
+		_, err := client.Index("infinireindex", esRes.Hits.Hits[idMap[doc.ID.(string)]].ID, source)
+		return err
+	}
+	return nil
 }
 
 func buildTermsQuery(terms []string) string {

api/init.go

@@ -1,7 +1,10 @@
 package api
 
 import (
+	"log"
+
 	"infini.sh/framework/core/api"
+	"infini.sh/framework/core/task"
 	"infini.sh/framework/core/ui"
 	"infini.sh/search-center/api/index_management"
 	"infini.sh/search-center/config"
@@ -19,11 +22,24 @@ func Init(cfg *config.AppConfig) {
 	ui.HandleUIMethod(api.DELETE, pathPrefix+"dict/:id", handler.DeleteDictItemAction)
 	//ui.HandleUIMethod(api.DELETE, "/api/dict/", handler.DeleteDictItemAction2)
 	ui.HandleUIMethod(api.POST, pathPrefix+"dict/_update", handler.UpdateDictItemAction)
-	ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index", handler.HandleDocumentAction)
+	ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index/_search", handler.HandleSearchDocumentAction)
+	ui.HandleUIMethod(api.POST, pathPrefix+"doc/:index/_create", handler.HandleAddDocumentAction)
+	ui.HandleUIMethod(api.PUT, pathPrefix+"doc/:index/:id", handler.HandleUpdateDocumentAction)
+	ui.HandleUIMethod(api.DELETE, pathPrefix+"doc/:index/:id", handler.HandleDeleteDocumentAction)
 	ui.HandleUIMethod(api.GET, pathPrefix+"indices/_cat", handler.HandleGetIndicesAction)
 
 	ui.HandleUIMethod(api.POST, pathPrefix+"rebuild/_create", handler.ReindexAction)
 	ui.HandleUIMethod(api.GET, pathPrefix+"rebuild/list", handler.HandleGetRebuildListAction)
 	ui.HandleUIMethod(api.POST, pathPrefix+"rebuild/_delete", handler.HandleDeleteRebuildAction)
 	ui.HandleUIMethod(api.GET, pathPrefix+"indices/_mappings/:index", handler.HandleGetMappingsAction)
+
+	task.RegisterScheduleTask(task.ScheduleTask{
+		Description: "sync reindex task result to index infinireindex",
+		Task: func() {
+			err := index_management.SyncRebuildResult(cfg.Elasticsearch)
+			if err != nil {
+				log.Println(err)
+			}
+		},
+	})
 }

model/reindex.go

@@ -8,6 +8,14 @@ import (
 	"infini.sh/framework/core/elastic"
 )
 
+type ReindexStatus string
+
+const (
+	ReindexStatusRunning ReindexStatus = "RUNNING"
+	ReindexStatusSuccess ReindexStatus = "SUCCEED"
+	ReindexStatusFailed  ReindexStatus = "FAILED"
+)
+
 type InfiniReindex struct {
 	ID     string `json:"id" elastic_meta:"_id"`
 	Name   string `json:"name" elastic_mapping:"name:{type:text}"`
@@ -24,8 +32,8 @@ type InfiniReindex struct {
 		Pipeline string `json:"pipeline"`
 	} `json:"dest" elastic_mapping:"dest:{type:object}"`
 
-	CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"`
+	CreatedAt time.Time     `json:"created_at" elastic_mapping:"created_at:{type:date}"`
-	Status    string    `json:"status" elastic_mapping:"status:{type:keyword}"`
+	Status    ReindexStatus `json:"status" elastic_mapping:"status:{type:keyword}"`
 }
 
 func GetRebuildList(esName string, from, size int, name string) (*elastic.SearchResponse, error) {

web/config/router.config.js

@@ -120,12 +120,12 @@ export default [
             ]
           }, {
             path: '/data/rebuild',
-            name: 'rebuild',
-            component: './DataManagement/Rebuild',
-          },{
-            path: '/data/rebuild/list',
             name: 'rebuildlist',
             component: './DataManagement/RebuildList',
+          },{
+            path: '/data/rebuild/new',
+            name: 'rebuild',
+            component: './DataManagement/Rebuild',
             hideInMenu: true,
           }, {
             path: '/data/import',

web/mock/datamanagement/rebuild.js

@@ -0,0 +1,432 @@
+let data = {
+  "errmsg": "",
+  "errno": "0",
+  "payload": {
+    "data": [
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-06T15:55:31.426604+08:00",
+        "desc": "test source index not exists",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": ""
+        },
+        "id": "bvqmpstath24tgfo107g",
+        "name": "test failed",
+        "source": {
+          "_source": [],
+          "index": "infini-testx",
+          "query": null
+        },
+        "status": "FAILED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:824925",
+        "task_source": {
+          "completed": true,
+          "error": {
+            "index": "infini-testx",
+            "index_uuid": "_na_",
+            "reason": "no such index [infini-testx]",
+            "resource.id": "infini-testx",
+            "resource.type": "index_or_alias",
+            "type": "index_not_found_exception"
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-testx] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 824925,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 172714,
+            "start_time_in_millis": 1609919731425,
+            "status": {
+              "batches": 0,
+              "created": 0,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": 0,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 0,
+              "updated": 0,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      },
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-06T15:48:46.585112+08:00",
+        "desc": "test query param again",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": ""
+        },
+        "id": "bvqmmnlath24tgfo1070",
+        "name": "test query one",
+        "source": {
+          "_source": [],
+          "index": "infini-test",
+          "query": {
+            "match": {
+              "name": "cincky"
+            }
+          }
+        },
+        "status": "SUCCEED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:822567",
+        "task_source": {
+          "completed": true,
+          "response": {
+            "batches": 1,
+            "created": 0,
+            "deleted": 0,
+            "failures": [],
+            "noops": 0,
+            "requests_per_second": -1,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
+            "throttled": "0s",
+            "throttled_millis": 0,
+            "throttled_until": "0s",
+            "throttled_until_millis": 0,
+            "timed_out": false,
+            "took": 44,
+            "total": 1,
+            "updated": 1,
+            "version_conflicts": 0
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-test] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 822567,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 44277146,
+            "start_time_in_millis": 1609919326584,
+            "status": {
+              "batches": 1,
+              "created": 0,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": -1,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 1,
+              "updated": 1,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      },
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-06T15:46:04.745132+08:00",
+        "desc": "test query param",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": ""
+        },
+        "id": "bvqmlf5ath24tgfo106g",
+        "name": "test query",
+        "source": {
+          "_source": [],
+          "index": "infini-test1",
+          "query": {
+            "match": {
+              "name": "test"
+            }
+          }
+        },
+        "status": "SUCCEED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:821548",
+        "task_source": {
+          "completed": true,
+          "response": {
+            "batches": 0,
+            "created": 0,
+            "deleted": 0,
+            "failures": [],
+            "noops": 0,
+            "requests_per_second": -1,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
+            "throttled": "0s",
+            "throttled_millis": 0,
+            "throttled_until": "0s",
+            "throttled_until_millis": 0,
+            "timed_out": false,
+            "took": 0,
+            "total": 0,
+            "updated": 0,
+            "version_conflicts": 0
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-test1] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 821548,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 982379,
+            "start_time_in_millis": 1609919164744,
+            "status": {
+              "batches": 0,
+              "created": 0,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": -1,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 0,
+              "updated": 0,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      },
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-06T15:44:10.535851+08:00",
+        "desc": "test source param",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": ""
+        },
+        "id": "bvqmkilath24tgfo1060",
+        "name": "test source",
+        "source": {
+          "_source": [
+            "email",
+            "hobbies",
+            "created_at",
+            "name"
+          ],
+          "index": "infini-test1",
+          "query": null
+        },
+        "status": "SUCCEED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:820833",
+        "task_source": {
+          "completed": true,
+          "response": {
+            "batches": 1,
+            "created": 0,
+            "deleted": 0,
+            "failures": [],
+            "noops": 0,
+            "requests_per_second": -1,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
+            "throttled": "0s",
+            "throttled_millis": 0,
+            "throttled_until": "0s",
+            "throttled_until_millis": 0,
+            "timed_out": false,
+            "took": 53,
+            "total": 4,
+            "updated": 4,
+            "version_conflicts": 0
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-test1] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 820833,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 53101238,
+            "start_time_in_millis": 1609919050535,
+            "status": {
+              "batches": 1,
+              "created": 0,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": -1,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 4,
+              "updated": 4,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      },
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-06T15:41:08.159721+08:00",
+        "desc": "test pipeline param",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": "test"
+        },
+        "id": "bvqmj55ath24tgfo105g",
+        "name": "test pipeline",
+        "source": {
+          "_source": [],
+          "index": "infini-test1",
+          "query": null
+        },
+        "status": "SUCCEED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:819744",
+        "task_source": {
+          "completed": true,
+          "response": {
+            "batches": 1,
+            "created": 0,
+            "deleted": 0,
+            "failures": [],
+            "noops": 0,
+            "requests_per_second": -1,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
+            "throttled": "0s",
+            "throttled_millis": 0,
+            "throttled_until": "0s",
+            "throttled_until_millis": 0,
+            "timed_out": false,
+            "took": 251,
+            "total": 4,
+            "updated": 4,
+            "version_conflicts": 0
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-test1] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 819744,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 251872120,
+            "start_time_in_millis": 1609918868159,
+            "status": {
+              "batches": 1,
+              "created": 0,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": -1,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 4,
+              "updated": 4,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      },
+      {
+        "_index": "infinireindex",
+        "created_at": "2021-01-05T23:25:24.751473+08:00",
+        "desc": "test new rebuild api",
+        "dest": {
+          "index": "infini-test8",
+          "pipeline": ""
+        },
+        "id": "bvq89p5ath243f63hngg",
+        "name": "test new rebuild api",
+        "source": {
+          "_source": [],
+          "index": "infini-test",
+          "query": null
+        },
+        "status": "SUCCEED",
+        "task_id": "F0D6OfeVSzuMhf5528ANTw:707730",
+        "task_source": {
+          "completed": true,
+          "response": {
+            "batches": 1,
+            "created": 11,
+            "deleted": 0,
+            "failures": [],
+            "noops": 0,
+            "requests_per_second": -1,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
+            "throttled": "0s",
+            "throttled_millis": 0,
+            "throttled_until": "0s",
+            "throttled_until_millis": 0,
+            "timed_out": false,
+            "took": 117,
+            "total": 11,
+            "updated": 0,
+            "version_conflicts": 0
+          },
+          "task": {
+            "action": "indices:data/write/reindex",
+            "cancellable": true,
+            "description": "reindex from [infini-test] to [infini-test8][_doc]",
+            "headers": {},
+            "id": 707730,
+            "node": "F0D6OfeVSzuMhf5528ANTw",
+            "running_time_in_nanos": 118081190,
+            "start_time_in_millis": 1609860324750,
+            "status": {
+              "batches": 1,
+              "created": 11,
+              "deleted": 0,
+              "noops": 0,
+              "requests_per_second": -1,
+              "retries": {
+                "bulk": 0,
+                "search": 0
+              },
+              "throttled_millis": 0,
+              "throttled_until_millis": 0,
+              "total": 11,
+              "updated": 0,
+              "version_conflicts": 0
+            },
+            "type": "transport"
+          }
+        }
+      }
+    ],
+    "total": {
+      "relation": "eq",
+      "value": 6
+    }
+  }
+};
+
+export default {
+  'get /_search-center/rebuild/list': function(req, res){
+    res.send(data)
+  }
+}

web/src/locales/en-US.js

@@ -96,7 +96,8 @@ export default {
   'menu.data.pipes': 'PIPELINE',
   'menu.data.index': 'INDEX',
   'menu.data.snapshot': 'SNAPSHOT',
-  'menu.data.rebuild': 'REBUILD',
+  'menu.data.rebuildlist': 'REBUILD',
+  'menu.data.rebuild': 'NEW REBUILD',
   'menu.data.export': 'IMPORT&EXPORT',
   'menu.data.query': 'DOCUMENT',
 

web/src/locales/zh-CN.js

@@ -104,7 +104,8 @@ export default {
   'menu.data.pipes': '数据加工',
   'menu.data.index': '索引管理',
   'menu.data.snapshot': '快照管理',
-  'menu.data.rebuild': '重建管理',
+  'menu.data.rebuildlist': '重建管理',
+  'menu.data.rebuild': '新建重建',
   'menu.data.export': '导入导出',
   'menu.data.query': '文档管理',
 

web/src/pages/DataManagement/Import.js

@@ -22,9 +22,7 @@ const operationTabList = [
   ];
 
 @connect(({logstash,loading }) => ({
-    data: logstash.logstash,
+
-    loading: loading.models.logstash,
-    submitting: loading.effects['logstash/submitLogstashConfig'],
 }))
 
 @Form.create()
@@ -70,7 +68,7 @@ class Import extends Component {
 
   render() {
     const { operationkey } = this.state;
-    const { submitting, data, loading } = this.props;
+    const { submitting, data } = this.props;
     const {
       form: { getFieldDecorator, getFieldValue },
     } = this.props;
@@ -114,7 +112,7 @@ class Import extends Component {
             <Form onSubmit={this.handleSubmit}  hideRequiredMark style={{ marginTop: 8 }}>
             <FormItem {...formItemLayout} label="选择索引">
                 {getFieldDecorator('index', {
-                 initialValue: data.index,
+                 initialValue: '',
                 rules: [
                     {
                     required: true,
@@ -131,7 +129,7 @@ class Import extends Component {
                 </FormItem>
                 <FormItem {...formItemLayout} label="_bulk数据">
                 {getFieldDecorator('bulk', {
-                initialValue: data.bulk,
+                initialValue: '',
                 rules: [
                     {
                     required: true,

web/src/pages/DataManagement/Rebuild.js

@@ -46,7 +46,10 @@ class Rebuild extends Component {
         }
       }
     }else{
-      filterMappings[index] = mappings[index] || {};
+      if(!mappings[index]){
+        return [];
+      }
+      filterMappings[index] = mappings[index];
     }
     
     let fields = [];
@@ -279,32 +282,32 @@ class Rebuild extends Component {
         <Button key="back" onClick={()=>this.backward(currentStep)}>
           上一步
         </Button>,
-        <Button key="cancel"  style={{margin:'auto 2em'}} onClick={() =>{}}>
+        // <Button key="cancel"  style={{margin:'auto 2em'}} onClick={() =>{}}>
-          取消
+        //   取消
-        </Button>,
+        // </Button>,
-        <Button key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
+        <Button key="forward" style={{ marginLeft: '2em' }} type="primary" onClick={() => this.handleNext(currentStep)}>
           下一步
         </Button>,
       ];
     }
     if (currentStep === 2) {
       return [
-        <Button key="back" style={{ float: 'left' }} onClick={()=>this.backward(currentStep)}>
+        <Button key="back" onClick={()=>this.backward(currentStep)}>
           上一步
         </Button>,
-        <Button key="cancel" style={{margin:'auto 2em'}} onClick={() => {}}>
+        // <Button key="cancel" style={{margin:'auto 2em'}} onClick={() => {}}>
-          取消
+        //   取消
-        </Button>,
+        // </Button>,
-        <Button key="submit" type="primary" onClick={() =>this.handleNext(currentStep)}>
+        <Button key="submit"  style={{ marginLeft: '2em' }} type="primary" onClick={() =>this.handleNext(currentStep)}>
           完成
         </Button>,
       ];
     }
     return [
-      <Button key="cancel" onClick={() => {}}>
+      // <Button key="cancel" onClick={() => {}}>
-        取消
+      //   取消
-      </Button>,
+      // </Button>,
-      <Button style={{marginLeft:'2em'}} key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
+      <Button key="forward" type="primary" onClick={() => this.handleNext(currentStep)}>
         下一步
       </Button>,
     ];

web/src/pages/DataManagement/RebuildList.js

@@ -2,6 +2,7 @@ import { Card, Table, Form, Row, Input, Col, Button, Divider, Tooltip,Popconfirm
 import Link from 'umi/link';
 import React from 'react';
 import {connect} from 'dva'
+import moment from 'moment';
 
 @connect(({rebuildlist}) => ({
   rebuildlist,
@@ -22,41 +23,52 @@ class RebuildList extends React.Component {
     })
   }
   columns = [{
-    title: 'id',
-    dataIndex: 'id',
-    key: 'id',
-  },{
     title: 'rebuild name',
     dataIndex: 'name',
     key: 'name',
-  },{
-    title: 'description',
-    dataIndex: 'desc',
-    key: 'desc',
   },{
     title: 'status',
     dataIndex: 'status',
     key: 'status',
     render: (text, record) => (
-      <span style={{color: text== 'SUCCESS' ? 'green': (text=='FAILED' ? 'red': 'blue')}}>
+      <span style={{color: text== 'SUCCEED' ? 'green': (text=='FAILED' ? 'red': 'blue')}}>
-        {text == 'FAILED'? <Tooltip placeholder="top" title={record.error}>{text}</Tooltip> : text}
+        {text == 'FAILED'? <Tooltip placeholder="top" title={record.task_source.error.reason}>{text}</Tooltip> : text}
       </span>
     ),
   },{
     title: 'took_time',
     dataIndex: 'took_time',
     key: 'took_time',
+    render: (_, record)=>(record.task_source &&record.task_source.response ?record.task_source.response.took: -1)
+  },{
+    title: 'total',
+    dataIndex: 'total',
+    key: 'total',
+    render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.total: 0)
+  },{
+    title: 'updated',
+    dataIndex: 'updated',
+    key: 'updated',
+    render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.updated: 0)
+  },{
+    title: 'created',
+    dataIndex: 'created',
+    key: 'created',
+    render: (_, record)=>(record.task_source && record.task_source.response ?record.task_source.response.created: 0)
   },{
     title: 'created_at',
     dataIndex: 'created_at',
     key: 'created_at',
+    render: (text) =>(<span>{moment(text).format('YYYY-MM-DD HH:mm:ss')}</span>)
   },{
     title: 'Operation',
     render: (text, record) => (
       <div>
          <Popconfirm title="Sure to delete?" onConfirm={() => this.handleDeleteClick(record)}><a key="delete">Delete</a>
         </Popconfirm> 
-        {record.status=='FAILED' ? <span><Divider type="vertical" /><a key="redo" onClick={()=>this.handleRedoClick(record)}>Redo</a></span>
+        {record.status=='FAILED' ? <span><Divider type="vertical" />
+        <Popconfirm title="Sure to redo?" onConfirm={() => this.handleRedoClick(record)}><a key="redo">Redo</a></Popconfirm>
+        </span>
         : ''}
        
       </div>
@@ -100,6 +112,15 @@ class RebuildList extends React.Component {
       name: nameVal,
     })
   }
+
+  handleRefreshClick = ()=>{
+    const {rebuildlist} = this.props;
+    this.fetchData({
+      pageIndex: rebuildlist.pageIndex,
+      pageSize: rebuildlist.pageSize,
+      name: rebuildlist.name,
+    })
+  }
   
   render(){
     const {getFieldDecorator} = this.props.form;
@@ -120,7 +141,7 @@ class RebuildList extends React.Component {
             </Col>
             <Col md={8} sm={8}>
               <div style={{paddingTop:4}}>
-                <Button type="primary" onClick={this.handleSearch}>
+                <Button type="primary" icon="search" onClick={this.handleSearch}>
                   Search
                 </Button>
               </div>
@@ -130,13 +151,19 @@ class RebuildList extends React.Component {
         <Divider style={{marginBottom:0}} />
         <Card 
           bodyStyle={{padding:0}}
-          extra={<Link to="/data/rebuild"> <Button type="primary" icon="plus">New</Button></Link>}
+          extra={<div><Button onClick={this.handleRefreshClick} icon="redo">Refresh</Button><Link to="/data/rebuild/new"> <Button type="primary" icon="plus">New</Button></Link></div>}
           bordered={false}>
           <Table columns={this.columns}
             loading={rebuildlist.isLoading}
             bordered
             rowKey="id"
             expandedRowRender={record => <div>
+              <Row>
+                <Col span={24}>
+                <span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>description: </span>
+                <span>{record.desc}</span>
+                </Col>
+              </Row>
               <Row>
                 <Col span={12}><span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>source</span><pre>{JSON.stringify(record.source, null, 2)}</pre></Col>
                 <Col span={12}><span style={{fontSize: 16,color: 'rgba(0, 0, 0, 0.85)'}}>dest</span><pre>{JSON.stringify(record.dest, null, 2)}</pre></Col>

web/src/pages/DataManagement/indices.js

@@ -32,7 +32,7 @@ class Indices extends Component {
       },
       {
         key: 'doc',
-        tab: '索引文档管理',
+        tab: '索引管理',
       },
       {
         key: 'template',

web/src/pages/DataManagement/models/document.js

@@ -115,6 +115,9 @@ export default {
         }
       });
       let res = yield call(deleteDoc, payload);
+      if(typeof res == 'string'){
+        res = JSON.parse(res);
+      }
       if(res.errno != "0"){
         message.warn("删除数据失败")
         return
@@ -152,6 +155,7 @@ export default {
           docItem: res.payload,
           extra: {
             isLoading: false,
+            isAddNew: false,
           }
         } 
       })

web/src/pages/DataManagement/models/rebuild.js

@@ -15,7 +15,6 @@ export default {
   effects:{
     *addTask({payload}, {call, put}){
       let resp = yield call(reindex, payload);
-      console.log(resp);
       if(resp.errno != "0"){
         message.warn("rebuild failed")
         return
@@ -34,7 +33,6 @@ export default {
     },
     *fetchMappings({payload}, {call, put}){
       let resp = yield call(getMappings, payload);
-      console.log(resp);
       if(resp.errno != "0"){
         message.warn("get mappings failed")
         return

web/src/pages/DataManagement/models/rebuildlist.js

@@ -88,14 +88,17 @@ export default {
         message.error("delete failed")
         return
       }
-      yield call(delay, 1000);
+      let {data, total} = yield select(state=>state.rebuildlist);
-      let {pageIndex, pageSize, name} = select(state=>state.rebuildlist);
+      let newData = data.filter(item=> !payload.includes(item.id));
       yield put({
-        type: 'fetchRebuildList',
+        type: 'saveData',
         payload: {
-          pageIndex: pageIndex,
+          data: newData,
-          pageSize: pageSize,
+          total: {
-          name: name
+            ...total,
+            value: total.value - payload.length,
+          },
+          isLoading: false,
         }
       })
     }

web/src/services/doc.js

@@ -2,7 +2,7 @@ import request from '@/utils/request';
 import {pathPrefix} from './common';
 
 export async function getDocList(params) {
-  return request(`${pathPrefix}/doc/${params.index}`, {
+  return request(`${pathPrefix}/doc/${params.index}/_search`, {
     method: 'POST',
     body: {
       action: 'SEARCH',
@@ -12,35 +12,23 @@ export async function getDocList(params) {
 }
 
 export async function saveDoc(params) {
-  return request(`${pathPrefix}/doc/${params.index}`, {
+  return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
-    method: 'POST',
+    method: 'PUT',
-    body: {
+    body: params.data,
-      payload: params.data,
-      index: params.index,
-      action: 'SAVE',
-    },
   });
 }
 
 export async function deleteDoc(params) {
-  return request(`${pathPrefix}/doc/${params.index}`, {
+  return request(`${pathPrefix}/doc/${params.index}/${params.data.id}`, {
-    method: 'POST',
+    method: 'DELETE',
-    body: {
-      index: params.index,
-      action: 'DELETE',
-      payload: params.data,
-    },
   });
 }
 
 export async function addDoc(params) {
-  return request(`${pathPrefix}/doc/${params.index}`, {
+  delete(params.data, 'id');
+  return request(`${pathPrefix}/doc/${params.index}/_create`, {
     method: 'POST',
-    body: {
+    body: params.data,
-      payload: params.data,
-      index: params.index,
-      action: 'ADD',
-    },
   });
 }