floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[migration] index_migration & pipeline define struct

This commit is contained in:
Kassian Sun 2023-04-13 11:25:40 +08:00
parent db6d33f22b
commit c73c44724d
3 changed files with 197 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
plugin/migration/api.go
View File

@ -49,7 +49,7 @@ type APIHandler struct {
} }
func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterTaskConfig := &ElasticDataConfig{} clusterTaskConfig := &ClusterMigrationTaskConfig{}
err := h.DecodeJSON(req, clusterTaskConfig) err := h.DecodeJSON(req, clusterTaskConfig)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -90,10 +90,10 @@ func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Re
"source_total_docs": totalDocs, "source_total_docs": totalDocs,
}, },
}, },
Cancellable: true, Cancellable: true,
Runnable: false, Runnable: false,
Status: task2.StatusInit, Status: task2.StatusInit,
Config: clusterTaskConfig, ConfigString: util.MustToJSON(clusterTaskConfig),
} }
t.ID = util.GetUUID() t.ID = util.GetUUID()
err = orm.Create(nil, &t) err = orm.Create(nil, &t)
@ -174,7 +174,7 @@ func (h *APIHandler) searchDataMigrationTask(w http.ResponseWriter, req *http.Re
for _, hit := range searchRes.Hits.Hits { for _, hit := range searchRes.Hits.Hits {
sourceM := util.MapStr(hit.Source) sourceM := util.MapStr(hit.Source)
buf := util.MustToJSONBytes(sourceM["config"]) buf := util.MustToJSONBytes(sourceM["config"])
dataConfig := ElasticDataConfig{} dataConfig := ClusterMigrationTaskConfig{}
err = util.FromJSONBytes(buf, &dataConfig) err = util.FromJSONBytes(buf, &dataConfig)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -317,12 +317,14 @@ func (h *APIHandler) stopDataMigrationTask(w http.ResponseWriter, req *http.Requ
} }
func getTaskConfig(task *task2.Task, config interface{}) error { func getTaskConfig(task *task2.Task, config interface{}) error {
configBytes, err := util.ToJSONBytes(task.Config) if task.Config_ == nil {
return util.FromJSONBytes([]byte(task.ConfigString), config)
}
buf, err := util.ToJSONBytes(task.Config_)
if err != nil { if err != nil {
return err return err
} }
return util.FromJSONBytes(buf, config)
return util.FromJSONBytes(configBytes, config)
} }
func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API) (map[string]string, error) { func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API) (map[string]string, error) {
@ -377,7 +379,7 @@ func (h *APIHandler) getIndexRefreshIntervals(w http.ResponseWriter, req *http.R
}, http.StatusNotFound) }, http.StatusNotFound)
return return
} }
taskConfig := &ElasticDataConfig{} taskConfig := &ClusterMigrationTaskConfig{}
err = getTaskConfig(&obj, taskConfig) err = getTaskConfig(&obj, taskConfig)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -415,7 +417,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
}, http.StatusNotFound) }, http.StatusNotFound)
return return
} }
taskConfig := &ElasticDataConfig{} taskConfig := &ClusterMigrationTaskConfig{}
err = getTaskConfig(&obj, taskConfig) err = getTaskConfig(&obj, taskConfig)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -461,7 +463,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
"index_name": migrationConfig.LogIndexName, "index_name": migrationConfig.LogIndexName,
} }
} }
obj.Config = taskConfig obj.ConfigString = util.MustToJSON(taskConfig)
obj.Metadata.Labels["completed_indices"] = completedIndices obj.Metadata.Labels["completed_indices"] = completedIndices
h.WriteJSON(w, obj, http.StatusOK) h.WriteJSON(w, obj, http.StatusOK)
} }
@ -735,17 +737,16 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
startTime = subTasks[0].StartTimeInMillis startTime = subTasks[0].StartTimeInMillis
} }
for i, ptask := range subTasks { for i, ptask := range subTasks {
var ( cfg := IndexMigrationTaskConfig{}
cfg map[string]interface{} err := getTaskConfig(&ptask, &cfg)
ok bool if err != nil {
) log.Errorf("failed to get task config, err: %v", err)
if cfg, ok = ptask.Config.(map[string]interface{}); !ok {
continue continue
} }
start, _ := util.MapStr(cfg).GetValue("source.start") start := cfg.Source.Start
end, _ := util.MapStr(cfg).GetValue("source.end") end := cfg.Source.End
if i == 0 { if i == 0 {
step, _ := util.MapStr(cfg).GetValue("source.step") step := cfg.Source.Step
taskInfo["step"] = step taskInfo["step"] = step
} }
var durationInMS int64 = 0 var durationInMS int64 = 0
@ -786,7 +787,7 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
} }
} }
partitionTotalDocs, _ := util.MapStr(cfg).GetValue("source.doc_count") partitionTotalDocs := cfg.Source.DocCount
partitionTaskInfos = append(partitionTaskInfos, util.MapStr{ partitionTaskInfos = append(partitionTaskInfos, util.MapStr{
"task_id": ptask.ID, "task_id": ptask.ID,
"status": ptask.Status, "status": ptask.Status,

83
plugin/migration/model.go
View File

@ -4,9 +4,13 @@
package migration package migration
import "fmt" import (
"fmt"
type ElasticDataConfig struct { "infini.sh/framework/core/util"
)
type ClusterMigrationTaskConfig struct {
Cluster struct { Cluster struct {
Source ClusterInfo `json:"source"` Source ClusterInfo `json:"source"`
Target ClusterInfo `json:"target"` Target ClusterInfo `json:"target"`
@ -20,15 +24,8 @@ type ElasticDataConfig struct {
Docs int `json:"docs"` Docs int `json:"docs"`
Timeout string `json:"timeout"` Timeout string `json:"timeout"`
} `json:"scroll"` } `json:"scroll"`
Bulk struct { Bulk ClusterMigrationBulkConfig `json:"bulk"`
Docs int `json:"docs"` Execution ExecutionConfig `json:"execution"`
StoreSizeInMB int `json:"store_size_in_mb"`
MaxWorkerSize int `json:"max_worker_size"`
IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"`
SliceSize int `json:"slice_size"`
Compress bool `json:"compress"`
} `json:"bulk"`
Execution ExecutionConfig `json:"execution"`
} `json:"settings"` } `json:"settings"`
Creator struct { Creator struct {
Name string `json:"name"` Name string `json:"name"`
@ -36,6 +33,15 @@ type ElasticDataConfig struct {
} `json:"creator"` } `json:"creator"`
} }
type ClusterMigrationBulkConfig struct {
Docs int `json:"docs"`
StoreSizeInMB int `json:"store_size_in_mb"`
MaxWorkerSize int `json:"max_worker_size"`
IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"`
SliceSize int `json:"slice_size"`
Compress bool `json:"compress"`
}
type ExecutionConfig struct { type ExecutionConfig struct {
TimeWindow []TimeWindowItem `json:"time_window"` TimeWindow []TimeWindowItem `json:"time_window"`
Nodes struct { Nodes struct {
@ -65,6 +71,7 @@ type IndexConfig struct {
Percent float64 `json:"percent,omitempty"` Percent float64 `json:"percent,omitempty"`
ErrorPartitions int `json:"error_partitions,omitempty"` ErrorPartitions int `json:"error_partitions,omitempty"`
} }
type IndexPartition struct { type IndexPartition struct {
FieldType string `json:"field_type"` FieldType string `json:"field_type"`
FieldName string `json:"field_name"` FieldName string `json:"field_name"`
@ -108,3 +115,57 @@ type IndexStateInfo struct {
ErrorPartitions int ErrorPartitions int
IndexDocs float64 IndexDocs float64
} }
type IndexMigrationTaskConfig struct {
Source IndexMigrationSourceConfig `json:"source"`
Target IndexMigrationTargetConfig `json:"target"`
Execution ExecutionConfig `json:"execution"`
}
type IndexMigrationSourceConfig struct {
ClusterId string `json:"cluster_id"`
Indices string `json:"indices"`
SliceSize int `json:"slice_size"`
BatchSize int `json:"batch_size"`
ScrollTime string `json:"scroll_time"`
IndexRename util.MapStr `json:"index_rename,omitempty"`
TypeRename util.MapStr `json:"type_rename,omitempty"`
QueryString string `json:"query_string,omitempty'`
QueryDSL util.MapStr `json:"query_dsl,omitempty"`
// Parition configs
Start float64 `json:"start"`
End float64 `json:"end"`
Docs int64 `json:"docs"`
DocCount int64 `json:"doc_count"`
Step interface{} `json:"step"`
PartitionId int `json:"partition_id"`
}
type IndexMigrationBulkConfig struct {
BatchSizeInDocs int `json:"batch_size_in_docs"`
BatchSizeInMB int `json:"batch_size_in_mb"`
MaxWorkerSize int `json:"max_worker_size"`
IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"`
SliceSize int `json:"slice_size"`
Compress bool `json:"compress"`
}
type IndexMigrationTargetConfig struct {
ClusterId string `json:"cluster_id"`
Bulk IndexMigrationBulkConfig `json:"bulk"`
QueryDSL util.MapStr `json:"query_dsl,omitempty"`
}
type PipelineTaskLoggingConfig struct {
Enabled bool `json:"enabled"`
}
type PipelineTaskConfig struct {
Name string `json:"name"`
Logging PipelineTaskLoggingConfig `json:"logging"`
Labels util.MapStr `json:"labels"`
AutoStart bool `json:"auto_start"`
KeepRunning bool `json:"keep_running"`
Processor []util.MapStr `json:"processor"`
}

259
plugin/migration/pipeline.go
View File

@ -13,7 +13,6 @@ import (
"time" "time"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"github.com/mitchellh/mapstructure"
"infini.sh/console/model" "infini.sh/console/model"
"infini.sh/framework/core/config" "infini.sh/framework/core/config"
@ -345,7 +344,7 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error {
p.saveTaskAndWriteLog(taskItem, "", &task2.TaskResult{ p.saveTaskAndWriteLog(taskItem, "", &task2.TaskResult{
Success: state.Error == "", Success: state.Error == "",
Error: state.Error, Error: state.Error,
}, fmt.Sprintf("task [%s] completed", taskItem.ID)) }, fmt.Sprintf("task [%s] finished with status [%s]", taskItem.ID, taskItem.Status))
p.cleanGatewayPipelines(taskItem, state.PipelineIds) p.cleanGatewayPipelines(taskItem, state.PipelineIds)
} else { } else {
if state.RunningPhase == 1 && taskItem.Metadata.Labels["running_phase"] == float64(1) { if state.RunningPhase == 1 && taskItem.Metadata.Labels["running_phase"] == float64(1) {
@ -374,7 +373,7 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error {
log.Errorf("failed to get instance, err: %v", err) log.Errorf("failed to get instance, err: %v", err)
return err return err
} }
err = inst.CreatePipeline(util.MustToJSONBytes(bulkTask.Config)) err = inst.CreatePipeline([]byte(bulkTask.ConfigString))
if err != nil { if err != nil {
log.Errorf("failed to create bulk_indexing pipeline, err: %v", err) log.Errorf("failed to create bulk_indexing pipeline, err: %v", err)
return err return err
@ -516,10 +515,8 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
return nil return nil
} }
for i, t := range ptasks { for i, t := range ptasks {
ptasks[i].RetryTimes = taskItem.RetryTimes + 1
if t.Metadata.Labels != nil { if t.Metadata.Labels != nil {
if cfg, ok := ptasks[i].Config.(map[string]interface{}); ok {
util.MapStr(cfg).Put("labels.retry_no", taskItem.RetryTimes+1)
}
if t.Metadata.Labels["pipeline_id"] == "es_scroll" { if t.Metadata.Labels["pipeline_id"] == "es_scroll" {
scrollTask = &ptasks[i] scrollTask = &ptasks[i]
} else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" { } else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" {
@ -537,35 +534,24 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
pids = append(pids, taskItem.ParentId...) pids = append(pids, taskItem.ParentId...)
pids = append(pids, taskItem.ID) pids = append(pids, taskItem.ID)
scrollID := util.GetUUID() scrollID := util.GetUUID()
var ( cfg := IndexMigrationTaskConfig{}
cfg map[string]interface{} err := getTaskConfig(taskItem, &cfg)
ok bool if err != nil {
) return fmt.Errorf("got wrong config [%v] with task [%s], err: %v", taskItem.ConfigString, taskItem.ID, err)
if cfg, ok = taskItem.Config.(map[string]interface{}); !ok {
return fmt.Errorf("got wrong config [%v] with task [%s]", taskItem.Config, taskItem.ID)
}
cfgm := util.MapStr(cfg)
var (
sourceClusterID string
targetClusterID string
)
if sourceClusterID, ok = getMapValue(cfgm, "source.cluster_id").(string); !ok {
return fmt.Errorf("got wrong source cluster id of task [%v]", *taskItem)
}
if targetClusterID, ok = getMapValue(cfgm, "target.cluster_id").(string); !ok {
return fmt.Errorf("got wrong target cluster id of task [%v]", *taskItem)
} }
sourceClusterID := cfg.Source.ClusterId
targetClusterID := cfg.Target.ClusterId
esConfig := elastic.GetConfig(sourceClusterID) esConfig := elastic.GetConfig(sourceClusterID)
esTargetConfig := elastic.GetConfig(targetClusterID) esTargetConfig := elastic.GetConfig(targetClusterID)
docType := common.GetClusterDocType(targetClusterID) docType := common.GetClusterDocType(targetClusterID)
if len(taskItem.ParentId) == 0 { if len(taskItem.ParentId) == 0 {
return fmt.Errorf("got wrong parent id of task [%v]", *taskItem) return fmt.Errorf("got wrong parent id of task [%v]", *taskItem)
} }
queryDsl := getMapValue(cfgm, "source.query_dsl") queryDsl := cfg.Source.QueryDSL
scrollQueryDsl := util.MustToJSON(util.MapStr{ scrollQueryDsl := util.MustToJSON(util.MapStr{
"query": queryDsl, "query": queryDsl,
}) })
indexName := getMapValue(cfgm, "source.indices") indexName := cfg.Source.Indices
scrollTask = &task2.Task{ scrollTask = &task2.Task{
ParentId: pids, ParentId: pids,
Runnable: true, Runnable: true,
@ -579,24 +565,24 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
"unique_index_name": taskItem.Metadata.Labels["unique_index_name"], "unique_index_name": taskItem.Metadata.Labels["unique_index_name"],
}, },
}, },
Config: util.MapStr{ RetryTimes: taskItem.RetryTimes,
"name": scrollID, ConfigString: util.MustToJSON(PipelineTaskConfig{
"logging": util.MapStr{ Name: scrollID,
"enabled": true, Logging: PipelineTaskLoggingConfig{
Enabled: true,
}, },
"labels": util.MapStr{ Labels: util.MapStr{
"parent_task_id": pids, "parent_task_id": pids,
"unique_index_name": taskItem.Metadata.Labels["unique_index_name"], "unique_index_name": taskItem.Metadata.Labels["unique_index_name"],
"retry_no": taskItem.RetryTimes,
}, },
"auto_start": true, AutoStart: true,
"keep_running": false, KeepRunning: false,
"processor": []util.MapStr{ Processor: []util.MapStr{
{ {
"es_scroll": util.MapStr{ "es_scroll": util.MapStr{
"remove_type": docType == "", "remove_type": docType == "",
"slice_size": getMapValue(cfgm, "source.slice_size"), "slice_size": cfg.Source.SliceSize,
"batch_size": getMapValue(cfgm, "source.batch_size"), "batch_size": cfg.Source.BatchSize,
"indices": indexName, "indices": indexName,
"elasticsearch": sourceClusterID, "elasticsearch": sourceClusterID,
"elasticsearch_config": util.MapStr{ "elasticsearch_config": util.MapStr{
@ -612,14 +598,14 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
}, },
}, },
"partition_size": 1, "partition_size": 1,
"scroll_time": getMapValue(cfgm, "source.scroll_time"), "scroll_time": cfg.Source.ScrollTime,
"query_dsl": scrollQueryDsl, "query_dsl": scrollQueryDsl,
"index_rename": getMapValue(cfgm, "source.index_rename"), "index_rename": cfg.Source.IndexRename,
"type_rename": getMapValue(cfgm, "source.type_rename"), "type_rename": cfg.Source.TypeRename,
}, },
}, },
}, },
}, }),
} }
scrollTask.ID = scrollID scrollTask.ID = scrollID
@ -637,36 +623,31 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
"unique_index_name": taskItem.Metadata.Labels["unique_index_name"], "unique_index_name": taskItem.Metadata.Labels["unique_index_name"],
}, },
}, },
Config: util.MapStr{ RetryTimes: taskItem.RetryTimes,
"name": bulkID, ConfigString: util.MustToJSON(PipelineTaskConfig{
"logging": util.MapStr{ Name: bulkID,
"enabled": true, Logging: PipelineTaskLoggingConfig{
Enabled: true,
}, },
"labels": util.MapStr{ Labels: util.MapStr{
"parent_task_id": pids, "parent_task_id": pids,
"unique_index_name": taskItem.Metadata.Labels["unique_index_name"], "unique_index_name": taskItem.Metadata.Labels["unique_index_name"],
"retry_no": taskItem.RetryTimes,
}, },
"auto_start": true, AutoStart: true,
"keep_running": false, KeepRunning: false,
"processor": []util.MapStr{ Processor: []util.MapStr{
{ {
"bulk_indexing": util.MapStr{ "bulk_indexing": util.MapStr{
"detect_active_queue": false, "detect_active_queue": false,
"bulk": util.MapStr{ "bulk": util.MapStr{
"batch_size_in_mb": getMapValue(cfgm, "target.bulk.batch_size_in_mb"), "batch_size_in_mb": cfg.Target.Bulk.BatchSizeInMB,
"batch_size_in_docs": getMapValue(cfgm, "target.bulk.batch_size_in_docs"), "batch_size_in_docs": cfg.Target.Bulk.BatchSizeInDocs,
"invalid_queue": "bulk_indexing_400", "invalid_queue": "bulk_indexing_400",
"compress": getMapValue(cfgm, "target.bulk.compress"), "compress": cfg.Target.Bulk.Compress,
//"retry_rules": util.MapStr{
// "default": false,
// "retry_4xx": false,
// "retry_429": true,
//},
}, },
"max_worker_size": getMapValue(cfgm, "target.bulk.max_worker_size"), "max_worker_size": cfg.Target.Bulk.MaxWorkerSize,
"num_of_slices": getMapValue(cfgm, "target.bulk.slice_size"), "num_of_slices": cfg.Target.Bulk.SliceSize,
"idle_timeout_in_seconds": getMapValue(cfgm, "target.bulk.idle_timeout_in_seconds"), "idle_timeout_in_seconds": cfg.Target.Bulk.IdleTimeoutInSeconds,
"elasticsearch": targetClusterID, "elasticsearch": targetClusterID,
"elasticsearch_config": util.MapStr{ "elasticsearch_config": util.MapStr{
"name": targetClusterID, "name": targetClusterID,
@ -681,7 +662,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
}, },
}, },
}, },
}, }),
} }
bulkTask.ID = bulkID bulkTask.ID = bulkID
} }
@ -707,7 +688,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
} }
//call instance api to create pipeline task //call instance api to create pipeline task
err = instance.CreatePipeline(util.MustToJSONBytes(scrollTask.Config)) err = instance.CreatePipeline([]byte(scrollTask.ConfigString))
if err != nil { if err != nil {
log.Errorf("create scroll pipeline failed, err: %+v", err) log.Errorf("create scroll pipeline failed, err: %+v", err)
return err return err
@ -765,15 +746,13 @@ func (p *DispatcherProcessor) getPreferenceInstance(majorTaskID string) (instanc
majorTask.ID = majorTaskID majorTask.ID = majorTaskID
_, err = orm.Get(&majorTask) _, err = orm.Get(&majorTask)
if err != nil { if err != nil {
log.Errorf("failed to get major task, err: %v", err)
return return
} }
cfg := ElasticDataConfig{} cfg := ClusterMigrationTaskConfig{}
buf, err := util.ToJSONBytes(majorTask.Config) err = getTaskConfig(&majorTask, &cfg)
if err != nil {
return
}
err = util.FromJSONBytes(buf, &cfg)
if err != nil { if err != nil {
log.Errorf("failed to get task config, err: %v", err)
return return
} }
var ( var (
@ -891,7 +870,7 @@ func writeLog(taskItem *task2.Task, taskResult *task2.TaskResult, message string
Fields: util.MapStr{ Fields: util.MapStr{
"task": util.MapStr{ "task": util.MapStr{
"logging": util.MapStr{ "logging": util.MapStr{
"config": util.MustToJSON(taskItem.Config), "config": taskItem.ConfigString,
"status": taskItem.Status, "status": taskItem.Status,
"message": message, "message": message,
"result": taskResult, "result": taskResult,
@ -913,40 +892,40 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
return nil return nil
} }
buf := util.MustToJSONBytes(taskItem.Config) clusterMigrationTask := ClusterMigrationTaskConfig{}
clusterMigrationTask := ElasticDataConfig{} err := getTaskConfig(taskItem, &clusterMigrationTask)
err := util.FromJSONBytes(buf, &clusterMigrationTask)
if err != nil { if err != nil {
log.Errorf("failed to get task config, err: %v", err)
return err return err
} }
defer func() { defer func() {
taskItem.Config = clusterMigrationTask taskItem.ConfigString = util.MustToJSON(clusterMigrationTask)
}() }()
esSourceClient := elastic.GetClient(clusterMigrationTask.Cluster.Source.Id) esSourceClient := elastic.GetClient(clusterMigrationTask.Cluster.Source.Id)
targetType := common.GetClusterDocType(clusterMigrationTask.Cluster.Target.Id) targetType := common.GetClusterDocType(clusterMigrationTask.Cluster.Target.Id)
for _, index := range clusterMigrationTask.Indices { for _, index := range clusterMigrationTask.Indices {
source := util.MapStr{ source := IndexMigrationSourceConfig{
"cluster_id": clusterMigrationTask.Cluster.Source.Id, ClusterId: clusterMigrationTask.Cluster.Source.Id,
"indices": index.Source.Name, Indices: index.Source.Name,
"slice_size": clusterMigrationTask.Settings.Scroll.SliceSize, SliceSize: clusterMigrationTask.Settings.Scroll.SliceSize,
"batch_size": clusterMigrationTask.Settings.Scroll.Docs, BatchSize: clusterMigrationTask.Settings.Scroll.Docs,
"scroll_time": clusterMigrationTask.Settings.Scroll.Timeout, ScrollTime: clusterMigrationTask.Settings.Scroll.Timeout,
} }
if index.IndexRename != nil { if index.IndexRename != nil {
source["index_rename"] = index.IndexRename source.IndexRename = index.IndexRename
} }
if index.Target.Name != "" { if index.Target.Name != "" {
source["index_rename"] = util.MapStr{ source.IndexRename = util.MapStr{
index.Source.Name: index.Target.Name, index.Source.Name: index.Target.Name,
} }
} }
if index.TypeRename != nil { if index.TypeRename != nil {
source["type_rename"] = index.TypeRename source.TypeRename = index.TypeRename
} }
if v, ok := index.RawFilter.(string); ok { if v, ok := index.RawFilter.(string); ok {
source["query_string"] = v source.QueryString = v
} else { } else {
var must []interface{} var must []interface{}
if index.RawFilter != nil { if index.RawFilter != nil {
@ -954,7 +933,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
} }
if index.Source.DocType != "" { if index.Source.DocType != "" {
if index.Target.DocType != "" { if index.Target.DocType != "" {
source["type_rename"] = util.MapStr{ source.TypeRename = util.MapStr{
index.Source.DocType: index.Target.DocType, index.Source.DocType: index.Target.DocType,
} }
} }
@ -965,13 +944,13 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
}) })
} else { } else {
if targetType != "" { if targetType != "" {
source["type_rename"] = util.MapStr{ source.TypeRename = util.MapStr{
"*": index.Target.DocType, "*": index.Target.DocType,
} }
} }
} }
if len(must) > 0 { if len(must) > 0 {
source["query_dsl"] = util.MapStr{ source.QueryDSL = util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
}, },
@ -990,20 +969,20 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
}) })
} }
target := util.MapStr{ target := IndexMigrationTargetConfig{
"cluster_id": clusterMigrationTask.Cluster.Target.Id, ClusterId: clusterMigrationTask.Cluster.Target.Id,
"bulk": util.MapStr{ Bulk: IndexMigrationBulkConfig{
"batch_size_in_mb": clusterMigrationTask.Settings.Bulk.StoreSizeInMB, BatchSizeInMB: clusterMigrationTask.Settings.Bulk.StoreSizeInMB,
"batch_size_in_docs": clusterMigrationTask.Settings.Bulk.Docs, BatchSizeInDocs: clusterMigrationTask.Settings.Bulk.Docs,
"max_worker_size": clusterMigrationTask.Settings.Bulk.MaxWorkerSize, MaxWorkerSize: clusterMigrationTask.Settings.Bulk.MaxWorkerSize,
"idle_timeout_in_seconds": clusterMigrationTask.Settings.Bulk.IdleTimeoutInSeconds, IdleTimeoutInSeconds: clusterMigrationTask.Settings.Bulk.IdleTimeoutInSeconds,
"slice_size": clusterMigrationTask.Settings.Bulk.SliceSize, SliceSize: clusterMigrationTask.Settings.Bulk.SliceSize,
"compress": clusterMigrationTask.Settings.Bulk.Compress, Compress: clusterMigrationTask.Settings.Bulk.Compress,
}, },
} }
indexParameters := util.MapStr{ indexParameters := IndexMigrationTaskConfig{
"source": source, Source: source,
"target": target, Target: target,
} }
indexMigrationTask := task2.Task{ indexMigrationTask := task2.Task{
ParentId: []string{taskItem.ID}, ParentId: []string{taskItem.ID},
@ -1022,7 +1001,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
"unique_index_name": index.Source.GetUniqueIndexName(), "unique_index_name": index.Source.GetUniqueIndexName(),
}, },
}, },
Config: indexParameters, ConfigString: util.MustToJSON(indexParameters),
} }
indexMigrationTask.ID = util.GetUUID() indexMigrationTask.ID = util.GetUUID()
@ -1034,7 +1013,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
FieldType: index.Partition.FieldType, FieldType: index.Partition.FieldType,
Step: index.Partition.Step, Step: index.Partition.Step,
//Filter: index.RawFilter, //Filter: index.RawFilter,
Filter: source["query_dsl"], Filter: source.QueryDSL,
} }
partitions, err := elastic.GetPartitions(partitionQ, esSourceClient) partitions, err := elastic.GetPartitions(partitionQ, esSourceClient)
if err != nil { if err != nil {
@ -1052,20 +1031,14 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
continue continue
} }
partitionID++ partitionID++
partitionSource := util.MapStr{ partitionSource := source
"start": partition.Start, partitionSource.Start = partition.Start
"end": partition.End, partitionSource.End = partition.End
"doc_count": partition.Docs, partitionSource.DocCount = partition.Docs
"step": index.Partition.Step, partitionSource.Step = index.Partition.Step
"partition_id": partitionID, partitionSource.PartitionId = partitionID
} partitionSource.QueryDSL = partition.Filter
for k, v := range source { partitionSource.QueryString = ""
if k == "query_string" {
continue
}
partitionSource[k] = v
}
partitionSource["query_dsl"] = partition.Filter
var must []interface{} var must []interface{}
if partition.Other { if partition.Other {
@ -1084,8 +1057,9 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
if targetMust != nil { if targetMust != nil {
must = append(must, targetMust...) must = append(must, targetMust...)
} }
partitionTarget := target
if len(must) > 0 { if len(must) > 0 {
target["query_dsl"] = util.MapStr{ partitionTarget.QueryDSL = util.MapStr{
"query": util.MapStr{ "query": util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": must, "must": must,
@ -1109,22 +1083,22 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
"unique_index_name": index.Source.GetUniqueIndexName(), "unique_index_name": index.Source.GetUniqueIndexName(),
}, },
}, },
Config: util.MapStr{ ConfigString: util.MustToJSON(IndexMigrationTaskConfig{
"source": partitionSource, Source: partitionSource,
"target": target, Target: partitionTarget,
"execution": clusterMigrationTask.Settings.Execution, Execution: clusterMigrationTask.Settings.Execution,
}, }),
} }
partitionMigrationTask.ID = util.GetUUID() partitionMigrationTask.ID = util.GetUUID()
err = orm.Create(nil, &partitionMigrationTask) err = orm.Create(nil, &partitionMigrationTask)
delete(target, "query_dsl") target.QueryDSL = nil
if err != nil { if err != nil {
return fmt.Errorf("store index migration task(partition) error: %w", err) return fmt.Errorf("store index migration task(partition) error: %w", err)
} }
} }
} else { } else {
source["doc_count"] = index.Source.Docs source.DocCount = index.Source.Docs
err = orm.Create(nil, &indexMigrationTask) err = orm.Create(nil, &indexMigrationTask)
if err != nil { if err != nil {
return fmt.Errorf("store index migration task error: %w", err) return fmt.Errorf("store index migration task error: %w", err)
@ -1237,23 +1211,13 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
log.Errorf("search task log from es failed, err: %v", err) log.Errorf("search task log from es failed, err: %v", err)
return nil, err return nil, err
} }
var ( cfg := IndexMigrationTaskConfig{}
cfg map[string]interface{} err = getTaskConfig(subTask, &cfg)
ok bool if err != nil {
) log.Errorf("failed to get task config, err: %v", err)
if cfg, ok = subTask.Config.(map[string]interface{}); !ok {
return nil, fmt.Errorf("got wrong config of task %v", *subTask) return nil, fmt.Errorf("got wrong config of task %v", *subTask)
} }
totalDocsVal, err := util.MapStr(cfg).GetValue("source.doc_count") totalDocs := cfg.Source.DocCount
if err != nil {
log.Errorf("failed to get source.doc_count, err: %v", err)
return nil, err
}
totalDocs, err := util.ExtractInt(totalDocsVal)
if err != nil {
log.Errorf("failed to extract source.doc_count, err: %v", err)
return nil, err
}
var ( var (
indexDocs int64 indexDocs int64
@ -1457,7 +1421,8 @@ func (p *DispatcherProcessor) getInstanceTaskState() (map[string]DispatcherState
} }
func (p *DispatcherProcessor) sendMajorTaskNotification(taskItem *task2.Task) { func (p *DispatcherProcessor) sendMajorTaskNotification(taskItem *task2.Task) {
config, err := p.extractTaskConfig(taskItem) config := ClusterMigrationTaskConfig{}
err := getTaskConfig(taskItem, &config)
if err != nil { if err != nil {
log.Errorf("failed to parse config info from major task, id: %s, err: %v", taskItem.ID, err) log.Errorf("failed to parse config info from major task, id: %s, err: %v", taskItem.ID, err)
return return
@ -1500,21 +1465,3 @@ func (p *DispatcherProcessor) sendMajorTaskNotification(taskItem *task2.Task) {
} }
return return
} }
func (p *DispatcherProcessor) extractTaskConfig(taskItem *task2.Task) (*ElasticDataConfig, error) {
origConfig, ok := taskItem.Config.(ElasticDataConfig)
if ok {
return &origConfig, nil
}
rawConfig, ok := taskItem.Config.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("failed to extract configuration from major task, id: %s, type: %T", taskItem.ID, taskItem.Config)
}
config := &ElasticDataConfig{}
err := mapstructure.Decode(rawConfig, config)
if err != nil {
return nil, err
}
return config, nil
}