floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[migration] batch task by task type

This commit is contained in:
Kassian Sun 2023-04-21 18:21:01 +08:00
parent 492bd7dd66
commit 651c86feff
1 changed files with 67 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
plugin/migration/pipeline.go
View File

@ -111,7 +111,8 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
if ctx.IsCanceled() {
return nil
}
tasks, err := p.getMigrationTasks(p.config.TaskBatchSize)
for _, taskType := range []string{"cluster_migration", "index_migration", "pipeline"} {
tasks, err := p.getMigrationTasks(taskType, p.config.TaskBatchSize)
if err != nil {
log.Errorf("failed to get migration tasks, err: %v", err)
return err
@ -175,6 +176,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
}, fmt.Sprintf("error handling task [%s]", t.ID))
}
}
}
//es index refresh
time.Sleep(time.Millisecond * 1200)
}
@ -741,7 +743,7 @@ func (p *DispatcherProcessor) getPreferenceInstance(majorTaskID string) (instanc
return
}
func (p *DispatcherProcessor) getMigrationTasks(size int) ([]task2.Task, error) {
func (p *DispatcherProcessor) getMigrationTasks(taskType string, size int) ([]task2.Task, error) {
queryDsl := util.MapStr{
"size": size,
"sort": []util.MapStr{
@ -759,6 +761,13 @@ func (p *DispatcherProcessor) getMigrationTasks(size int) ([]task2.Task, error)
"status": []string{task2.StatusReady, task2.StatusRunning, task2.StatusPendingStop},
},
},
{
"term": util.MapStr{
"metadata.type": util.MapStr{
"value": taskType,
},
},
},
},
},
},