floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[plugins][migration] status_log -> logging (#40) 

[plugins][migration] status_log -> logging

Co-authored-by: Kassian Sun <kassiansun@outlook.com>
This commit is contained in:
sunjiacheng 2023-03-28 20:12:23 +08:00 committed by medcl
parent df6b4a1737
commit 633d0333f2
1 changed files with 156 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
plugin/migration/pipeline.go
View File

@ -7,6 +7,11 @@ package migration
import ( import (
"errors" "errors"
"fmt" "fmt"
"math"
"strings"
"syscall"
"time"
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"infini.sh/console/model" "infini.sh/console/model"
"infini.sh/framework/core/config" "infini.sh/framework/core/config"
@ -19,10 +24,6 @@ import (
task2 "infini.sh/framework/core/task" task2 "infini.sh/framework/core/task"
"infini.sh/framework/core/util" "infini.sh/framework/core/util"
"infini.sh/framework/modules/elastic/common" "infini.sh/framework/modules/elastic/common"
"math"
"strings"
"syscall"
"time"
) )
type DispatcherProcessor struct { type DispatcherProcessor struct {
@ -59,13 +60,13 @@ func newMigrationDispatcherProcessor(c *config.Config) (pipeline.Processor, erro
ormConfig := common.ORMConfig{} ormConfig := common.ORMConfig{}
ok, err := env.ParseConfig("elastic.orm", &ormConfig) ok, err := env.ParseConfig("elastic.orm", &ormConfig)
if ok && err == nil { if ok && err == nil {
if cfg.IndexName == ""{ if cfg.IndexName == "" {
cfg.IndexName = fmt.Sprintf("%stask", ormConfig.IndexPrefix) cfg.IndexName = fmt.Sprintf("%stask", ormConfig.IndexPrefix)
} }
if cfg.LogIndexName == "" { if cfg.LogIndexName == "" {
cfg.LogIndexName = fmt.Sprintf("%slogs", ormConfig.IndexPrefix) cfg.LogIndexName = fmt.Sprintf("%slogs", ormConfig.IndexPrefix)
} }
}else{ } else {
err = fmt.Errorf("parse config elastic.orm error: %w", err) err = fmt.Errorf("parse config elastic.orm error: %w", err)
log.Error(err) log.Error(err)
return nil, err return nil, err
@ -130,7 +131,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
case task2.StatusPendingStop: case task2.StatusPendingStop:
err = p.handlePendingStopMajorTask(&t) err = p.handlePendingStopMajorTask(&t)
} }
}else if t.Metadata.Labels["business_id"] == "index_migration" { } else if t.Metadata.Labels["business_id"] == "index_migration" {
//handle sub migration task //handle sub migration task
switch t.Status { switch t.Status {
case task2.StatusReady: case task2.StatusReady:
@ -157,7 +158,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
}, },
Message: fmt.Sprintf("failed to handling task [%s]: [%v]", t.ID, err), Message: fmt.Sprintf("failed to handling task [%s]: [%v]", t.ID, err),
Timestamp: time.Now().UTC(), Timestamp: time.Now().UTC(),
},"") }, "")
} }
} }
//es index refresh //es index refresh
@ -165,7 +166,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
} }
} }
func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error {
if taskItem.Metadata.Labels == nil { if taskItem.Metadata.Labels == nil {
return fmt.Errorf("got migration task with empty labels, skip handling: %v", taskItem) return fmt.Errorf("got migration task with empty labels, skip handling: %v", taskItem)
} }
@ -175,7 +176,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
return err return err
} }
taskItem.Metadata.Labels["is_split"] = true taskItem.Metadata.Labels["is_split"] = true
}else{ } else {
taskItem.RetryTimes++ taskItem.RetryTimes++
} }
//update status of subtask to ready //update status of subtask to ready
@ -212,7 +213,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
} }
esClient := elastic.GetClient(p.config.Elasticsearch) esClient := elastic.GetClient(p.config.Elasticsearch)
_, err := esClient.UpdateByQuery(p.config.IndexName, util.MustToJSONBytes(queryDsl) ) _, err := esClient.UpdateByQuery(p.config.IndexName, util.MustToJSONBytes(queryDsl))
if err != nil { if err != nil {
return err return err
} }
@ -233,7 +234,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
return nil return nil
} }
func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) error {
//check whether all pipeline task is stopped or not, then update task status //check whether all pipeline task is stopped or not, then update task status
q := util.MapStr{ q := util.MapStr{
"size": 200, "size": 200,
@ -269,11 +270,11 @@ func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) e
Config: taskItem.Config, Config: taskItem.Config,
Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID), Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID),
Timestamp: time.Now().UTC(), Timestamp: time.Now().UTC(),
},"") }, "")
} }
return nil return nil
} }
func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error {
ts, err := p.getMajorTaskState(taskItem) ts, err := p.getMajorTaskState(taskItem)
if err != nil { if err != nil {
return err return err
@ -291,12 +292,12 @@ func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error
Config: taskItem.Config, Config: taskItem.Config,
Message: fmt.Sprintf("task [%s] is complete", taskItem.ID), Message: fmt.Sprintf("task [%s] is complete", taskItem.ID),
Timestamp: time.Now().UTC(), Timestamp: time.Now().UTC(),
},"") }, "")
} }
return nil return nil
} }
func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error {
state, err := p.getTaskCompleteState(taskItem) state, err := p.getTaskCompleteState(taskItem)
if err != nil { if err != nil {
return err return err
@ -337,7 +338,7 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
} }
if state.Error != "" && state.TotalDocs != state.SuccessDocs { if state.Error != "" && state.TotalDocs != state.SuccessDocs {
taskItem.Status = task2.StatusError taskItem.Status = task2.StatusError
}else { } else {
taskItem.Status = task2.StatusComplete taskItem.Status = task2.StatusComplete
} }
@ -355,8 +356,8 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
}, },
Message: fmt.Sprintf("task [%s] is complete", taskItem.ID), Message: fmt.Sprintf("task [%s] is complete", taskItem.ID),
Timestamp: time.Now().UTC(), Timestamp: time.Now().UTC(),
},"") }, "")
}else{ } else {
if state.RunningPhase == 1 && taskItem.Metadata.Labels["running_phase"] == float64(1) { if state.RunningPhase == 1 && taskItem.Metadata.Labels["running_phase"] == float64(1) {
ptasks, err := p.getPipelineTasks(taskItem.ID) ptasks, err := p.getPipelineTasks(taskItem.ID)
if err != nil { if err != nil {
@ -388,13 +389,13 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
taskItem.Metadata.Labels["running_phase"] = 2 taskItem.Metadata.Labels["running_phase"] = 2
} }
} }
p.saveTaskAndWriteLog(taskItem,nil, "wait_for") p.saveTaskAndWriteLog(taskItem, nil, "wait_for")
} }
} }
return nil return nil
} }
func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) error {
//check whether all pipeline task is stopped or not, then update task status //check whether all pipeline task is stopped or not, then update task status
ptasks, err := p.getPipelineTasks(taskItem.ID) ptasks, err := p.getPipelineTasks(taskItem.ID)
if err != nil { if err != nil {
@ -409,7 +410,7 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
"size": len(taskIDs), "size": len(taskIDs),
"sort": []util.MapStr{ "sort": []util.MapStr{
{ {
"payload.pipeline.status_log.steps": util.MapStr{ "payload.pipeline.logging.steps": util.MapStr{
"order": "desc", "order": "desc",
}, },
}, },
@ -430,9 +431,9 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
if len(searchRes.Hits.Hits) == 0 { if len(searchRes.Hits.Hits) == 0 {
return nil return nil
} }
MainLoop: MainLoop:
for _, hit := range searchRes.Hits.Hits { for _, hit := range searchRes.Hits.Hits {
status, _ := util.MapStr(hit.Source).GetValue("payload.pipeline.status_log.status") status, _ := util.MapStr(hit.Source).GetValue("payload.pipeline.logging.status")
if status != "STOPPED" { if status != "STOPPED" {
//call instance api to stop scroll/bulk_indexing pipeline task //call instance api to stop scroll/bulk_indexing pipeline task
if instID, ok := taskItem.Metadata.Labels["execution_instance_id"].(string); ok { if instID, ok := taskItem.Metadata.Labels["execution_instance_id"].(string); ok {
@ -446,7 +447,7 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
for _, pipelineID := range taskIDs { for _, pipelineID := range taskIDs {
err = inst.StopPipelineWithTimeout(pipelineID, time.Second) err = inst.StopPipelineWithTimeout(pipelineID, time.Second)
if err != nil { if err != nil {
if !errors.Is(err, syscall.ECONNREFUSED) && !strings.Contains(err.Error(), "task not found"){ if !errors.Is(err, syscall.ECONNREFUSED) && !strings.Contains(err.Error(), "task not found") {
hasStopped = false hasStopped = false
break break
} }
@ -500,11 +501,11 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
Config: taskItem.Config, Config: taskItem.Config,
Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID), Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID),
Timestamp: time.Now().UTC(), Timestamp: time.Now().UTC(),
},"") }, "")
return nil return nil
} }
func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
if taskItem.Metadata.Labels == nil { if taskItem.Metadata.Labels == nil {
return fmt.Errorf("empty labels") return fmt.Errorf("empty labels")
} }
@ -521,11 +522,11 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
for i, t := range ptasks { for i, t := range ptasks {
if t.Metadata.Labels != nil { if t.Metadata.Labels != nil {
if cfg, ok := ptasks[i].Config.(map[string]interface{}); ok { if cfg, ok := ptasks[i].Config.(map[string]interface{}); ok {
util.MapStr(cfg).Put("labels.retry_no", taskItem.RetryTimes + 1) util.MapStr(cfg).Put("labels.retry_no", taskItem.RetryTimes+1)
} }
if t.Metadata.Labels["pipeline_id"] == "es_scroll" { if t.Metadata.Labels["pipeline_id"] == "es_scroll" {
scrollTask = &ptasks[i] scrollTask = &ptasks[i]
}else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" { } else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" {
bulkTask = &ptasks[i] bulkTask = &ptasks[i]
} }
} }
@ -534,7 +535,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
return fmt.Errorf("es_scroll or bulk_indexing pipeline task not found") return fmt.Errorf("es_scroll or bulk_indexing pipeline task not found")
} }
taskItem.RetryTimes++ taskItem.RetryTimes++
}else { } else {
//split task to scroll/bulk_indexing pipeline and then persistent //split task to scroll/bulk_indexing pipeline and then persistent
var pids []string var pids []string
pids = append(pids, taskItem.ParentId...) pids = append(pids, taskItem.ParentId...)
@ -727,7 +728,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
if err != nil { if err != nil {
return fmt.Errorf("create bulk_indexing pipeline task error: %w", err) return fmt.Errorf("create bulk_indexing pipeline task error: %w", err)
} }
}else{ } else {
err = orm.Update(nil, scrollTask) err = orm.Update(nil, scrollTask)
if err != nil { if err != nil {
return fmt.Errorf("update scroll pipeline task error: %w", err) return fmt.Errorf("update scroll pipeline task error: %w", err)
@ -766,7 +767,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
return nil return nil
} }
func getMapValue(m util.MapStr, key string) interface{}{ func getMapValue(m util.MapStr, key string) interface{} {
v, _ := m.GetValue(key) v, _ := m.GetValue(key)
return v return v
} }
@ -819,7 +820,7 @@ func (p *DispatcherProcessor) getPreferenceInstance(majorTaskID string) (instanc
_, err = orm.Get(&instance) _, err = orm.Get(&instance)
return return
} }
func (p *DispatcherProcessor) getMigrationTasks(size int)([]task2.Task, error){ func (p *DispatcherProcessor) getMigrationTasks(size int) ([]task2.Task, error) {
majorTaskQ := util.MapStr{ majorTaskQ := util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
@ -876,8 +877,8 @@ func (p *DispatcherProcessor) getMigrationTasks(size int)([]task2.Task, error){
func (p *DispatcherProcessor) saveTaskAndWriteLog(taskItem *task2.Task, logItem *task2.Log, refresh string) { func (p *DispatcherProcessor) saveTaskAndWriteLog(taskItem *task2.Task, logItem *task2.Log, refresh string) {
esClient := elastic.GetClient(p.config.Elasticsearch) esClient := elastic.GetClient(p.config.Elasticsearch)
_, err := esClient.Index(p.config.IndexName,"", taskItem.ID, taskItem, refresh ) _, err := esClient.Index(p.config.IndexName, "", taskItem.ID, taskItem, refresh)
if err != nil{ if err != nil {
log.Error(err) log.Error(err)
} }
if logItem != nil { if logItem != nil {
@ -953,7 +954,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
if v, ok := index.RawFilter.(string); ok { if v, ok := index.RawFilter.(string); ok {
source["query_string"] = v source["query_string"] = v
}else{ } else {
source["query_dsl"] = index.RawFilter source["query_dsl"] = index.RawFilter
if index.Source.DocType != "" { if index.Source.DocType != "" {
if index.Target.DocType != "" { if index.Target.DocType != "" {
@ -976,7 +977,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
"must": must, "must": must,
}, },
} }
}else{ } else {
if esSourceClient.GetMajorVersion() >= 8 { if esSourceClient.GetMajorVersion() >= 8 {
source["type_rename"] = util.MapStr{ source["type_rename"] = util.MapStr{
"*": index.Target.DocType, "*": index.Target.DocType,
@ -1045,7 +1046,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
if err != nil { if err != nil {
return err return err
} }
if partitions == nil || len(partitions) == 0{ if partitions == nil || len(partitions) == 0 {
return fmt.Errorf("empty data with filter: %s", util.MustToJSON(index.RawFilter)) return fmt.Errorf("empty data with filter: %s", util.MustToJSON(index.RawFilter))
} }
var ( var (
@ -1064,8 +1065,8 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
"step": index.Partition.Step, "step": index.Partition.Step,
"partition_id": partitionID, "partition_id": partitionID,
} }
for k, v := range source{ for k, v := range source {
if k == "query_string"{ if k == "query_string" {
continue continue
} }
partitionSource[k] = v partitionSource[k] = v
@ -1075,7 +1076,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
if partition.Other { if partition.Other {
must = append(must, partition.Filter) must = append(must, partition.Filter)
}else{ } else {
must = append(must, util.MapStr{ must = append(must, util.MapStr{
"range": util.MapStr{ "range": util.MapStr{
index.Partition.FieldName: util.MapStr{ index.Partition.FieldName: util.MapStr{
@ -1126,7 +1127,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
} }
} }
}else{ } else {
source["doc_count"] = index.Source.Docs source["doc_count"] = index.Source.Docs
err = orm.Create(nil, &indexMigrationTask) err = orm.Create(nil, &indexMigrationTask)
if err != nil { if err != nil {
@ -1137,7 +1138,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
return nil return nil
} }
func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task, error){ func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task, error) {
queryDsl := util.MapStr{ queryDsl := util.MapStr{
"size": 2, "size": 2,
"query": util.MapStr{ "query": util.MapStr{
@ -1157,7 +1158,7 @@ func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task,
return p.getTasks(queryDsl) return p.getTasks(queryDsl)
} }
func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error){ func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error) {
esClient := elastic.GetClient(p.config.Elasticsearch) esClient := elastic.GetClient(p.config.Elasticsearch)
res, err := esClient.SearchWithRawQueryDSL(p.config.IndexName, util.MustToJSONBytes(query)) res, err := esClient.SearchWithRawQueryDSL(p.config.IndexName, util.MustToJSONBytes(query))
if err != nil { if err != nil {
@ -1182,7 +1183,7 @@ func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error){
return migrationTasks, nil return migrationTasks, nil
} }
func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCompleteState, error){ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCompleteState, error) {
ptasks, err := p.getPipelineTasks(subTask.ID) ptasks, err := p.getPipelineTasks(subTask.ID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -1228,7 +1229,6 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
}, },
}, },
}, },
}, },
} }
esClient := elastic.GetClient(p.config.Elasticsearch) esClient := elastic.GetClient(p.config.Elasticsearch)
@ -1263,7 +1263,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
state.IsComplete = true state.IsComplete = true
state.ClearPipeline = true state.ClearPipeline = true
} }
for _, key := range []string{"payload.pipeline.logging.context.bulk_indexing.success.count", "payload.pipeline.logging.context.bulk_indexing.failure.count", "payload.pipeline.logging.context.bulk_indexing.invalid.count"}{ for _, key := range []string{"payload.pipeline.logging.context.bulk_indexing.success.count", "payload.pipeline.logging.context.bulk_indexing.failure.count", "payload.pipeline.logging.context.bulk_indexing.invalid.count"} {
v, err := util.MapStr(hit.Source).GetValue(key) v, err := util.MapStr(hit.Source).GetValue(key)
if err == nil { if err == nil {
if fv, ok := v.(float64); ok { if fv, ok := v.(float64); ok {
@ -1273,7 +1273,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
state.SuccessDocs = successDocs state.SuccessDocs = successDocs
} }
} }
}else{ } else {
break break
} }
} }
@ -1293,7 +1293,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
if state.Error == "" { if state.Error == "" {
if successDocs > 0 { if successDocs > 0 {
state.Error = "partial complete" state.Error = "partial complete"
}else{ } else {
state.Error = "invalid request" state.Error = "invalid request"
} }
} }
@ -1320,7 +1320,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
return &state, nil return &state, nil
} }
func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskState MajorTaskState, err error){ func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskState MajorTaskState, err error) {
query := util.MapStr{ query := util.MapStr{
"size": 0, "size": 0,
"aggs": util.MapStr{ "aggs": util.MapStr{
@ -1354,11 +1354,10 @@ func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskStat
}, },
}, },
}, },
}, },
} }
esClient := elastic.GetClient(p.config.Elasticsearch) esClient := elastic.GetClient(p.config.Elasticsearch)
res, err := esClient.SearchWithRawQueryDSL( p.config.IndexName, util.MustToJSONBytes(query)) res, err := esClient.SearchWithRawQueryDSL(p.config.IndexName, util.MustToJSONBytes(query))
if err != nil { if err != nil {
return taskState, err return taskState, err
} }
@ -1379,13 +1378,13 @@ func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskStat
} }
if hasError { if hasError {
taskState.Status = task2.StatusError taskState.Status = task2.StatusError
}else { } else {
taskState.Status = task2.StatusComplete taskState.Status = task2.StatusComplete
} }
return taskState, nil return taskState, nil
} }
func (p *DispatcherProcessor) getInstanceTaskState()(map[string]DispatcherState, error){ func (p *DispatcherProcessor) getInstanceTaskState() (map[string]DispatcherState, error) {
query := util.MapStr{ query := util.MapStr{
"size": 0, "size": 0,
"aggs": util.MapStr{ "aggs": util.MapStr{