[plugins][migration] status_log -> logging (#40)
[plugins][migration] status_log -> logging Co-authored-by: Kassian Sun <kassiansun@outlook.com>
This commit is contained in:
sunjiacheng
medcl
parent
df6b4a1737
commit
633d0333f2
|
|
@ -7,6 +7,11 @@ package migration
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "github.com/cihub/seelog"
|
||||
"infini.sh/console/model"
|
||||
"infini.sh/framework/core/config"
|
||||
|
|
@ -19,10 +24,6 @@ import (
|
|||
task2 "infini.sh/framework/core/task"
|
||||
"infini.sh/framework/core/util"
|
||||
"infini.sh/framework/modules/elastic/common"
|
||||
"math"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type DispatcherProcessor struct {
|
||||
|
|
@ -59,13 +60,13 @@ func newMigrationDispatcherProcessor(c *config.Config) (pipeline.Processor, erro
|
|||
ormConfig := common.ORMConfig{}
|
||||
ok, err := env.ParseConfig("elastic.orm", &ormConfig)
|
||||
if ok && err == nil {
|
||||
if cfg.IndexName == ""{
|
||||
if cfg.IndexName == "" {
|
||||
cfg.IndexName = fmt.Sprintf("%stask", ormConfig.IndexPrefix)
|
||||
}
|
||||
if cfg.LogIndexName == "" {
|
||||
cfg.LogIndexName = fmt.Sprintf("%slogs", ormConfig.IndexPrefix)
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
err = fmt.Errorf("parse config elastic.orm error: %w", err)
|
||||
log.Error(err)
|
||||
return nil, err
|
||||
|
|
@ -130,7 +131,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
|
|||
case task2.StatusPendingStop:
|
||||
err = p.handlePendingStopMajorTask(&t)
|
||||
}
|
||||
}else if t.Metadata.Labels["business_id"] == "index_migration" {
|
||||
} else if t.Metadata.Labels["business_id"] == "index_migration" {
|
||||
//handle sub migration task
|
||||
switch t.Status {
|
||||
case task2.StatusReady:
|
||||
|
|
@ -157,7 +158,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
|
|||
},
|
||||
Message: fmt.Sprintf("failed to handling task [%s]: [%v]", t.ID, err),
|
||||
Timestamp: time.Now().UTC(),
|
||||
},"")
|
||||
}, "")
|
||||
}
|
||||
}
|
||||
//es index refresh
|
||||
|
|
@ -165,7 +166,7 @@ func (p *DispatcherProcessor) Process(ctx *pipeline.Context) error {
|
|||
}
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error {
|
||||
if taskItem.Metadata.Labels == nil {
|
||||
return fmt.Errorf("got migration task with empty labels, skip handling: %v", taskItem)
|
||||
}
|
||||
|
|
@ -175,7 +176,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
|
|||
return err
|
||||
}
|
||||
taskItem.Metadata.Labels["is_split"] = true
|
||||
}else{
|
||||
} else {
|
||||
taskItem.RetryTimes++
|
||||
}
|
||||
//update status of subtask to ready
|
||||
|
|
@ -212,7 +213,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
|
|||
}
|
||||
|
||||
esClient := elastic.GetClient(p.config.Elasticsearch)
|
||||
_, err := esClient.UpdateByQuery(p.config.IndexName, util.MustToJSONBytes(queryDsl) )
|
||||
_, err := esClient.UpdateByQuery(p.config.IndexName, util.MustToJSONBytes(queryDsl))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -233,7 +234,7 @@ func (p *DispatcherProcessor) handleReadyMajorTask(taskItem *task2.Task) error{
|
|||
return nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) error {
|
||||
//check whether all pipeline task is stopped or not, then update task status
|
||||
q := util.MapStr{
|
||||
"size": 200,
|
||||
|
|
@ -269,11 +270,11 @@ func (p *DispatcherProcessor) handlePendingStopMajorTask(taskItem *task2.Task) e
|
|||
Config: taskItem.Config,
|
||||
Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID),
|
||||
Timestamp: time.Now().UTC(),
|
||||
},"")
|
||||
}, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error {
|
||||
ts, err := p.getMajorTaskState(taskItem)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -291,12 +292,12 @@ func (p *DispatcherProcessor) handleRunningMajorTask(taskItem *task2.Task) error
|
|||
Config: taskItem.Config,
|
||||
Message: fmt.Sprintf("task [%s] is complete", taskItem.ID),
|
||||
Timestamp: time.Now().UTC(),
|
||||
},"")
|
||||
}, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error {
|
||||
state, err := p.getTaskCompleteState(taskItem)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -337,7 +338,7 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
|
|||
}
|
||||
if state.Error != "" && state.TotalDocs != state.SuccessDocs {
|
||||
taskItem.Status = task2.StatusError
|
||||
}else {
|
||||
} else {
|
||||
taskItem.Status = task2.StatusComplete
|
||||
}
|
||||
|
||||
|
|
@ -355,8 +356,8 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
|
|||
},
|
||||
Message: fmt.Sprintf("task [%s] is complete", taskItem.ID),
|
||||
Timestamp: time.Now().UTC(),
|
||||
},"")
|
||||
}else{
|
||||
}, "")
|
||||
} else {
|
||||
if state.RunningPhase == 1 && taskItem.Metadata.Labels["running_phase"] == float64(1) {
|
||||
ptasks, err := p.getPipelineTasks(taskItem.ID)
|
||||
if err != nil {
|
||||
|
|
@ -388,13 +389,13 @@ func (p *DispatcherProcessor) handleRunningSubTask(taskItem *task2.Task) error{
|
|||
taskItem.Metadata.Labels["running_phase"] = 2
|
||||
}
|
||||
}
|
||||
p.saveTaskAndWriteLog(taskItem,nil, "wait_for")
|
||||
p.saveTaskAndWriteLog(taskItem, nil, "wait_for")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) error {
|
||||
//check whether all pipeline task is stopped or not, then update task status
|
||||
ptasks, err := p.getPipelineTasks(taskItem.ID)
|
||||
if err != nil {
|
||||
|
|
@ -409,7 +410,7 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
|
|||
"size": len(taskIDs),
|
||||
"sort": []util.MapStr{
|
||||
{
|
||||
"payload.pipeline.status_log.steps": util.MapStr{
|
||||
"payload.pipeline.logging.steps": util.MapStr{
|
||||
"order": "desc",
|
||||
},
|
||||
},
|
||||
|
|
@ -430,9 +431,9 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
|
|||
if len(searchRes.Hits.Hits) == 0 {
|
||||
return nil
|
||||
}
|
||||
MainLoop:
|
||||
MainLoop:
|
||||
for _, hit := range searchRes.Hits.Hits {
|
||||
status, _ := util.MapStr(hit.Source).GetValue("payload.pipeline.status_log.status")
|
||||
status, _ := util.MapStr(hit.Source).GetValue("payload.pipeline.logging.status")
|
||||
if status != "STOPPED" {
|
||||
//call instance api to stop scroll/bulk_indexing pipeline task
|
||||
if instID, ok := taskItem.Metadata.Labels["execution_instance_id"].(string); ok {
|
||||
|
|
@ -446,7 +447,7 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
|
|||
for _, pipelineID := range taskIDs {
|
||||
err = inst.StopPipelineWithTimeout(pipelineID, time.Second)
|
||||
if err != nil {
|
||||
if !errors.Is(err, syscall.ECONNREFUSED) && !strings.Contains(err.Error(), "task not found"){
|
||||
if !errors.Is(err, syscall.ECONNREFUSED) && !strings.Contains(err.Error(), "task not found") {
|
||||
hasStopped = false
|
||||
break
|
||||
}
|
||||
|
|
@ -500,11 +501,11 @@ func (p *DispatcherProcessor) handlePendingStopSubTask(taskItem *task2.Task) err
|
|||
Config: taskItem.Config,
|
||||
Message: fmt.Sprintf("task [%s] is stopped", taskItem.ID),
|
||||
Timestamp: time.Now().UTC(),
|
||||
},"")
|
||||
}, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
|
||||
func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error {
|
||||
if taskItem.Metadata.Labels == nil {
|
||||
return fmt.Errorf("empty labels")
|
||||
}
|
||||
|
|
@ -521,11 +522,11 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
|
|||
for i, t := range ptasks {
|
||||
if t.Metadata.Labels != nil {
|
||||
if cfg, ok := ptasks[i].Config.(map[string]interface{}); ok {
|
||||
util.MapStr(cfg).Put("labels.retry_no", taskItem.RetryTimes + 1)
|
||||
util.MapStr(cfg).Put("labels.retry_no", taskItem.RetryTimes+1)
|
||||
}
|
||||
if t.Metadata.Labels["pipeline_id"] == "es_scroll" {
|
||||
scrollTask = &ptasks[i]
|
||||
}else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" {
|
||||
} else if t.Metadata.Labels["pipeline_id"] == "bulk_indexing" {
|
||||
bulkTask = &ptasks[i]
|
||||
}
|
||||
}
|
||||
|
|
@ -534,7 +535,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
|
|||
return fmt.Errorf("es_scroll or bulk_indexing pipeline task not found")
|
||||
}
|
||||
taskItem.RetryTimes++
|
||||
}else {
|
||||
} else {
|
||||
//split task to scroll/bulk_indexing pipeline and then persistent
|
||||
var pids []string
|
||||
pids = append(pids, taskItem.ParentId...)
|
||||
|
|
@ -727,7 +728,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
|
|||
if err != nil {
|
||||
return fmt.Errorf("create bulk_indexing pipeline task error: %w", err)
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
err = orm.Update(nil, scrollTask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update scroll pipeline task error: %w", err)
|
||||
|
|
@ -766,7 +767,7 @@ func (p *DispatcherProcessor) handleReadySubTask(taskItem *task2.Task) error{
|
|||
return nil
|
||||
}
|
||||
|
||||
func getMapValue(m util.MapStr, key string) interface{}{
|
||||
func getMapValue(m util.MapStr, key string) interface{} {
|
||||
v, _ := m.GetValue(key)
|
||||
return v
|
||||
}
|
||||
|
|
@ -819,7 +820,7 @@ func (p *DispatcherProcessor) getPreferenceInstance(majorTaskID string) (instanc
|
|||
_, err = orm.Get(&instance)
|
||||
return
|
||||
}
|
||||
func (p *DispatcherProcessor) getMigrationTasks(size int)([]task2.Task, error){
|
||||
func (p *DispatcherProcessor) getMigrationTasks(size int) ([]task2.Task, error) {
|
||||
majorTaskQ := util.MapStr{
|
||||
"bool": util.MapStr{
|
||||
"must": []util.MapStr{
|
||||
|
|
@ -876,8 +877,8 @@ func (p *DispatcherProcessor) getMigrationTasks(size int)([]task2.Task, error){
|
|||
|
||||
func (p *DispatcherProcessor) saveTaskAndWriteLog(taskItem *task2.Task, logItem *task2.Log, refresh string) {
|
||||
esClient := elastic.GetClient(p.config.Elasticsearch)
|
||||
_, err := esClient.Index(p.config.IndexName,"", taskItem.ID, taskItem, refresh )
|
||||
if err != nil{
|
||||
_, err := esClient.Index(p.config.IndexName, "", taskItem.ID, taskItem, refresh)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
if logItem != nil {
|
||||
|
|
@ -953,7 +954,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
|
||||
if v, ok := index.RawFilter.(string); ok {
|
||||
source["query_string"] = v
|
||||
}else{
|
||||
} else {
|
||||
source["query_dsl"] = index.RawFilter
|
||||
if index.Source.DocType != "" {
|
||||
if index.Target.DocType != "" {
|
||||
|
|
@ -976,7 +977,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
"must": must,
|
||||
},
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
if esSourceClient.GetMajorVersion() >= 8 {
|
||||
source["type_rename"] = util.MapStr{
|
||||
"*": index.Target.DocType,
|
||||
|
|
@ -1045,7 +1046,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if partitions == nil || len(partitions) == 0{
|
||||
if partitions == nil || len(partitions) == 0 {
|
||||
return fmt.Errorf("empty data with filter: %s", util.MustToJSON(index.RawFilter))
|
||||
}
|
||||
var (
|
||||
|
|
@ -1064,8 +1065,8 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
"step": index.Partition.Step,
|
||||
"partition_id": partitionID,
|
||||
}
|
||||
for k, v := range source{
|
||||
if k == "query_string"{
|
||||
for k, v := range source {
|
||||
if k == "query_string" {
|
||||
continue
|
||||
}
|
||||
partitionSource[k] = v
|
||||
|
|
@ -1075,7 +1076,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
|
||||
if partition.Other {
|
||||
must = append(must, partition.Filter)
|
||||
}else{
|
||||
} else {
|
||||
must = append(must, util.MapStr{
|
||||
"range": util.MapStr{
|
||||
index.Partition.FieldName: util.MapStr{
|
||||
|
|
@ -1126,7 +1127,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
}
|
||||
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
source["doc_count"] = index.Source.Docs
|
||||
err = orm.Create(nil, &indexMigrationTask)
|
||||
if err != nil {
|
||||
|
|
@ -1137,7 +1138,7 @@ func (p *DispatcherProcessor) splitMajorMigrationTask(taskItem *task2.Task) erro
|
|||
return nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task, error){
|
||||
func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task, error) {
|
||||
queryDsl := util.MapStr{
|
||||
"size": 2,
|
||||
"query": util.MapStr{
|
||||
|
|
@ -1157,7 +1158,7 @@ func (p *DispatcherProcessor) getPipelineTasks(subTaskID string) ([]task2.Task,
|
|||
return p.getTasks(queryDsl)
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error){
|
||||
func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error) {
|
||||
esClient := elastic.GetClient(p.config.Elasticsearch)
|
||||
res, err := esClient.SearchWithRawQueryDSL(p.config.IndexName, util.MustToJSONBytes(query))
|
||||
if err != nil {
|
||||
|
|
@ -1182,7 +1183,7 @@ func (p *DispatcherProcessor) getTasks(query interface{}) ([]task2.Task, error){
|
|||
return migrationTasks, nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCompleteState, error){
|
||||
func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCompleteState, error) {
|
||||
ptasks, err := p.getPipelineTasks(subTask.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -1228,7 +1229,6 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
|
|||
},
|
||||
},
|
||||
},
|
||||
|
||||
},
|
||||
}
|
||||
esClient := elastic.GetClient(p.config.Elasticsearch)
|
||||
|
|
@ -1263,7 +1263,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
|
|||
state.IsComplete = true
|
||||
state.ClearPipeline = true
|
||||
}
|
||||
for _, key := range []string{"payload.pipeline.logging.context.bulk_indexing.success.count", "payload.pipeline.logging.context.bulk_indexing.failure.count", "payload.pipeline.logging.context.bulk_indexing.invalid.count"}{
|
||||
for _, key := range []string{"payload.pipeline.logging.context.bulk_indexing.success.count", "payload.pipeline.logging.context.bulk_indexing.failure.count", "payload.pipeline.logging.context.bulk_indexing.invalid.count"} {
|
||||
v, err := util.MapStr(hit.Source).GetValue(key)
|
||||
if err == nil {
|
||||
if fv, ok := v.(float64); ok {
|
||||
|
|
@ -1273,7 +1273,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
|
|||
state.SuccessDocs = successDocs
|
||||
}
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -1293,7 +1293,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
|
|||
if state.Error == "" {
|
||||
if successDocs > 0 {
|
||||
state.Error = "partial complete"
|
||||
}else{
|
||||
} else {
|
||||
state.Error = "invalid request"
|
||||
}
|
||||
}
|
||||
|
|
@ -1320,7 +1320,7 @@ func (p *DispatcherProcessor) getTaskCompleteState(subTask *task2.Task) (*TaskCo
|
|||
return &state, nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskState MajorTaskState, err error){
|
||||
func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskState MajorTaskState, err error) {
|
||||
query := util.MapStr{
|
||||
"size": 0,
|
||||
"aggs": util.MapStr{
|
||||
|
|
@ -1354,11 +1354,10 @@ func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskStat
|
|||
},
|
||||
},
|
||||
},
|
||||
|
||||
},
|
||||
}
|
||||
esClient := elastic.GetClient(p.config.Elasticsearch)
|
||||
res, err := esClient.SearchWithRawQueryDSL( p.config.IndexName, util.MustToJSONBytes(query))
|
||||
res, err := esClient.SearchWithRawQueryDSL(p.config.IndexName, util.MustToJSONBytes(query))
|
||||
if err != nil {
|
||||
return taskState, err
|
||||
}
|
||||
|
|
@ -1379,13 +1378,13 @@ func (p *DispatcherProcessor) getMajorTaskState(majorTask *task2.Task) (taskStat
|
|||
}
|
||||
if hasError {
|
||||
taskState.Status = task2.StatusError
|
||||
}else {
|
||||
} else {
|
||||
taskState.Status = task2.StatusComplete
|
||||
}
|
||||
return taskState, nil
|
||||
}
|
||||
|
||||
func (p *DispatcherProcessor) getInstanceTaskState()(map[string]DispatcherState, error){
|
||||
func (p *DispatcherProcessor) getInstanceTaskState() (map[string]DispatcherState, error) {
|
||||
query := util.MapStr{
|
||||
"size": 0,
|
||||
"aggs": util.MapStr{
|
||||
|
|
|
|||
Loading…
Reference in New Issue