floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix calc migration task progress info with a large number of sub task

This commit is contained in:
liugq 2023-08-14 10:41:17 +08:00
parent c8aab5c566
commit ef36af329f
1 changed files with 123 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
plugin/task_manager/migration_api.go
View File

@ -76,7 +76,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
for i, index := range taskConfig.Indices { for i, index := range taskConfig.Indices {
indexName := index.Source.GetUniqueIndexName() indexName := index.Source.GetUniqueIndexName()
count := indexState[indexName].IndexDocs count := indexState[indexName].IndexDocs
sourceDocs := indexState[indexName].SourceDocs sourceDocs := index.Source.Docs
var percent float64 var percent float64
if sourceDocs <= 0 { if sourceDocs <= 0 {
percent = 100 percent = 100
@ -86,7 +86,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
percent = 100 percent = 100
} }
} }
taskConfig.Indices[i].Source.Docs = sourceDocs //taskConfig.Indices[i].Source.Docs = sourceDocs
taskConfig.Indices[i].Target.Docs = count taskConfig.Indices[i].Target.Docs = count
taskConfig.Indices[i].Percent = util.ToFixed(percent, 2) taskConfig.Indices[i].Percent = util.ToFixed(percent, 2)
taskConfig.Indices[i].ErrorPartitions = indexState[indexName].ErrorPartitions taskConfig.Indices[i].ErrorPartitions = indexState[indexName].ErrorPartitions
@ -257,8 +257,26 @@ We count data from two sources:
- realtime bulk indexing info is only available for running index_migrations - realtime bulk indexing info is only available for running index_migrations
*/ */
func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_model.ClusterMigrationTaskState, indexState map[string]MigrationIndexStateInfo, err error) { func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_model.ClusterMigrationTaskState, indexState map[string]MigrationIndexStateInfo, err error) {
var pipelineTaskIDs = map[string][]string{}
var pipelineIndexNames = map[string]string{}
indexState = map[string]MigrationIndexStateInfo{}
const size = 500
var (
from = -size
hasMore = true
)
for hasMore {
from += size
taskQuery := util.MapStr{ taskQuery := util.MapStr{
"size": 500, "from": from,
"size": size,
"sort": []util.MapStr{
{
"created": util.MapStr{
"order": "asc",
},
},
},
"query": util.MapStr{ "query": util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
@ -284,9 +302,11 @@ func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_m
if err != nil { if err != nil {
return taskStats, indexState, err return taskStats, indexState, err
} }
if len(subTasks) < size {
hasMore = false
}
var indexMigrationTaskIDs []string var indexMigrationTaskIDs []string
indexState = map[string]MigrationIndexStateInfo{}
for _, subTask := range subTasks { for _, subTask := range subTasks {
taskLabels := util.MapStr(subTask.Metadata.Labels) taskLabels := util.MapStr(subTask.Metadata.Labels)
indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name")
@ -323,11 +343,11 @@ func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_m
} }
if len(indexMigrationTaskIDs) == 0 { if len(indexMigrationTaskIDs) == 0 {
return taskStats, indexState, nil continue
} }
taskQuery = util.MapStr{ taskQuery = util.MapStr{
"size": 500, "size": len(indexMigrationTaskIDs),
"query": util.MapStr{ "query": util.MapStr{
"bool": util.MapStr{ "bool": util.MapStr{
"must": []util.MapStr{ "must": []util.MapStr{
@ -352,8 +372,6 @@ func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_m
return taskStats, indexState, err return taskStats, indexState, err
} }
var pipelineTaskIDs = map[string][]string{}
var pipelineIndexNames = map[string]string{}
for _, subTask := range subTasks { for _, subTask := range subTasks {
taskLabels := util.MapStr(subTask.Metadata.Labels) taskLabels := util.MapStr(subTask.Metadata.Labels)
indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name") indexName := migration_util.GetMapStringValue(taskLabels, "unique_index_name")
@ -367,6 +385,7 @@ func (h *APIHandler) getMigrationMajorTaskInfo(id string) (taskStats migration_m
pipelineTaskIDs[instID] = append(pipelineTaskIDs[instID], subTask.ID) pipelineTaskIDs[instID] = append(pipelineTaskIDs[instID], subTask.ID)
} }
} }
}
pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs) pipelineContexts := h.getChildPipelineInfosFromGateway(pipelineTaskIDs)
for pipelineID, pipelineContext := range pipelineContexts { for pipelineID, pipelineContext := range pipelineContexts {