floraachy
/
console
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migration skip exists docs with bulk operation

This commit is contained in:
hardy 2024-02-05 10:09:56 +08:00
parent a338af5c9a
commit 855c4849cb
No known key found for this signature in database
GPG Key ID: D1ED7F7A9ED520C3
2 changed files with 12 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
plugin/task_manager/cluster_migration/cluster_migration.go
View File

@ -200,15 +200,16 @@ func (p *processor) splitMajorMigrationTask(taskItem *task.Task) error {
IdleTimeoutInSeconds: clusterMigrationTask.Settings.Bulk.IdleTimeoutInSeconds, IdleTimeoutInSeconds: clusterMigrationTask.Settings.Bulk.IdleTimeoutInSeconds,
SliceSize: clusterMigrationTask.Settings.Bulk.SliceSize, SliceSize: clusterMigrationTask.Settings.Bulk.SliceSize,
Compress: clusterMigrationTask.Settings.Bulk.Compress, Compress: clusterMigrationTask.Settings.Bulk.Compress,
Operation: clusterMigrationTask.Settings.Bulk.Operation,
}, },
} }
if index.Partition != nil { if index.Partition != nil {
partitionQ := &elastic.PartitionQuery{ partitionQ := &elastic.PartitionQuery{
IndexName: index.Source.Name, IndexName: index.Source.Name,
FieldName: index.Partition.FieldName, FieldName: index.Partition.FieldName,
FieldType: index.Partition.FieldType, FieldType: index.Partition.FieldType,
Step: index.Partition.Step, Step: index.Partition.Step,
UseEvenStrategy: index.Partition.UseEvenStrategy, UseEvenStrategy: index.Partition.UseEvenStrategy,
} }
if source.QueryDSL != nil { if source.QueryDSL != nil {

13
plugin/task_manager/model/pipeline.go
View File

@ -23,12 +23,13 @@ type IndexDiffConfig struct {
// tunable `bulk_indexing` configurations // tunable `bulk_indexing` configurations
type BulkIndexingConfig struct { type BulkIndexingConfig struct {
Docs int `json:"docs"` Docs int `json:"docs"`
StoreSizeInMB int `json:"store_size_in_mb"` StoreSizeInMB int `json:"store_size_in_mb"`
MaxWorkerSize int `json:"max_worker_size"` MaxWorkerSize int `json:"max_worker_size"`
IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"` IdleTimeoutInSeconds int `json:"idle_timeout_in_seconds"`
SliceSize int `json:"slice_size"` SliceSize int `json:"slice_size"`
Compress bool `json:"compress"` Compress bool `json:"compress"`
Operation string `json:"operation"`
} }
type PipelineTaskLoggingConfig struct { type PipelineTaskLoggingConfig struct {