From e97d66bf3085903bced9d833b5ed1549e3ebc518 Mon Sep 17 00:00:00 2001 From: "sylor_huang@126.com" Date: Wed, 15 Jul 2020 15:43:02 +0800 Subject: [PATCH 1/3] add exists check --- app/controllers/sync_forge_controller.rb | 36 ++++++++--- app/jobs/sync_projects_job.rb | 78 +++++++++++++++++------- 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/app/controllers/sync_forge_controller.rb b/app/controllers/sync_forge_controller.rb index eb42f607..adfe6900 100644 --- a/app/controllers/sync_forge_controller.rb +++ b/app/controllers/sync_forge_controller.rb @@ -5,14 +5,31 @@ class SyncForgeController < ApplicationController ActiveRecord::Base.transaction do params.permit! sync_params = params[:sync_params] + project_user = User.where(login: sync_params[:owner_login])&.first #以前已同步的项目,那么肯定存在仓库 - if Project.exists?(identifier: sync_params[:identifier]) + + user_projects = Project.where(user_id: project_user.id) + if user_projects.where(id: sync_params[:id]), identifier: sync_params[:identifier]).present? + has_project = true + project = user_projects.where(id: sync_params[:id]), identifier: sync_params[:identifier])&.first + elsif user_projects.where(id: sync_params[:id]).present? + has_project = true + project = user_projects.where(id: sync_params[:id]))&.first + elsif user_projects.where(identifier: sync_params[:identifier]).present? + has_project = true + project = user_projects.where(identifier: sync_params[:identifier])&.first + else + has_project = false + end + + if has_project SyncLog.sync_log("=================begin_to_update_project========") - project = Project.find_by(identifier: sync_params[:identifier]) + # project = user_projects.where(id: sync_params[:id]), identifier: sync_params[:identifier])&.first || + check_sync_project(project, sync_params) else #新建项目 SyncLog.sync_log("=================begin_to_create_new_project========") - project_user = User.where(login: sync_params[:owner_login]).first + project_params = { repository_name: sync_params[:identifier], user_id: project_user.id, @@ -30,6 +47,8 @@ class SyncForgeController < ApplicationController SyncRepositoryJob.perform_later(sync_params[:owner_login], sync_params[:identifier], sync_params[:repository], get_sudomain) if sync_params[:repository].present? check_new_project(project, sync_params) + else + SyncLog.sync_project_log("=============new_project_create_failed, trustie_project_id==:#{params[:sync_params][:id]}") end end end @@ -175,11 +194,10 @@ class SyncForgeController < ApplicationController begin forge_issue_ids = project&.issues&.select(:id)&.pluck(:id) sync_projects_params = {} - unless forge_issue_ids.size.to_i < old_issues_params[:count].to_i - forge_journal_ids = Journal.select([:id, :journalized_id, :journalized_type]).where(journalized_id: forge_issue_ids).pluck(:id) + unless forge_issue_ids.size.to_i <= old_issues_params[:count].to_i diff_issue_ids = old_issues_params[:ids] - forge_issue_ids - if diff_issue_ids.size == 0 #issue数量一样，判断评论是否有增减 + forge_journal_ids = Journal.select([:id, :journalized_id, :journalized_type]).where(journalized_id: forge_issue_ids).pluck(:id) diff_journal_ids = old_issues_params[:journals][:ids] - forge_journal_ids unless diff_journal_ids.size == 0 sync_projects_params = { @@ -209,7 +227,7 @@ class SyncForgeController < ApplicationController def change_project_watchers(project, watchers,gitea_main) SyncLog.sync_log("***5. begin_to_sync_watchers---------------") forge_watchers_ids = project&.watchers&.select(:id)&.pluck(:id) - unless forge_watchers_ids.size.to_i < watchers[:count].to_i + unless forge_watchers_ids.size.to_i <= watchers[:count].to_i diff_target_ids = watchers[:ids] - forge_watchers_ids if diff_target_ids.size > 0 sync_projects_params = { @@ -228,7 +246,7 @@ class SyncForgeController < ApplicationController def change_project_versions(project, versions,gitea_main) SyncLog.sync_log("***4. begin_to_sync_versions---------------") forge_version_ids = project&.versions&.select(:id)&.pluck(:id) - unless forge_version_ids.size < versions[:count].to_i + unless forge_version_ids.size <= versions[:count].to_i diff_version_ids = versions[:ids] - forge_version_ids if diff_version_ids.size > 0 sync_projects_params = { @@ -247,7 +265,7 @@ class SyncForgeController < ApplicationController def change_project_members(project, members,gitea_main) SyncLog.sync_log("***3. begin_to_sync_members---------------") forge_member_ids = project&.members&.select(:id)&.pluck(:id) - unless forge_member_ids.size < members[:count] + unless forge_member_ids.size <= members[:count] diff_member_ids = members[:ids] - forge_member_ids if diff_member_ids.size > 0 sync_projects_params = { diff --git a/app/jobs/sync_projects_job.rb b/app/jobs/sync_projects_job.rb index cfa52d22..fb6cbed0 100644 --- a/app/jobs/sync_projects_job.rb +++ b/app/jobs/sync_projects_job.rb @@ -57,23 +57,42 @@ class SyncProjectsJob < ApplicationJob u_id = User.select(:id, :login).where(login: re[:user_login]).pluck(:id).first re[:target_params].delete(:id) if target_type == "Issue" - new_target = target_type.constantize.new(re[:target_params].merge(author_id: u_id)) + is_exists = Issue.exists?(author_id: u_id, project_id: re[:target_params][:project_id], subject: re[:target_params][:subject]) + unless is_exists + assing_u_id = User.select(:id, :login).where(login: re[:assign_login]).pluck(:id).first + new_target = target_type.constantize.new(re[:target_params].merge(author_id: u_id)) + new_target.assigned_to_id = assing_u_id + end else - new_target = target_type.constantize.new(re[:target_params].merge(user_id: u_id)) + case target_type + when "Journal" + is_exists = Journal.exists?(user_id: u_id, journalized_id: re[:target_params][:journalized_id], created_on: re[:target_params][:created_on].to_s.to_time) + when "Member" + is_exists = Member.exists?(user_id: u_id, project_id: re[:target_params][:project_id], created_on: re[:target_params][:created_on].to_s.to_time) + when "Version" + is_exists = Version.exists?(user_id: u_id, project_id: re[:target_params][:project_id], created_on: re[:target_params][:created_on].to_s.to_time) + when "Watcher" + is_exists = Watcher.exists?(user_id: u_id, watchable_id: re[:target_params][:watchable_id], created_at: re[:target_params][:created_at].to_s.to_time) + when "PraiseTread" + is_exists = PraiseTread.exists?(user_id: u_id, praise_tread_object_id: re[:target_params][:praise_tread_object_id], created_at: re[:target_params][:created_at].to_s.to_time) + else + is_exists = false + end + unless is_exists + new_target = target_type.constantize.new(re[:target_params].merge(user_id: u_id)) + end end - - if target_type == "Issue" - assing_u_id = User.select(:id, :login).where(login: re[:assign_login]).pluck(:id).first - new_target.assigned_to_id = assing_u_id - end - if new_target.save! + + if !is_exists && new_target.save! SyncLog.sync_project_log("***【#{target_type}】. create_success---------------") if re[:journals].present? create_journals(re[:journals], "Journal", new_target.id) end if re[:journal_details].present? re[:journal_details].each do |j| - JournalDetail.create!(j.merge(journal_id: new_target.id)) if j.present? + unless JournalDetail.exists?(journal_id: new_target.id) + JournalDetail.create!(j.merge(journal_id: new_target.id)) if j.present? + end end end if re[:member_roles].present? @@ -82,7 +101,7 @@ class SyncProjectsJob < ApplicationJob end end else - SyncLog.sync_project_log("***【#{target_type}】. create_failed---------------") + SyncLog.sync_project_log("***【#{target_type}】. create_failed---or has_exists---------------") end end end @@ -100,14 +119,22 @@ class SyncProjectsJob < ApplicationJob SyncLog.sync_log("***user_login:#{re[:user_login]}----target_type:#{target_type}-----#{index+1}") if re[:target_params].present? u_id = User.select(:id, :login).where(login: re[:user_login]).pluck(:id).first - re[:target_params].delete(:id) - new_target = Journal.new(re[:target_params].merge(user_id: u_id)) - new_target.journalized_id = issue_id - if new_target.save! - if re[:journal_details].present? - re[:journal_details].each do |j| - JournalDetail.create!(j.merge(journal_id: new_target.id)) + is_exists = Journal.exists?(user_id: u_id, journalized_id: re[:target_params][:journalized_id], created_on: re[:target_params][:created_on].to_s.to_time) + + if is_exists + SyncLog.sync_project_log("***00000. Journal:#{re[:target_params][:id]}-is exists--------------") + else + re[:target_params].delete(:id) + new_target = Journal.new(re[:target_params].merge(user_id: u_id)) + new_target.journalized_id = issue_id + if new_target.save! + if re[:journal_details].present? + re[:journal_details].each do |j| + JournalDetail.create!(j.merge(journal_id: new_target.id)) + end end + else + SyncLog.sync_project_log("***111222. journal_create failed---------------") end end end @@ -118,15 +145,22 @@ class SyncProjectsJob < ApplicationJob def create_versions(project, target_jsons) SyncLog.sync_log("***【Versions】. begin_to_create_verison---------------") return SyncLog.sync_log("*** no target_jsons") if target_jsons.blank? - all_issues = project.issues.select(:id, :project_id, :fixed_version_id) target_jsons.each do |re| old_id = re[:target_params][:id] if re[:target_params].present? u_id = User.select(:id, :login).where(login: re[:user_login]).pluck(:id).first - re[:target_params].delete(:id) - new_target = Version.new(re[:target_params].merge(user_id: u_id)) - if new_target.save! - all_issues&.where(fixed_version_id: old_id)&.update_all(fixed_version_id: new_target.id) + is_exists = Version.exists?(user_id: u_id, project_id: re[:target_params][:project_id], created_on: re[:target_params][:created_on].to_s.to_time) + if is_exists + SyncLog.sync_project_log("***00000. Version:#{re[:target_params][:id]}-is exists--------------") + else + re[:target_params].delete(:id) + new_target = Version.new(re[:target_params].merge(user_id: u_id)) + if new_target.save! + all_issues = project.issues.select(:id, :project_id, :fixed_version_id) + all_issues&.where(fixed_version_id: old_id)&.update_all(fixed_version_id: new_target.id) + else + SyncLog.sync_project_log("***111222. Version_create failed---------------") + end end end end From 09835dda2480ab37f9ffbccc877256960e6bb3d1 Mon Sep 17 00:00:00 2001 From: "sylor_huang@126.com" Date: Wed, 15 Jul 2020 16:25:31 +0800 Subject: [PATCH 2/3] change bugs --- app/controllers/sync_forge_controller.rb | 14 +++++++------- app/models/sync_log.rb | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/sync_forge_controller.rb b/app/controllers/sync_forge_controller.rb index adfe6900..147961a2 100644 --- a/app/controllers/sync_forge_controller.rb +++ b/app/controllers/sync_forge_controller.rb @@ -9,12 +9,12 @@ class SyncForgeController < ApplicationController #以前已同步的项目,那么肯定存在仓库 user_projects = Project.where(user_id: project_user.id) - if user_projects.where(id: sync_params[:id]), identifier: sync_params[:identifier]).present? + if user_projects.where(id: sync_params[:id], identifier: sync_params[:identifier]).present? has_project = true - project = user_projects.where(id: sync_params[:id]), identifier: sync_params[:identifier])&.first + project = user_projects.where(id: sync_params[:id], identifier: sync_params[:identifier])&.first elsif user_projects.where(id: sync_params[:id]).present? has_project = true - project = user_projects.where(id: sync_params[:id]))&.first + project = user_projects.where(id: sync_params[:id])&.first elsif user_projects.where(identifier: sync_params[:identifier]).present? has_project = true project = user_projects.where(identifier: sync_params[:identifier])&.first @@ -194,7 +194,7 @@ class SyncForgeController < ApplicationController begin forge_issue_ids = project&.issues&.select(:id)&.pluck(:id) sync_projects_params = {} - unless forge_issue_ids.size.to_i <= old_issues_params[:count].to_i + if forge_issue_ids.size.to_i <= old_issues_params[:count].to_i diff_issue_ids = old_issues_params[:ids] - forge_issue_ids if diff_issue_ids.size == 0 #issue数量一样，判断评论是否有增减 forge_journal_ids = Journal.select([:id, :journalized_id, :journalized_type]).where(journalized_id: forge_issue_ids).pluck(:id) @@ -227,7 +227,7 @@ class SyncForgeController < ApplicationController def change_project_watchers(project, watchers,gitea_main) SyncLog.sync_log("***5. begin_to_sync_watchers---------------") forge_watchers_ids = project&.watchers&.select(:id)&.pluck(:id) - unless forge_watchers_ids.size.to_i <= watchers[:count].to_i + if forge_watchers_ids.size.to_i <= watchers[:count].to_i diff_target_ids = watchers[:ids] - forge_watchers_ids if diff_target_ids.size > 0 sync_projects_params = { @@ -246,7 +246,7 @@ class SyncForgeController < ApplicationController def change_project_versions(project, versions,gitea_main) SyncLog.sync_log("***4. begin_to_sync_versions---------------") forge_version_ids = project&.versions&.select(:id)&.pluck(:id) - unless forge_version_ids.size <= versions[:count].to_i + if forge_version_ids.size <= versions[:count].to_i diff_version_ids = versions[:ids] - forge_version_ids if diff_version_ids.size > 0 sync_projects_params = { @@ -265,7 +265,7 @@ class SyncForgeController < ApplicationController def change_project_members(project, members,gitea_main) SyncLog.sync_log("***3. begin_to_sync_members---------------") forge_member_ids = project&.members&.select(:id)&.pluck(:id) - unless forge_member_ids.size <= members[:count] + if forge_member_ids.size <= members[:count] diff_member_ids = members[:ids] - forge_member_ids if diff_member_ids.size > 0 sync_projects_params = { diff --git a/app/models/sync_log.rb b/app/models/sync_log.rb index f24a0a92..73fd8c78 100644 --- a/app/models/sync_log.rb +++ b/app/models/sync_log.rb @@ -5,7 +5,7 @@ class SyncLog end def self.sync_project_log(message=nil) - @my_log ||= Logger.new("#{Rails.root}/log/sync_error_project.log") + @my_log ||= Logger.new("#{Rails.root}/log/sync_project_log.log") @my_log.debug(message) unless message.nil? end end \ No newline at end of file From 6c116042f263a2576e1c6d8d2dc8f051e4c57649 Mon Sep 17 00:00:00 2001 From: "sylor_huang@126.com" Date: Wed, 15 Jul 2020 17:08:02 +0800 Subject: [PATCH 3/3] change bugs --- app/controllers/sync_forge_controller.rb | 6 ++-- app/jobs/sync_projects_job.rb | 38 +++++++++++++----------- app/jobs/sync_repository_job.rb | 6 ++-- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/app/controllers/sync_forge_controller.rb b/app/controllers/sync_forge_controller.rb index 147961a2..939b72a1 100644 --- a/app/controllers/sync_forge_controller.rb +++ b/app/controllers/sync_forge_controller.rb @@ -48,12 +48,12 @@ class SyncForgeController < ApplicationController SyncRepositoryJob.perform_later(sync_params[:owner_login], sync_params[:identifier], sync_params[:repository], get_sudomain) if sync_params[:repository].present? check_new_project(project, sync_params) else - SyncLog.sync_project_log("=============new_project_create_failed, trustie_project_id==:#{params[:sync_params][:id]}") + SyncLog.sync_log("=============new_project_create_failed, trustie_project_id==:#{params[:sync_params][:id]}") end end end rescue Exception => e - SyncLog.sync_project_log("=============sync_has_errors:==#{e.message}, project_id==:#{params[:sync_params][:id]}") + SyncLog.sync_log("=============sync_has_errors:==#{e.message}, project_id==:#{params[:sync_params][:id]}") end def sync_users @@ -95,7 +95,7 @@ class SyncForgeController < ApplicationController # SyncLog.sync_log("=================sync_to_user_failed,user_login==#{new_user.login}") # end # else - # SyncLog.sync_project_log("=============sync_to_user_failed,user_login====#{new_user.login}") + # SyncLog.sync_log("=============sync_to_user_failed,user_login====#{new_user.login}") # SyncLog.sync_log("=================sync_to_user_failed,user_login====#{new_user.login}") # end # end diff --git a/app/jobs/sync_projects_job.rb b/app/jobs/sync_projects_job.rb index fb6cbed0..2f912204 100644 --- a/app/jobs/sync_projects_job.rb +++ b/app/jobs/sync_projects_job.rb @@ -19,17 +19,17 @@ class SyncProjectsJob < ApplicationJob if response.code == '200' target_jsons = eval(response.body) if sync_params[:type] == "Project" - SyncLog.sync_project_log("==========target_jsons: #{target_jsons}============") + SyncLog.sync_log("==========target_jsons: #{target_jsons}============") update_new_project(target_jsons[:targets_params][0], sync_params[:new_project_id]) else - SyncLog.sync_project_log("========== #{sync_params[:type]}============") + SyncLog.sync_log("========== #{sync_params[:type]}============") create_target(target_jsons[:targets_params], sync_params[:type].to_s) end else - SyncLog.sync_project_log("==========sync_project_to_forge_failed #{sync_params[:type]}============") + SyncLog.sync_log("==========sync_project_to_forge_failed #{sync_params[:type]}============") end rescue => e - SyncLog.sync_project_log("==========sync_project_to_forge_failed #{sync_params[:type]}============errors:#{e}") + SyncLog.sync_log("==========sync_project_to_forge_failed #{sync_params[:type]}============errors:#{e}") end end @@ -39,19 +39,19 @@ class SyncProjectsJob < ApplicationJob SyncLog.sync_log("=========begin_to_update_project=project_id: #{project_id}============") project = Project.find_by(id: project_id) project.update(re[:target_params]) if re[:target_params].present? - create_target(re[:issues_params], "Issue") if re[:issues_params].present? create_target(re[:member_params], "Member") if re[:member_params].present? create_target(re[:watcher_params], "Watcher") if re[:watcher_params].present? create_target(re[:praise_treads], "PraiseTread") if re[:praise_treads].present? create_versions(project, re[:versions_params]) if re[:versions_params].present? + create_target(re[:issues_params], "Issue") if re[:issues_params].present? end def create_target(target_jsons, target_type) begin - SyncLog.sync_project_log("***【#{target_type}】. begin_to_create_target---------------") + SyncLog.sync_log("***【#{target_type}】. begin_to_create_target---------------") return SyncLog.sync_log("*** no target_jsons") if target_jsons.blank? target_jsons.each_with_index do |re,index| - SyncLog.sync_project_log("***user_login:#{re[:user_login]}----target_type:#{target_type}-----#{index+1}") + SyncLog.sync_log("***user_login:#{re[:user_login]}----target_type:#{target_type}-----#{index+1}") if re[:target_params].present? SyncLog.sync_log("***user_login:#{re[:user_login]}----target_type:#{target_type}") u_id = User.select(:id, :login).where(login: re[:user_login]).pluck(:id).first @@ -59,9 +59,12 @@ class SyncProjectsJob < ApplicationJob if target_type == "Issue" is_exists = Issue.exists?(author_id: u_id, project_id: re[:target_params][:project_id], subject: re[:target_params][:subject]) unless is_exists + version_name = re[:target_params][:re_version] + version_id = Version.where(project_id: re[:target_params][:project_id], name: version_name)&.first&.id if version_name.present? assing_u_id = User.select(:id, :login).where(login: re[:assign_login]).pluck(:id).first new_target = target_type.constantize.new(re[:target_params].merge(author_id: u_id)) new_target.assigned_to_id = assing_u_id + new_target.fixed_version_id = version_id if version_id.present? end else case target_type @@ -84,7 +87,7 @@ class SyncProjectsJob < ApplicationJob end if !is_exists && new_target.save! - SyncLog.sync_project_log("***【#{target_type}】. create_success---------------") + SyncLog.sync_log("***【#{target_type}】. create_success---------------") if re[:journals].present? create_journals(re[:journals], "Journal", new_target.id) end @@ -101,13 +104,13 @@ class SyncProjectsJob < ApplicationJob end end else - SyncLog.sync_project_log("***【#{target_type}】. create_failed---or has_exists---------------") + SyncLog.sync_log("***【#{target_type}】. create_failed---or has_exists---------------") end end end - SyncLog.sync_project_log("***111222. end_to_create_target---------------") + SyncLog.sync_log("***111222. end_to_create_target---------------") rescue => e - SyncLog.sync_project_log("=========***【#{target_type}】creat_had_erros:#{e}===================") + SyncLog.sync_log("=========***【#{target_type}】creat_had_erros:#{e}===================") end end @@ -122,7 +125,7 @@ class SyncProjectsJob < ApplicationJob is_exists = Journal.exists?(user_id: u_id, journalized_id: re[:target_params][:journalized_id], created_on: re[:target_params][:created_on].to_s.to_time) if is_exists - SyncLog.sync_project_log("***00000. Journal:#{re[:target_params][:id]}-is exists--------------") + SyncLog.sync_log("***00000. Journal:#{re[:target_params][:id]}-is exists--------------") else re[:target_params].delete(:id) new_target = Journal.new(re[:target_params].merge(user_id: u_id)) @@ -134,7 +137,7 @@ class SyncProjectsJob < ApplicationJob end end else - SyncLog.sync_project_log("***111222. journal_create failed---------------") + SyncLog.sync_log("***111222. journal_create failed---------------") end end end @@ -151,15 +154,16 @@ class SyncProjectsJob < ApplicationJob u_id = User.select(:id, :login).where(login: re[:user_login]).pluck(:id).first is_exists = Version.exists?(user_id: u_id, project_id: re[:target_params][:project_id], created_on: re[:target_params][:created_on].to_s.to_time) if is_exists - SyncLog.sync_project_log("***00000. Version:#{re[:target_params][:id]}-is exists--------------") + SyncLog.sync_log("***00000. Version:#{re[:target_params][:id]}-is exists--------------") else re[:target_params].delete(:id) new_target = Version.new(re[:target_params].merge(user_id: u_id)) if new_target.save! - all_issues = project.issues.select(:id, :project_id, :fixed_version_id) - all_issues&.where(fixed_version_id: old_id)&.update_all(fixed_version_id: new_target.id) + SyncLog.sync_log("***111222. Version_create success-#{new_target.id}--------------") + # all_issues = project.issues.select(:id, :project_id, :fixed_version_id) + # all_issues&.where(fixed_version_id: old_id)&.update_all(fixed_version_id: new_target.id) else - SyncLog.sync_project_log("***111222. Version_create failed---------------") + SyncLog.sync_log("***111222. Version_create failed-#{old_id}--------------") end end end diff --git a/app/jobs/sync_repository_job.rb b/app/jobs/sync_repository_job.rb index c110ce15..c59154d9 100644 --- a/app/jobs/sync_repository_job.rb +++ b/app/jobs/sync_repository_job.rb @@ -25,13 +25,13 @@ class SyncRepositoryJob < ApplicationJob gitlab_branches.each do |branch| shell5 = system("cd #{path}/#{image_repo_name} && git checkout #{branch} && git push --force --set-upstream origin #{branch}") if !shell5 - SyncLog.sync_project_log("=============force_push_erros==#{path}/#{image_repo_name}++branch:#{branch}") + SyncLog.sync_log("=============force_push_erros==#{path}/#{image_repo_name}++branch:#{branch}") else - SyncLog.sync_project_log("=============force_push_success==#{path}/#{image_repo_name}++branch+++#{branch}") + SyncLog.sync_log("=============force_push_success==#{path}/#{image_repo_name}++branch+++#{branch}") end end else - SyncLog.sync_project_log("=============check_clone_erros==#{path}/#{image_repo_name}") + SyncLog.sync_log("=============check_clone_erros==#{path}/#{image_repo_name}") SyncLog.sync_log("++++++++++++++++++check_clone_erros++++++++++++++++++#{image_repo_name}") end SyncLog.sync_log("=================end to sync repository=====================#{image_repo_name}")