diff --git a/README.md b/README.md index a09252ec5..c00e5785a 100644 --- a/README.md +++ b/README.md @@ -440,6 +440,9 @@ http://localhost:3000/api/projects/migrate | jq |repository_name |是|string |仓库名称, 只含有数字、字母、下划线不能以下划线开头和结尾，且唯一 | |project_category_id|是|int |项目类别id | |project_language_id|是|int |项目语言id | +|is_mirror |否|boolean|是否设置为镜像， true：是， false：否，默认为否 | +|auth_username |否|string|镜像源仓库的登录用户名 | +|auth_password |否|string|镜像源仓库的登录秘密 | |private |否|boolean|项目是否私有, true：为私有，false: 非私有，默认为公开 | @@ -458,6 +461,49 @@ http://localhost:3000/api/projects/migrate | jq "name": "ni项目" } ``` +#### 手动同步镜像 +``` +POST api/repositories/:id/sync_mirror +``` +*示例* +``` +curl -X POST \ +-d "user_id=36401" \ +http://localhost:3000/api/repositories/1244/sync_mirror | jq +``` +*请求参数说明:* + +|参数名|必选|类型|说明| +|-|-|-|-| +|user_id |是|int |用户id或者组织id | +|name |是|string |项目名称 | +|clone_addr |是|string |镜像项目clone地址 | +|description |否|string |项目描述 | +|repository_name |是|string |仓库名称, 只含有数字、字母、下划线不能以下划线开头和结尾，且唯一 | +|project_category_id|是|int |项目类别id | +|project_language_id|是|int |项目语言id | +|is_mirror |否|boolean|是否设置为镜像， true：是， false：否，默认为否 | +|auth_username |否|string|镜像源仓库的登录用户名 | +|auth_password |否|string|镜像源仓库的登录秘密 | +|private |否|boolean|项目是否私有, true：为私有，false: 非私有，默认为公开 | + + +*返回参数说明:* + +|参数名|类型|说明| +|-|-|-| +|status |int |状态码， 0:标识请求成功 | +|message |string|服务端返回的信息说明| + + +返回值 +``` +{ + "status": 0, + "message": "success" +} +``` + --- #### 项目详情 diff --git a/app/controllers/concerns/operate_project_ability_able.rb b/app/controllers/concerns/operate_project_ability_able.rb index c4797f33d..c470586aa 100644 --- a/app/controllers/concerns/operate_project_ability_able.rb +++ b/app/controllers/concerns/operate_project_ability_able.rb @@ -5,7 +5,7 @@ module OperateProjectAbilityAble end def authorizate_user_can_edit_project! - return if current_user.project_manager? @project || current_user.admin? + return if @project.manager?(current_user) || current_user.admin? render_forbidden('你没有权限操作.') end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index d95cccdde..aeadbd930 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -17,7 +17,7 @@ class ProjectsController < ApplicationController ActiveRecord::Base.transaction do Projects::CreateForm.new(project_params).validate! @project = Projects::CreateService.new(current_user, project_params).call - + end rescue Exception => e uid_logger_error(e.message) @@ -107,8 +107,8 @@ class ProjectsController < ApplicationController end def mirror_params - params.permit(:user_id, :name, :description, :repository_name, - :project_category_id, :project_language_id, :clone_addr, :private) + params.permit(:user_id, :name, :description, :repository_name, :is_mirror, :auth_username, + :auth_password, :project_category_id, :project_language_id, :clone_addr, :private) end def project_public? diff --git a/app/controllers/repositories_controller.rb b/app/controllers/repositories_controller.rb index 49e095c8e..b834935b8 100644 --- a/app/controllers/repositories_controller.rb +++ b/app/controllers/repositories_controller.rb @@ -1,7 +1,10 @@ class RepositoriesController < ApplicationController include ApplicationHelper - before_action :require_login, only: %i[edit update create_file update_file delete_file] + include OperateProjectAbilityAble + before_action :require_login, only: %i[edit update create_file update_file delete_file sync_mirror] before_action :find_project, :authorizate! + before_action :find_repository, only: %i[sync_mirror] + before_action :authorizate_user_can_edit_project!, only: %i[sync_mirror] def show @branches_count = Gitea::Repository::BranchesService.new(@project.owner, @project.identifier).call&.size @@ -125,8 +128,8 @@ class RepositoriesController < ApplicationController def repo_hook hook_type = request.headers["X-Gitea-Event"].to_s # 获取推送的方式 - ownername = @project.owner.try(:login) - reponame = @project.identifier + ownername = @project.owner.try(:login) + reponame = @project.identifier username = current_user.try(:login) user_params = { "ownername": ownername, @@ -142,6 +145,12 @@ class RepositoriesController < ApplicationController @project.update_attribute(:token, @project.token + uploadPushInfo[:modificationLines].to_i) end + def sync_mirror + @repo&.mirror.set_status!(Mirror.statuses[:waiting]) + SyncMirroredRepositoryJob(@repo, current_user) + render_ok + end + private def find_project @@ -149,6 +158,10 @@ class RepositoriesController < ApplicationController render_not_found("未找到相关的仓库") unless @project end + def find_repository + @repo = Repository.find params[:id] + end + def authorizate! if @project.repository.hidden? && !@project.member?(current_user) render_forbidden @@ -166,13 +179,13 @@ class RepositoriesController < ApplicationController } end - def hook_params(hook_type, params) + def hook_params(hook_type, params) if hook_type == "push" - # TODO hook返回的记录中，暂时没有文件代码数量的增减，暂时根据 commits数量来计算 + # TODO hook返回的记录中，暂时没有文件代码数量的增减，暂时根据 commits数量来计算 uploadPushInfo = { - "shas": params["commits"].present? ? params["commits"].map{|c| c["id"]} : "", + "shas": params["commits"].present? ? params["commits"].map{|c| c["id"]} : "", "branch": params["ref"].to_s.split("/").last, - "modificationLines": params["commits"].length + "modificationLines": params["commits"].length } elsif hook_type == "pull_request" && params["action"].to_s == "closed" #合并请求合并后才会有上链操作 uploadPushInfo = { @@ -183,7 +196,7 @@ class RepositoriesController < ApplicationController "shas": [params["pull_request"]["merge_commit_sha"], params["pull_request"]["merge_base"]], "modificationLines": 1 #pull_request中没有commits数量 } - else + else uploadPushInfo = {} end diff --git a/app/forms/projects/migrate_form.rb b/app/forms/projects/migrate_form.rb index 1c0b25493..f17b75a5a 100644 --- a/app/forms/projects/migrate_form.rb +++ b/app/forms/projects/migrate_form.rb @@ -2,7 +2,7 @@ class Projects::MigrateForm < BaseForm REPOSITORY_NAME_REGEX = /^(?!_)(?!.*?_$)[a-zA-Z0-9_-]+$/ #只含有数字、字母、下划线不能以下划线开头和结尾 URL_REGEX = /\A(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:\/[^\s]*)?\z/i - attr_accessor :user_id, :name, :description, :repository_name, :project_category_id, :project_language_id, :clone_addr, :private + attr_accessor :user_id, :name, :description, :repository_name, :project_category_id, :project_language_id, :clone_addr, :private, :is_mirror, :auth_username, :auth_password validates :user_id, :name, :description,:repository_name, :project_category_id, :project_language_id, presence: true validates :repository_name, format: { with: REPOSITORY_NAME_REGEX, multiline: true, message: "只能含有数字、字母、下划线且不能以下划线开头和结尾" } diff --git a/app/jobs/migrate_remote_repository_job.rb b/app/jobs/migrate_remote_repository_job.rb new file mode 100644 index 000000000..bc17718ad --- /dev/null +++ b/app/jobs/migrate_remote_repository_job.rb @@ -0,0 +1,18 @@ +class MigrateRemoteRepositoryJob < ApplicationJob + queue_as :default + + def perform(repo, token, params) + gitea_repository = Gitea::Repository::MigrateService.new(token, params).call + sync_project(repo, gitea_repository) + sync_repository(repo, gitea_repository) + end + + private + def sync_project(repo, gitea_repository) + repo&.project.update_columns(gpid: gitea_repository["id"], identifier: gitea_repository["name"]) if gitea_repository + end + + def sync_repository(repository, gitea_repository) + repository.mirror.update_columns(statuses: Mirror.statuses[:succeeded]) if gitea_repository + end +end diff --git a/app/jobs/sync_mirrored_repository_job.rb b/app/jobs/sync_mirrored_repository_job.rb new file mode 100644 index 000000000..7230e1195 --- /dev/null +++ b/app/jobs/sync_mirrored_repository_job.rb @@ -0,0 +1,8 @@ +class SyncMirroredRepositoryJob < ApplicationJob + queue_as :default + + def perform(repo, current_user) + result = Gitea::Repository::SyncMirroredService.new(repo.user.login, repo.identifier, token: current_user.gitea_token).call + repo&.mirror.set_status! if result[:status] === 200 + end +end diff --git a/app/models/concerns/project_operable.rb b/app/models/concerns/project_operable.rb index 513eff3a1..bab4158ac 100644 --- a/app/models/concerns/project_operable.rb +++ b/app/models/concerns/project_operable.rb @@ -4,7 +4,9 @@ module ProjectOperable included do has_many :members, dependent: :destroy has_many :except_owner_members, -> { joins(:roles).where.not(roles: { name: 'Manager' }) }, class_name: 'Member' - has_many :manager_members, -> { joins(:roles).where(roles: { name: 'Manager' }) }, class_name: 'Member' + has_many :managers, -> { joins(:roles).where(roles: { name: 'Manager' }) }, class_name: 'Member' + has_many :developers, -> { joins(:roles).where(roles: { name: 'Developer' }) }, class_name: 'Member' + has_many :reporters, -> { joins(:roles).where(roles: { name: 'Reporter' }) }, class_name: 'Member' end def add_member!(user_id, role_name='Developer') @@ -35,6 +37,20 @@ module ProjectOperable self.owner == user end + # 项目管理员(包含项目拥有者)，权限：仓库设置、仓库可读可写 + def manager?(user) + managers.exists? user + end + + # 项目开发者，可读可写权限 + def develper?(user) + developers.exists? user + end + # 报告者，只有可读权限 + def reporter?(user) + reporters.exists? user + end + def set_developer_role(member) role = Role.find_by_name 'Developer' member.member_roles.create!(role: role) diff --git a/app/models/mirror.rb b/app/models/mirror.rb new file mode 100644 index 000000000..8b7e61b0a --- /dev/null +++ b/app/models/mirror.rb @@ -0,0 +1,13 @@ +class Mirror < ApplicationRecord + + # 0 - succeeded, 1 - waiting, 2 - failed + # 0: 同步镜像成功；1: 正在同步镜像；2: 同步失败，默认值为0 + enum status: { succeeded: 0, waiting: 1, failed: 2 } + + belongs_to :repository + + + def set_status!(status=Mirror.statuses[:succeeded]) + update_column(status: status) + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 0df17cd0a..a5ae46256 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -4,7 +4,11 @@ class Project < ApplicationRecord include Watchable include ProjectOperable - enum project_type: { mirror: 1, common: 0 } # common:开源托管项目, mirror:开源镜像项目 + # common:开源托管项目 + # mirror:普通镜像项目，没有定时同步功能 + # sync_mirror:同步镜像项目，有系统定时同步功能，且用户可手动同步操作 + # + enum project_type: { sync_mirror: 2, mirror: 1, common: 0 } belongs_to :ignore, optional: true belongs_to :license, optional: true @@ -124,8 +128,8 @@ class Project < ApplicationRecord def releases_size(current_user_id, type) if current_user_id == self.user_id && type.to_s == "all" - self.repository.version_releases_count - else + self.repository.version_releases_count + else self.repository.version_releases.releases_size end end diff --git a/app/models/repository.rb b/app/models/repository.rb index a1e30f785..0cd5ed511 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -2,6 +2,7 @@ class Repository < ApplicationRecord self.inheritance_column = nil # FIX The single-table inheritance mechanism failed belongs_to :project, :touch => true belongs_to :user + has_one :mirror, foreign_key: :repo_id has_many :version_releases, dependent: :destroy validates :identifier, presence: true @@ -9,4 +10,9 @@ class Repository < ApplicationRecord def to_param self.identifier.parameterize end + + # with repository is mirror + def set_mirror! + self.build_mirror(status: Mirror.statuses[:waiting]).save + end end diff --git a/app/services/gitea/client_service.rb b/app/services/gitea/client_service.rb index 1b539bb7b..67e804bf7 100644 --- a/app/services/gitea/client_service.rb +++ b/app/services/gitea/client_service.rb @@ -143,6 +143,8 @@ class Gitea::ClientService < ApplicationService when 409 message = "创建失败，请检查该分支合并是否已存在" raise Error, mark + message + when 403 + {status: 403, message: '你没有权限操作!'} else if response&.body.blank? message = "请求失败" diff --git a/app/services/gitea/repository/get_by_id_service.rb b/app/services/gitea/repository/get_by_id_service.rb new file mode 100644 index 000000000..c633c6b71 --- /dev/null +++ b/app/services/gitea/repository/get_by_id_service.rb @@ -0,0 +1,31 @@ +class Gitea::Repository::GetByIdService < Gitea::ClientService + attr_reader :owner, :repo_id + + def initialize(owner, repo_id) + @owner = owner + @repo_id = repo_id + end + + def call + response = get(url, params) + render_result(response) + end + + private + def params + Hash.new.merge(token: owner.gitea_token) + end + + def url + "/repositories/#{repo_id}".freeze + end + + def render_result(response) + case response.status + when 200 + JSON.parse(response.body) + else + nil + end + end +end diff --git a/app/services/gitea/repository/sync_mirrored_service.rb b/app/services/gitea/repository/sync_mirrored_service.rb new file mode 100644 index 000000000..6918a6ec2 --- /dev/null +++ b/app/services/gitea/repository/sync_mirrored_service.rb @@ -0,0 +1,30 @@ +# Sync a mirrored repository +class Gitea::Repository::SyncMirroredService < Gitea::ClientService + attr_reader :token, :owner, :repo + + # owner * + # owner of the repo to sync + # repo * + # name of the repo to sync + # example: + # Gitea::Repository::SyncMirroredService.new(owner.login, repo.identifier, user.gitea_token).call + def initialize(owner, repo, token=nil) + @token = token + @owner = owner + @repo = repo + end + + def call + post(url, request_params) + end + + private + + def request_params + Hash.new.merge(token: token) + end + + def url + "/repos/#{owner}/#{repo}/mirror-sync".freeze + end +end diff --git a/app/services/projects/migrate_service.rb b/app/services/projects/migrate_service.rb index aea0607d2..eb24bdc41 100644 --- a/app/services/projects/migrate_service.rb +++ b/app/services/projects/migrate_service.rb @@ -31,17 +31,23 @@ class Projects::MigrateService < ApplicationService project_category_id: params[:project_category_id], project_language_id: params[:project_language_id], is_public: project_secretion[:public], - project_type: Project.project_types[:mirror] + project_type: set_project_type } end + def set_project_type + ActiveModel::Type::Boolean.new.cast(params[:is_mirror]) == true ? Project.project_types[:sync_mirror] : Project.project_types[:mirror] + end + def repository_params { hidden: project_secretion[:hidden], identifier: params[:repository_name], mirror_url: params[:clone_addr], user_id: user.id, - login: user.login + login: params[:auth_username], + password: params[:auth_password], + is_mirror: params[:is_mirror] } end diff --git a/app/services/repositories/migrate_service.rb b/app/services/repositories/migrate_service.rb index 0991a75c8..0e773d54c 100644 --- a/app/services/repositories/migrate_service.rb +++ b/app/services/repositories/migrate_service.rb @@ -11,9 +11,8 @@ class Repositories::MigrateService < ApplicationService @repository = Repository.new(repository_params) ActiveRecord::Base.transaction do if @repository.save! - gitea_repository = Gitea::Repository::MigrateService.new(user.gitea_token, gitea_repository_params).call - sync_project(gitea_repository) - sync_repository(@repository, gitea_repository) + @repository.set_mirror! if wrapper_mirror + MigrateRemoteRepositoryJob.perform_later(@repository, user.gitea_token, gitea_repository_params) end @repository end @@ -23,15 +22,6 @@ class Repositories::MigrateService < ApplicationService end private - - def sync_project(gitea_repository) - project.update_columns(gpid: gitea_repository["id"], identifier: gitea_repository["name"]) if gitea_repository - end - - def sync_repository(repository, gitea_repository) - repository.update_columns(url: gitea_repository["clone_url"]) if gitea_repository - end - def repository_params params.merge(project_id: project.id) end @@ -41,7 +31,14 @@ class Repositories::MigrateService < ApplicationService clone_addr: params[:mirror_url], repo_name: params[:identifier], uid: user.gitea_uid, - private: params[:hidden] + private: params[:hidden], + mirror: wrapper_mirror || false, + auth_username: params[:login], + auth_password: params[:password] } end + + def wrapper_mirror + ActiveModel::Type::Boolean.new.cast(params[:is_mirror]) + end end diff --git a/config/routes.rb b/config/routes.rb index 61efab5b6..6e40144f4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -197,6 +197,7 @@ Rails.application.routes.draw do put :update_file delete :delete_file post :repo_hook + post :sync_mirror end end diff --git a/db/migrate/20200515024117_add_fork_url_to_repositories.rb b/db/migrate/20200515024117_add_fork_url_to_repositories.rb new file mode 100644 index 000000000..2afdb83a3 --- /dev/null +++ b/db/migrate/20200515024117_add_fork_url_to_repositories.rb @@ -0,0 +1,5 @@ +class AddForkUrlToRepositories < ActiveRecord::Migration[5.2] + def change + add_column :repositories, :fork_url, :string, default: nil + end +end diff --git a/db/migrate/20200518081445_create_mirrors.rb b/db/migrate/20200518081445_create_mirrors.rb new file mode 100644 index 000000000..f1691348e --- /dev/null +++ b/db/migrate/20200518081445_create_mirrors.rb @@ -0,0 +1,12 @@ +class CreateMirrors < ActiveRecord::Migration[5.2] + def change + create_table :mirrors do |t| + t.integer :repo_id, foreign_key: true + t.integer :status, default: 0, null: false, comment: "0 - succeeded, 1 - waiting, 2 - failed" + t.integer :interval, comment: "mirror interval with unix time" + t.datetime :next_update_time, comment: "next update mirror time, for datetime" + + t.timestamps + end + end +end diff --git a/db/migrate/20200518090640_add_is_mirror_to_repositories.rb b/db/migrate/20200518090640_add_is_mirror_to_repositories.rb new file mode 100644 index 000000000..5fbacd90d --- /dev/null +++ b/db/migrate/20200518090640_add_is_mirror_to_repositories.rb @@ -0,0 +1,5 @@ +class AddIsMirrorToRepositories < ActiveRecord::Migration[5.2] + def change + add_column :repositories, :is_mirror, :boolean, default: false + end +end