From f9bb74aba46b372caef21f8c596f0d1b13c07ee6 Mon Sep 17 00:00:00 2001 From: yystopf Date: Mon, 25 Oct 2021 16:57:11 +0800 Subject: [PATCH 01/11] add: cache service --- app/helpers/avatar_helper.rb | 26 +++ app/jobs/cache_async_reset_job.rb | 16 ++ app/jobs/cache_async_set_job.rb | 16 ++ app/services/cache/v2/owner_common_service.rb | 89 ++++++++ .../cache/v2/platform_statistic_service.rb | 182 +++++++++++++++ .../cache/v2/project_common_service.rb | 211 ++++++++++++++++++ .../cache/v2/project_date_rank_service.rb | 50 +++++ app/services/cache/v2/project_rank_service.rb | 71 ++++++ .../cache/v2/user_statistic_service.rb | 183 +++++++++++++++ 9 files changed, 844 insertions(+) create mode 100644 app/helpers/avatar_helper.rb create mode 100644 app/jobs/cache_async_reset_job.rb create mode 100644 app/jobs/cache_async_set_job.rb create mode 100644 app/services/cache/v2/owner_common_service.rb create mode 100644 app/services/cache/v2/platform_statistic_service.rb create mode 100644 app/services/cache/v2/project_common_service.rb create mode 100644 app/services/cache/v2/project_date_rank_service.rb create mode 100644 app/services/cache/v2/project_rank_service.rb create mode 100644 app/services/cache/v2/user_statistic_service.rb diff --git a/app/helpers/avatar_helper.rb b/app/helpers/avatar_helper.rb new file mode 100644 index 000000000..b703e1b4e --- /dev/null +++ b/app/helpers/avatar_helper.rb @@ -0,0 +1,26 @@ +module AvatarHelper + def relative_path + "avatars" + end + + def storage_path + File.join(Rails.root, "public", "images", relative_path) + end + + def disk_filename(source_type,source_id,image_file=nil) + File.join(storage_path, "#{source_type}", "#{source_id}") + end + + def url_to_avatar(source) + if File.exist?(disk_filename(source&.class, source&.id)) + ctime = File.ctime(disk_filename(source.class, source.id)).to_i + if %w(User Organization).include?(source.class.to_s) + File.join("images", relative_path, ["#{source.class}", "#{source.id}"]) + "?t=#{ctime}" + else + File.join("images/avatars", ["#{source.class}", "#{source.id}"]) + "?t=#{ctime}" + end + elsif source.class.to_s == 'User' + source.get_letter_avatar_url + end + end +end \ No newline at end of file diff --git a/app/jobs/cache_async_reset_job.rb b/app/jobs/cache_async_reset_job.rb new file mode 100644 index 000000000..a7da3f80e --- /dev/null +++ b/app/jobs/cache_async_reset_job.rb @@ -0,0 +1,16 @@ +class CacheAsyncResetJob < ApplicationJob + queue_as :cache + + def perform(type, id, params={}) + case type + when "owner_common_service" + Cache::V2::OwnerCommonService.new(id).reset + when "platform_statistic_service" + Cache::V2::PlatformStatisticService.new.reset + when "project_common_service" + Cache::V2::ProjectCommonService.new(id).reset + when "user_statistic_service" + Cache::V2::PlatformStatisticService.new(id).reset + end + end +end \ No newline at end of file diff --git a/app/jobs/cache_async_set_job.rb b/app/jobs/cache_async_set_job.rb new file mode 100644 index 000000000..d4f641e19 --- /dev/null +++ b/app/jobs/cache_async_set_job.rb @@ -0,0 +1,16 @@ +class CacheAsyncResetJob < ApplicationJob + queue_as :cache + + def perform(type, id, params={}) + case type + when "owner_common_service" + Cache::V2::OwnerCommonService.new(id, params).call + when "platform_statistic_service" + Cache::V2::PlatformStatisticService.new(params).call + when "project_common_service" + Cache::V2::ProjectCommonService.new(id, params).call + when "user_statistic_service" + Cache::V2::PlatformStatisticService.new(id, params).call + end + end +end \ No newline at end of file diff --git a/app/services/cache/v2/owner_common_service.rb b/app/services/cache/v2/owner_common_service.rb new file mode 100644 index 000000000..d65a60749 --- /dev/null +++ b/app/services/cache/v2/owner_common_service.rb @@ -0,0 +1,89 @@ +class Cache::V2::OwnerCommonService < ApplicationService + include AvatarHelper + attr_reader :owner_id, :login, :name, :avatar_url, :email + attr_accessor :owner + + def initialize(owner_id, params={}) + @owner_id = owner_id + @name = params[:name] + @email = params[:email] + end + + def read + owner_common + end + + def call + load_owner + set_owner_common + end + + def reset + reset_owner_common + end + + private + def load_owner + @owner = User.find_by_id(@owner_id) + end + + def owner_common_key + "v2-owner-common:#{@owner_id}" + end + + def owner_common + $redis_cache.hgetall(owner_common_key).blank? ? reset_owner_common : $redis_cache.hgetall(owner_common_key) + end + + def set_owner_common + if $redis_cache.hgetall(owner_common_key).blank? + reset_owner_common + return + end + if @name.present? + if $redis_cache.hget(owner_common_key, "name").nil? + reset_owner_name + else + $redis_cache.hset(owner_common_key, "name", @name) + $redis_cache.hset(owner_common_key, "avatar_url", url_to_avatar(owner)) + end + end + if @email.present? + if $redis_cache.hget(owner_common_key, "email").nil? + reset_owner_email + else + $redis_cache.hset(owner_common_key, "email", @email) + end + end + + $redis_cache.hgetall(owner_common_key) + end + + def reset_owner_type + $redis_cache.hset(owner_common_key, "type", owner&.type) + end + + def reset_owner_login + $redis_cache.hset(owner_common_key, "login", owner&.login) + end + + def reset_owner_email + $redis_cache.hset(owner_common_key, "email", owner&.mail) + end + + def reset_owner_name + $redis_cache.hset(owner_common_key, "name", owner&.real_name) + $redis_cache.hset(owner_common_key, "avatar_url", url_to_avatar(owner)) + end + + def reset_owner_common + load_owner + $redis_cache.del(owner_common_key) + reset_owner_type + reset_owner_login + reset_owner_email + reset_owner_name + + $redis_cache.hgetall(owner_common_key) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/platform_statistic_service.rb b/app/services/cache/v2/platform_statistic_service.rb new file mode 100644 index 000000000..bc6621a38 --- /dev/null +++ b/app/services/cache/v2/platform_statistic_service.rb @@ -0,0 +1,182 @@ +class Cache::V2::PlatformStatisticService < ApplicationService + attr_reader :follow_count, :fork_count, :issue_count, :project_count, :project_language_count_key, :project_language_count, :project_praise_count, :project_watcher_count, :pullrequest_count + + def initialize(params={}) + @follow_count = params[:follow_count] + @fork_count = params[:fork_count] + @issue_count = params[:issue_count] + @project_count = params[:project_count] + @project_language_count_key = params[:project_language_count_key] + @project_language_count = params[:project_language_count] + @project_praise_count = params[:project_praise_count] + @project_watcher_count = params[:project_watcher_count] + @pullrequest_count = params[:pullrequest_count] + end + + def read + platform_statistic + end + + def call + set_platform_statistic + end + + def reset + reset_platform_statistic + end + + private + + def platform_statistic_key + "v2-platform-statistic" + end + + def follow_count_key + "follow-count" + end + + def fork_count_key + "fork-count" + end + + def issue_count_key + "issue-count" + end + + def project_count_key + "project-count" + end + + def project_language_key + "project-language" + end + + def project_praise_count_key + "project-praise-count" + end + + def project_watcher_count_key + "project-watcher-count" + end + + def pullrequest_count_key + "pullrequest-count" + end + + def platform_statistic + $redis_cache.hgetall(platform_statistic_key).blank? ? reset_platform_statistic : $redis_cache.hgetall(platform_statistic_key) + end + + def set_platform_statistic + if $redis_cache.hgetall(platform_statistic_key).blank? + reset_platform_statistic + return + end + if @follow_count.present? + if $redis_cache.hget(platform_statistic_key, follow_count_key).nil? + reset_platform_follow_count + else + $redis_cache.hincrby(platform_statistic_key, follow_count_key, @follow_count) + end + end + if @fork_count.present? + if $redis_cache.hget(platform_statistic_key, fork_count_key).nil? + reset_platform_fork_count + else + $redis_cache.hincrby(platform_statistic_key, fork_count_key, @fork_count) + end + end + if @issue_count.present? + if $redis_cache.hget(platform_statistic_key, issue_count_key).nil? + reset_platform_issue_count + else + $redis_cache.hincrby(platform_statistic_key, issue_count_key, @issue_count) + end + end + if @project_count.present? + if $redis_cache.hget(platform_statistic_key, project_count_key).nil? + reset_platform_project_count + else + $redis_cache.hincrby(platform_statistic_key, project_count_key, @project_count) + end + end + if @project_language_count_key.present? && project_language_count.present? + if $redis_cache.hget(platform_statistic_key, project_language_key).nil? + reset_platform_project_language + else + result = JSON.parse($redis_cache.hget(platform_statistic_key, project_language_key)) + result[@project_language_count_key] ||= 0 + result[@project_language_count_key] += project_language_count.to_i + $redis_cache.hset(platform_statistic_key, project_language_key, result.to_json) + end + end + if @project_praise_count.present? + if $redis_cache.hget(platform_statistic_key, project_praise_count_key).nil? + reset_platform_project_praise_count + else + $redis_cache.hincrby(platform_statistic_key, project_praise_count_key, @project_praise_count) + end + end + if @project_watcher_count.present? + if $redis_cache.hget(platform_statistic_key, project_watcher_count_key).nil? + reset_platform_project_watcher_count + else + $redis_cache.hincrby(platform_statistic_key, project_watcher_count_key, @project_watcher_count) + end + end + if @pullrequest_count.present? + if $redis_cache.hget(platform_statistic_key, pullrequest_count_key).nil? + reset_platform_pullrequest_count + else + $redis_cache.hincrby(platform_statistic_key, pullrequest_count_key, @pullrequest_count) + end + end + $redis_cache.hgetall(platform_statistic_key) + end + + def reset_platform_follow_count + $redis_cache.hset(platform_statistic_key, follow_count_key, Watcher.where(watchable_type: 'User').count) + end + + def reset_platform_fork_count + $redis_cache.hset(platform_statistic_key, fork_count_key, ForkUser.count) + end + + def reset_platform_issue_count + $redis_cache.hset(platform_statistic_key, issue_count_key, Issue.count) + end + + def reset_platform_project_count + $redis_cache.hset(platform_statistic_key, project_count_key, Project.count) + end + + def reset_platform_project_language + $redis_cache.hset(platform_statistic_key, project_language_key, ProjectLanguage.where.not(projects_count: 0).group("project_languages.name").sum(:projects_count).to_json) + end + + def reset_platform_project_praise_count + $redis_cache.hset(platform_statistic_key, project_praise_count_key, PraiseTread.where(praise_tread_object_type: "Project").count) + end + + def reset_platform_project_watcher_count + $redis_cache.hset(platform_statistic_key, project_watcher_count_key, Watcher.where(watchable_type: 'Project').count) + end + + def reset_platform_pullrequest_count + $redis_cache.hset(platform_statistic_key, pullrequest_count_key, PullRequest.count) + end + + def reset_platform_statistic + $redis_cache.del(platform_statistic_key) + reset_platform_follow_count + reset_platform_fork_count + reset_platform_issue_count + reset_platform_project_count + reset_platform_project_language + reset_platform_project_praise_count + reset_platform_project_watcher_count + reset_platform_pullrequest_count + + $redis_cache.hgetall(platform_statistic_key) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/project_common_service.rb b/app/services/cache/v2/project_common_service.rb new file mode 100644 index 000000000..365f07a6e --- /dev/null +++ b/app/services/cache/v2/project_common_service.rb @@ -0,0 +1,211 @@ +class Cache::V2::ProjectCommonService < ApplicationService + attr_reader :project_id, :owner_id, :name, :identifier, :visits, :watchers, :praises, :forks, :issues, :pullrequests + attr_accessor :project + + def initialize(project_id, params={}) + @project_id = project_id + @owner_id = params[:owner_id] + @name = params[:name] + @identifier = params[:identifier] + @visits = params[:visits] + @watchers = params[:watchers] + @praises = params[:praises] + @forks = params[:forks] + @issues = params[:issues] + @pullrequests = params[:pullrequests] + end + + def read + project_common + end + + def call + set_project_common + end + + def reset + reset_project_common + end + + private + def load_project + @project = Project.find_by_id(project_id) + end + + def project_common_key + "v2-project-common:#{@project_id}" + end + + def owner_id_key + "owner_id" + end + + def name_key + "name" + end + + def identifier_key + "identifier" + end + + def visits_key + "visits" + end + + def watchers_key + "watchers" + end + + def praises_key + "praises" + end + + def forks_key + "forks" + end + + def issues_key + "issues" + end + + def pullrequests_key + "pullrequests" + end + + def project_common + $redis_cache.hgetall(project_common_key).blank? ? reset_project_common : $redis_cache.hgetall(project_common_key) + end + + def set_project_common + if $redis_cache.hgetall(project_common_key).blank? + reset_project_common + return + end + if @owner_id.present? + if $redis_cache.hget(project_common_key, owner_id_key).nil? + reset_project_owner_id + else + $redis_cache.hset(project_common_key, owner_id_key, @owner_id) + end + end + if @name.present? + if $redis_cache.hget(project_common_key, name_key).nil? + reset_project_name + else + $redis_cache.hset(project_common_key, name_key, @name) + end + end + if @identifier.present? + if $redis_cache.hget(project_common_key, identifier_key).nil? + reset_project_identifier + else + $redis_cache.hset(project_common_key, identifier_key, @identifier) + end + end + if @visits.present? + if $redis_cache.hget(project_common_key, visits_key).nil? + reset_project_visits + else + $redis_cache.hincrby(project_common_key, visits_key, @visits) + Cache::V2::ProjectRankService.call(@project_id, {visits: @visits}) + Cache::V2::ProjectDateRankService.call(@project_id, {visits: @visits}) + end + end + if @watchers.present? + if $redis_cache.hget(project_common_key, watchers_key).nil? + reset_project_watchers + else + $redis_cache.hincrby(project_common_key, watchers_key, @watchers) + end + end + if @praises.present? + if $redis_cache.hget(project_common_key, praises_key).nil? + reset_project_praises + else + $redis_cache.hincrby(project_common_key, praises_key, @praises) + Cache::V2::ProjectRankService.call(@project_id, {praises: @praises}) + Cache::V2::ProjectDateRankService.call(@project_id, {praises: @praises}) + end + end + if @forks.present? + if $redis_cache.hget(project_common_key, forks_key).nil? + reset_project_forks + else + $redis_cache.hincrby(project_common_key, forks_key, @forks) + Cache::V2::ProjectRankService.call(@project_id, {forks: @forks}) + Cache::V2::ProjectDateRankService.call(@project_id, {forks: @forks}) + end + end + if @issues.present? + if $redis_cache.hget(project_common_key, issues_key).nil? + reset_project_issues + else + $redis_cache.hincrby(project_common_key, issues_key, @issues) + Cache::V2::ProjectRankService.call(@project_id, {issues: @issues}) + Cache::V2::ProjectDateRankService.call(@project_id, {issues: @issues}) + end + end + if @pullrequests.present? + if $redis_cache.hget(project_common_key, pullrequests_key).nil? + reset_project_pullrequests + else + $redis_cache.hincrby(project_common_key, pullrequests_key, @pullrequests) + Cache::V2::ProjectRankService.call(@project_id, {pullrequests: @pullrequests}) + Cache::V2::ProjectDateRankService.call(@project_id, {pullrequests: @pullrequests}) + end + end + $redis_cache.hgetall(project_common_key) + end + + def reset_project_owner_id + $redis_cache.hset(project_common_key, owner_id_key, @project&.user_id) + end + + def reset_project_name + $redis_cache.hset(project_common_key, name_key, @project&.name) + end + + def reset_project_identifier + $redis_cache.hset(project_common_key, identifier_key, @project&.identifier) + end + + def reset_project_visits + $redis_cache.hset(project_common_key, visits_key, @project&.visits) + end + + def reset_project_watchers + $redis_cache.hset(project_common_key, watchers_key, Watcher.where(watchable_type: 'Project', watchable_id: @project_id).count) + end + + def reset_project_praises + $redis_cache.hset(project_common_key, praises_key, PraiseTread.where(praise_tread_object_type: 'Project', praise_tread_object_id: @project_id).count) + end + + def reset_project_forks + $redis_cache.hset(project_common_key, forks_key, ForkUser.where(project_id: @project_id).count) + end + + def reset_project_issues + $redis_cache.hset(project_common_key, issues_key, Issue.where(project_id: @project_id).count) + end + + def reset_project_pullrequests + $redis_cache.hset(project_common_key, pullrequests_key, PullRequest.where(project_id: @project_id).count) + end + + def reset_project_common + load_project + $redis_cache.del(project_common_key) + reset_project_owner_id + reset_project_name + reset_project_identifier + reset_project_visits + reset_project_watchers + reset_project_praises + reset_project_forks + reset_project_issues + reset_project_pullrequests + + $redis_cache.hgetall(project_common_key) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/project_date_rank_service.rb b/app/services/cache/v2/project_date_rank_service.rb new file mode 100644 index 000000000..38c11c9b9 --- /dev/null +++ b/app/services/cache/v2/project_date_rank_service.rb @@ -0,0 +1,50 @@ +class Cache::V2::ProjectDateRankService < ApplicationService + attr_reader :project_id, :rank_date, :visits, :praises, :forks, :issues, :pullrequests + attr_accessor :project_common + + def initialize(project_id, rank_date=Date.today, params={}) + @project_id = project_id + @visits = params[:visits] + @praises = params[:praises] + @forks = params[:forks] + @issues = params[:issues] + @pullrequests = params[:pullrequests] + end + + def read + project_rank + end + + def call + set_project_rank + end + + private + def project_rank_key + "v2-project-rank-#{rank_date.to_s}" + end + + def project_rank + $redis_cache.zscore(project_rank_key, @project_id) + end + + def set_project_rank + if @visits.present? + $redis_cache.zincrby(project_rank_key, @visits.to_i * 1, @project_id) + end + if @praises.present? + $redis_cache.zincrby(project_rank_key, @praises.to_i * 5, @project_id) + end + if @forks.present? + $redis_cache.zincrby(project_rank_key, @forks.to_i * 5, @project_id) + end + if @issues.present? + $redis_cache.zincrby(project_rank_key, @issues.to_i * 10, @project_id) + end + if @pullrequests.present? + $redis_cache.zincrby(project_rank_key, @pullrequests.to_i * 10, @project_id) + end + + $redis_cache.zscore(project_rank_key, @project_id) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/project_rank_service.rb b/app/services/cache/v2/project_rank_service.rb new file mode 100644 index 000000000..089258d96 --- /dev/null +++ b/app/services/cache/v2/project_rank_service.rb @@ -0,0 +1,71 @@ +class Cache::V2::ProjectRankService < ApplicationService + attr_reader :project_id, :visits, :praises, :forks, :issues, :pullrequests + attr_accessor :project_common + + def initialize(project_id, params={}) + @project_id = project_id + @visits = params[:visits] + @praises = params[:praises] + @forks = params[:forks] + @issues = params[:issues] + @pullrequests = params[:pullrequests] + end + + def read + project_rank + end + + def call + set_project_rank + end + + def reset + reset_project_rank + end + + private + def load_project_common + @project_common = Cache::V2::ProjectCommonService.new(@project_id).read + end + + def project_rank_key + "v2-project-rank" + end + + def project_rank + $redis_cache.zscore(project_rank_key, @project_id).blank? ? reset_project_rank : $redis_cache.zscore(project_rank_key, @project_id) + end + + def set_project_rank + if $redis_cache.zscore(project_rank_key, @project_id).blank? + reset_project_rank + return + end + if @visits.present? + $redis_cache.zincrby(project_rank_key, @visits.to_i * 1, @project_id) + end + if @praises.present? + $redis_cache.zincrby(project_rank_key, @praises.to_i * 5, @project_id) + end + if @forks.present? + $redis_cache.zincrby(project_rank_key, @forks.to_i * 5, @project_id) + end + if @issues.present? + $redis_cache.zincrby(project_rank_key, @issues.to_i * 10, @project_id) + end + if @pullrequests.present? + $redis_cache.zincrby(project_rank_key, @pullrequests.to_i * 10, @project_id) + end + + $redis_cache.zscore(project_rank_key, @project_id) + end + + def reset_project_rank + load_project_common + $redis_cache.zrem(project_rank_key, @project_id) + score = @project_common["visits"].to_i * 1 + @project_common["praises"].to_i * 5 + @project_common["forks"].to_i * 5 + @project_common["issues"].to_i * 10 + @project_common["pullrequests"].to_i * 10 + $redis_cache.zincrby(project_rank_key, score, @project_id) + + $redis_cache.zscore(project_rank_key, @project_id) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/user_statistic_service.rb b/app/services/cache/v2/user_statistic_service.rb new file mode 100644 index 000000000..0df49c8ae --- /dev/null +++ b/app/services/cache/v2/user_statistic_service.rb @@ -0,0 +1,183 @@ +class Cache::V2::UserStatisticService < ApplicationService + attr_reader :user_id, :follow_count, :fork_count, :issue_count, :project_count, :project_language_count_key, :project_language_count, :project_praise_count, :project_watcher_count, :pullrequest_count + + def initialize(user_id, params={}) + @user_id = user_id + @follow_count = params[:follow_count] + @fork_count = params[:fork_count] + @issue_count = params[:issue_count] + @project_count = params[:project_count] + @project_language_count_key = params[:project_language_count_key] + @project_language_count = params[:project_language_count] + @project_praise_count = params[:project_praise_count] + @project_watcher_count = params[:project_watcher_count] + @pullrequest_count = params[:pullrequest_count] + end + + def read + user_statistic + end + + def call + set_user_statistic + end + + def reset + reset_user_statistic + end + + private + + def user_statistic_key + "v2-user-statistic:#{@user_id}" + end + + def follow_count_key + "follow-count" + end + + def fork_count_key + "fork-count" + end + + def issue_count_key + "issue-count" + end + + def project_count_key + "project-count" + end + + def project_language_key + "project-language" + end + + def project_praise_count_key + "project-praise-count" + end + + def project_watcher_count_key + "project-watcher-count" + end + + def pullrequest_count_key + "pullrequest-count" + end + + def user_statistic + $redis_cache.hgetall(user_statistic_key).blank? ? reset_user_statistic : $redis_cache.hgetall(user_statistic_key) + end + + def set_user_statistic + if $redis_cache.hgetall(user_statistic_key).blank? + reset_user_statistic + return + end + if @follow_count.present? + if $redis_cache.hget(user_statistic_key, follow_count_key).nil? + reset_user_follow_count + else + $redis_cache.hincrby(user_statistic_key, follow_count_key, @follow_count) + end + end + if @fork_count.present? + if $redis_cache.hget(user_statistic_key, fork_count_key).nil? + reset_user_fork_count + else + $redis_cache.hincrby(user_statistic_key, fork_count_key, @fork_count) + end + end + if @issue_count.present? + if $redis_cache.hget(user_statistic_key, issue_count_key).nil? + reset_user_issue_count + else + $redis_cache.hincrby(user_statistic_key, issue_count_key, @issue_count) + end + end + if @project_count.present? + if $redis_cache.hget(user_statistic_key, project_count_key).nil? + reset_user_project_count + else + $redis_cache.hincrby(user_statistic_key, project_count_key, @project_count) + end + end + if @project_language_count_key.present? && project_language_count.present? + if $redis_cache.hget(user_statistic_key, project_language_key).nil? + reset_user_project_language + else + result = JSON.parse($redis_cache.hget(user_statistic_key, project_language_key)) + result[@project_language_count_key] ||= 0 + result[@project_language_count_key] += project_language_count.to_i + $redis_cache.hset(user_statistic_key, project_language_key, result.to_json) + end + end + if @project_praise_count.present? + if $redis_cache.hget(user_statistic_key, project_praise_count_key).nil? + reset_user_project_praise_count + else + $redis_cache.hincrby(user_statistic_key, project_praise_count_key, @project_praise_count) + end + end + if @project_watcher_count.present? + if $redis_cache.hget(user_statistic_key, project_watcher_count_key).nil? + reset_user_project_watcher_count + else + $redis_cache.hincrby(user_statistic_key, project_watcher_count_key, @project_watcher_count) + end + end + if @pullrequest_count.present? + if $redis_cache.hget(user_statistic_key, pullrequest_count_key).nil? + reset_user_pullrequest_count + else + $redis_cache.hincrby(user_statistic_key, pullrequest_count_key, @pullrequest_count) + end + end + $redis_cache.hgetall(user_statistic_key) + end + + def reset_user_follow_count + $redis_cache.hset(user_statistic_key, follow_count_key, Watcher.where(watchable_type: 'User', watchable_id: @user_id).count) + end + + def reset_user_fork_count + $redis_cache.hset(user_statistic_key, fork_count_key, ForkUser.joins(:project).where(projects: {user_id: @user_id}).count) + end + + def reset_user_issue_count + $redis_cache.hset(user_statistic_key, issue_count_key, Issue.where(author_id: @user_id).count) + end + + def reset_user_project_count + $redis_cache.hset(user_statistic_key, project_count_key, Project.where(user_id: @user_id).count) + end + + def reset_user_project_language + $redis_cache.hset(user_statistic_key, project_language_key, Project.where(user_id: @user_id).joins(:project_language).group("project_languages.name").count.to_json) + end + + def reset_user_project_praise_count + $redis_cache.hset(user_statistic_key, project_praise_count_key, PraiseTread.where(praise_tread_object_type: 'Project', praise_tread_object_id: Project.where(user_id: @user_id)).count) + end + + def reset_user_project_watcher_count + $redis_cache.hset(user_statistic_key, project_watcher_count_key, Watcher.where(watchable_type: 'Project', watchable_id: Project.where(user_id: @user_id)).count) + end + + def reset_user_pullrequest_count + $redis_cache.hset(user_statistic_key, pullrequest_count_key, PullRequest.where(user_id: @user_id).count) + end + + def reset_user_statistic + $redis_cache.del(user_statistic_key) + reset_user_follow_count + reset_user_fork_count + reset_user_issue_count + reset_user_project_count + reset_user_project_language + reset_user_project_praise_count + reset_user_project_watcher_count + reset_user_pullrequest_count + + $redis_cache.hgetall(user_statistic_key) + end +end \ No newline at end of file From fa476732e500ddbb37d9396c62e23faa6df037fe Mon Sep 17 00:00:00 2001 From: yystopf Date: Mon, 25 Oct 2021 17:23:17 +0800 Subject: [PATCH 02/11] add: user common info cache service --- app/controllers/users_controller.rb | 2 ++ app/jobs/cache_async_reset_job.rb | 4 +--- app/jobs/cache_async_set_job.rb | 2 -- app/services/cache/v2/owner_common_service.rb | 14 +++++++++----- app/views/users/_cache_simple_user.json.jbuilder | 16 ++++++++++++++++ 5 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 app/views/users/_cache_simple_user.json.jbuilder diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index b80827efa..697835b0e 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -51,6 +51,8 @@ class UsersController < ApplicationController @projects_common_count = user_projects.common.size @projects_mirrior_count = user_projects.mirror.size @projects_sync_mirrior_count = user_projects.sync_mirror.size + # 为了缓存活跃用户的基本信息，后续删除 + Cache::V2::OwnerCommonService.new(@user.login, @user.mail).read end def watch_users diff --git a/app/jobs/cache_async_reset_job.rb b/app/jobs/cache_async_reset_job.rb index a7da3f80e..3b7810918 100644 --- a/app/jobs/cache_async_reset_job.rb +++ b/app/jobs/cache_async_reset_job.rb @@ -1,10 +1,8 @@ class CacheAsyncResetJob < ApplicationJob queue_as :cache - def perform(type, id, params={}) + def perform(type, id) case type - when "owner_common_service" - Cache::V2::OwnerCommonService.new(id).reset when "platform_statistic_service" Cache::V2::PlatformStatisticService.new.reset when "project_common_service" diff --git a/app/jobs/cache_async_set_job.rb b/app/jobs/cache_async_set_job.rb index d4f641e19..5a0eb4d83 100644 --- a/app/jobs/cache_async_set_job.rb +++ b/app/jobs/cache_async_set_job.rb @@ -3,8 +3,6 @@ class CacheAsyncResetJob < ApplicationJob def perform(type, id, params={}) case type - when "owner_common_service" - Cache::V2::OwnerCommonService.new(id, params).call when "platform_statistic_service" Cache::V2::PlatformStatisticService.new(params).call when "project_common_service" diff --git a/app/services/cache/v2/owner_common_service.rb b/app/services/cache/v2/owner_common_service.rb index d65a60749..07183f811 100644 --- a/app/services/cache/v2/owner_common_service.rb +++ b/app/services/cache/v2/owner_common_service.rb @@ -3,10 +3,10 @@ class Cache::V2::OwnerCommonService < ApplicationService attr_reader :owner_id, :login, :name, :avatar_url, :email attr_accessor :owner - def initialize(owner_id, params={}) - @owner_id = owner_id + def initialize(login, email, params={}) + @login = login + @email = email @name = params[:name] - @email = params[:email] end def read @@ -24,11 +24,11 @@ class Cache::V2::OwnerCommonService < ApplicationService private def load_owner - @owner = User.find_by_id(@owner_id) + @owner = User.find_by(login: @login) end def owner_common_key - "v2-owner-common:#{@owner_id}" + "v2-owner-common:#{@login}-#{@email.to_s}" end def owner_common @@ -58,6 +58,9 @@ class Cache::V2::OwnerCommonService < ApplicationService $redis_cache.hgetall(owner_common_key) end + def reset_owner_id + $redis_cache.hset(owner_common_key, "id", owner&.id) + end def reset_owner_type $redis_cache.hset(owner_common_key, "type", owner&.type) @@ -79,6 +82,7 @@ class Cache::V2::OwnerCommonService < ApplicationService def reset_owner_common load_owner $redis_cache.del(owner_common_key) + reset_owner_id reset_owner_type reset_owner_login reset_owner_email diff --git a/app/views/users/_cache_simple_user.json.jbuilder b/app/views/users/_cache_simple_user.json.jbuilder new file mode 100644 index 000000000..60536e071 --- /dev/null +++ b/app/views/users/_cache_simple_user.json.jbuilder @@ -0,0 +1,16 @@ +user = $redis_cache.hgetall("v2-owner-common:#{name}-#{email}") +if user.blank? + json.id nil + json.type nil + json.login name + json.name name + json.email email + json.image_url User::Avatar.get_letter_avatar_url(name) +else + json.id user["id"] + json.type user["type"] + json.login user["login"] + json.name user["name"] + json.email user["email"] + json.image_url user["avatar_url"] +end \ No newline at end of file From 91f1f4090f173c95231e9a77c4ddcd8c6a228a09 Mon Sep 17 00:00:00 2001 From: yystopf Date: Mon, 25 Oct 2021 18:03:56 +0800 Subject: [PATCH 03/11] fix: old cache --- .../users/statistics_controller.rb | 34 +++++++++-------- app/jobs/cache_async_reset_job.rb | 4 +- app/jobs/cache_async_set_job.rb | 4 +- app/models/application_record.rb | 8 ---- app/models/fork_user.rb | 4 +- app/models/issue.rb | 4 +- app/models/praise_tread.rb | 38 +++++++++---------- app/models/project.rb | 7 ++-- app/models/pull_request.rb | 4 +- app/models/watcher.rb | 6 +-- 10 files changed, 53 insertions(+), 60 deletions(-) diff --git a/app/controllers/users/statistics_controller.rb b/app/controllers/users/statistics_controller.rb index 592a8be94..dffd3f607 100644 --- a/app/controllers/users/statistics_controller.rb +++ b/app/controllers/users/statistics_controller.rb @@ -188,30 +188,32 @@ class Users::StatisticsController < Users::BaseController @project_languages_count = time_filter(Project.where(user_id: observed_user.id), 'created_on').joins(:project_language).group("project_languages.name").count @platform_project_languages_count = time_filter(Project, 'created_on').joins(:project_language).group("project_languages.name").count else + @platform_result = Cache::V2::PlatformStatisticService.new.read + @user_result = Cache::V2::UserStatisticService.new(observed_user.id).read # 用户被follow数量 - @follow_count = Cache::UserFollowCountService.call(observed_user) - @platform_follow_count = Cache::PlatformFollowCountService.call + @follow_count = @user_result["follow-count"].to_i + @platform_follow_count = @platform_result["follow-count"].to_i # 用户pr数量 - @pullrequest_count = Cache::UserPullrequestCountService.call(observed_user) - @platform_pullrequest_count = Cache::PlatformPullrequestCountService.call + @pullrequest_count = @user_result["pullrequest-count"].to_i + @platform_pullrequest_count = @platform_result["pullrequest-count"].to_i # 用户issue数量 - @issues_count = Cache::UserIssueCountService.call(observed_user) - @platform_issues_count = Cache::PlatformIssueCountService.call + @issues_count = @user_result["issue-count"].to_i + @platform_issues_count = @platform_result["issue-count"].to_i # 用户总项目数 - @project_count = Cache::UserProjectCountService.call(observed_user) - @platform_project_count = Cache::PlatformProjectCountService.call + @project_count = @user_result["project-count"].to_i + @platform_project_count = @platform_result["project-count"].to_i # 用户项目被fork数量 - @fork_count = Cache::UserProjectForkCountService.call(observed_user) - @platform_fork_count = Cache::PlatformProjectForkCountService.call + @fork_count = @user_result["fork-count"].to_i + @platform_fork_count = @platform_result["fork-count"].to_i # 用户项目关注数 - @project_watchers_count = Cache::UserProjectWatchersCountService.call(observed_user) - @platform_project_watchers_count = Cache::PlatformProjectWatchersCountService.call + @project_watchers_count = @user_result["project-watcher-count"].to_i + @platform_project_watchers_count = @platform_result["project-watcher-count"].to_i # 用户项目点赞数 - @project_praises_count = Cache::UserProjectPraisesCountService.call(observed_user) - @platform_project_praises_count = Cache::PlatformProjectPraisesCountService.call + @project_praises_count = @user_result["project-praise-count"].to_i + @platform_project_praises_count = @platform_result["project-praise-count"].to_i # 用户不同语言项目数量 - @project_languages_count = Cache::UserProjectLanguagesCountService.call(observed_user) - @platform_project_languages_count = Cache::PlatformProjectLanguagesCountService.call + @project_languages_count = JSON.parse(@user_result["project-language"]) + @platform_project_languages_count = JSON.parse(@platform_result["project-language"]) end end end \ No newline at end of file diff --git a/app/jobs/cache_async_reset_job.rb b/app/jobs/cache_async_reset_job.rb index 3b7810918..da3f01168 100644 --- a/app/jobs/cache_async_reset_job.rb +++ b/app/jobs/cache_async_reset_job.rb @@ -1,14 +1,14 @@ class CacheAsyncResetJob < ApplicationJob queue_as :cache - def perform(type, id) + def perform(type, id=nil) case type when "platform_statistic_service" Cache::V2::PlatformStatisticService.new.reset when "project_common_service" Cache::V2::ProjectCommonService.new(id).reset when "user_statistic_service" - Cache::V2::PlatformStatisticService.new(id).reset + Cache::V2::UserStatisticService.new(id).reset end end end \ No newline at end of file diff --git a/app/jobs/cache_async_set_job.rb b/app/jobs/cache_async_set_job.rb index 5a0eb4d83..1c67f91a6 100644 --- a/app/jobs/cache_async_set_job.rb +++ b/app/jobs/cache_async_set_job.rb @@ -1,14 +1,14 @@ class CacheAsyncResetJob < ApplicationJob queue_as :cache - def perform(type, id, params={}) + def perform(type, id=nil, params={}) case type when "platform_statistic_service" Cache::V2::PlatformStatisticService.new(params).call when "project_common_service" Cache::V2::ProjectCommonService.new(id, params).call when "user_statistic_service" - Cache::V2::PlatformStatisticService.new(id, params).call + Cache::V2::UserStatisticService.new(id, params).call end end end \ No newline at end of file diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 0b95d7a58..77e5fe2db 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -17,14 +17,6 @@ class ApplicationRecord < ActiveRecord::Base Rails.env.production? && EduSetting.get('host_name') == 'https://www.educoder.net' end - def reset_user_cache_async_job(user) - ResetUserCacheJob.perform_later(user) - end - - def reset_platform_cache_async_job - ResetPlatformCacheJob.perform_later - end - def self.strip_param(key) key.to_s.strip.presence end diff --git a/app/models/fork_user.rb b/app/models/fork_user.rb index 0936f6bfa..0365af72b 100644 --- a/app/models/fork_user.rb +++ b/app/models/fork_user.rb @@ -24,8 +24,8 @@ class ForkUser < ApplicationRecord after_destroy :reset_cache_data def reset_cache_data - self.reset_platform_cache_async_job - self.reset_user_cache_async_job(self.project.owner) + CacheAsyncResetJob.perform_later("platform_statistic_service") + CacheAsyncResetJob.perform_later("user_statistic_service", self.project&.user_id) end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 826ad3a5b..d195837a7 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -79,8 +79,8 @@ class Issue < ApplicationRecord after_destroy :update_closed_issues_count_in_project!, :reset_cache_data def reset_cache_data - self.reset_platform_cache_async_job - self.reset_user_cache_async_job(self.user) + CacheAsyncResetJob.perform_later("platform_statistic_service") + CacheAsyncResetJob.perform_later("user_statistic_service", self.author_id) end def get_assign_user diff --git a/app/models/praise_tread.rb b/app/models/praise_tread.rb index 5a9c19164..30d226735 100644 --- a/app/models/praise_tread.rb +++ b/app/models/praise_tread.rb @@ -1,20 +1,20 @@ -# == Schema Information -# -# Table name: praise_treads -# -# id :integer not null, primary key -# user_id :integer not null -# praise_tread_object_id :integer -# praise_tread_object_type :string(255) -# praise_or_tread :integer default("1") -# created_at :datetime not null -# updated_at :datetime not null -# -# Indexes -# -# praise_tread (praise_tread_object_id,praise_tread_object_type) -# - +# == Schema Information +# +# Table name: praise_treads +# +# id :integer not null, primary key +# user_id :integer not null +# praise_tread_object_id :integer +# praise_tread_object_type :string(255) +# praise_or_tread :integer default("1") +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# praise_tread (praise_tread_object_id,praise_tread_object_type) +# + class PraiseTread < ApplicationRecord belongs_to :user @@ -26,9 +26,9 @@ class PraiseTread < ApplicationRecord after_destroy :reset_cache_data def reset_cache_data - self.reset_platform_cache_async_job + CacheAsyncResetJob.perform_later("platform_statistic_service") if self.praise_tread_object.is_a?(Project) - self.reset_user_cache_async_job(self.praise_tread_object&.owner) + CacheAsyncResetJob.perform_later("user_statistic_service", self.praise_tread_object&.user_id) end end diff --git a/app/models/project.rb b/app/models/project.rb index 293e6c478..d9933e13d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -151,11 +151,10 @@ class Project < ApplicationRecord def reset_cache_data if changes[:user_id].present? - first_owner = Owner.find_by_id(changes[:user_id].first) - self.reset_user_cache_async_job(first_owner) + CacheAsyncResetJob.perform_later("user_statistic_service", changes[:user_id].first) end - self.reset_platform_cache_async_job - self.reset_user_cache_async_job(self.owner) + CacheAsyncResetJob.perform_later("platform_statistic_service") + CacheAsyncResetJob.perform_later("user_statistic_service", self.user_id) end def reset_unmember_followed diff --git a/app/models/pull_request.rb b/app/models/pull_request.rb index ce0cb04ad..9c6825f27 100644 --- a/app/models/pull_request.rb +++ b/app/models/pull_request.rb @@ -46,8 +46,8 @@ class PullRequest < ApplicationRecord after_destroy :reset_cache_data def reset_cache_data - self.reset_platform_cache_async_job - self.reset_user_cache_async_job(self.user) + CacheAsyncResetJob.perform_later("platform_statistic_service") + CacheAsyncResetJob.perform_later("user_statistic_service", self.user_id) end def fork_project diff --git a/app/models/watcher.rb b/app/models/watcher.rb index 6a8c94fcc..abbcf07bc 100644 --- a/app/models/watcher.rb +++ b/app/models/watcher.rb @@ -28,12 +28,12 @@ class Watcher < ApplicationRecord def reset_cache_data if self.watchable.is_a?(User) - self.reset_user_cache_async_job(self.watchable) + CacheAsyncResetJob.perform_later("user_statistic_service", self.watchable_id) end if self.watchable.is_a?(Project) - self.reset_user_cache_async_job(self.watchable&.owner) + CacheAsyncResetJob.perform_later("user_statistic_service", self.watchable&.user_id) end - self.reset_platform_cache_async_job + CacheAsyncResetJob.perform_later("platform_statistic_service") end def send_create_message_to_notice_system From 69c87fd5a4f431a355c6bd5247c87750b523026d Mon Sep 17 00:00:00 2001 From: yystopf Date: Tue, 26 Oct 2021 15:22:42 +0800 Subject: [PATCH 04/11] add: callback cache --- app/controllers/projects_controller.rb | 2 + app/controllers/repositories_controller.rb | 2 +- app/jobs/cache_async_set_job.rb | 4 +- app/models/fork_user.rb | 28 ++++- app/models/issue.rb | 29 ++++- app/models/praise_tread.rb | 32 +++-- app/models/project.rb | 43 ++++++- app/models/pull_request.rb | 29 ++++- app/models/user.rb | 6 +- app/models/watcher.rb | 40 ++++-- .../cache/v2/project_common_service.rb | 41 ++++-- .../cache/v2/project_date_rank_service.rb | 3 +- .../cache/v2/user_date_rank_service.rb | 118 ++++++++++++++++++ .../cache/v2/user_statistic_service.rb | 17 +++ 14 files changed, 343 insertions(+), 51 deletions(-) create mode 100644 app/services/cache/v2/user_date_rank_service.rb diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 1c6e9868c..b1ab9d010 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -190,6 +190,8 @@ class ProjectsController < ApplicationController end def simple + # 为了缓存活跃项目的基本信息，后续删除 + Cache::V2::ProjectCommonService.new(@project.id).read json_response(@project, current_user) end diff --git a/app/controllers/repositories_controller.rb b/app/controllers/repositories_controller.rb index 35f11f476..9b94f0d5d 100644 --- a/app/controllers/repositories_controller.rb +++ b/app/controllers/repositories_controller.rb @@ -48,7 +48,7 @@ class RepositoriesController < ApplicationController def entries @project.increment!(:visits) - + CacheAsyncSetJob.perform_later("project_common_service", {visits: 1}, @project.id) if @project.educoder? @entries = Educoder::Repository::Entries::ListService.call(@project&.project_educoder.repo_name) else diff --git a/app/jobs/cache_async_set_job.rb b/app/jobs/cache_async_set_job.rb index 1c67f91a6..f617b1316 100644 --- a/app/jobs/cache_async_set_job.rb +++ b/app/jobs/cache_async_set_job.rb @@ -1,7 +1,7 @@ -class CacheAsyncResetJob < ApplicationJob +class CacheAsyncSetJob < ApplicationJob queue_as :cache - def perform(type, id=nil, params={}) + def perform(type, params={}, id=nil) case type when "platform_statistic_service" Cache::V2::PlatformStatisticService.new(params).call diff --git a/app/models/fork_user.rb b/app/models/fork_user.rb index 0365af72b..bddf8f75c 100644 --- a/app/models/fork_user.rb +++ b/app/models/fork_user.rb @@ -20,12 +20,30 @@ class ForkUser < ApplicationRecord belongs_to :user belongs_to :fork_project, class_name: 'Project', foreign_key: :fork_project_id - after_save :reset_cache_data - after_destroy :reset_cache_data + after_create :incre_project_common, :incre_user_statistic, :incre_platform_statistic + after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic - def reset_cache_data - CacheAsyncResetJob.perform_later("platform_statistic_service") - CacheAsyncResetJob.perform_later("user_statistic_service", self.project&.user_id) + def incre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {forks: 1}, self.project_id) end + def decre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {forks: -1}, self.project_id) + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {fork_count: 1}, self.project&.user_id) + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {fork_count: -1}, self.project&.user_id) + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {fork_count: 1}) + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {fork_count: -1}) + end end diff --git a/app/models/issue.rb b/app/models/issue.rb index d195837a7..c642e642b 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -74,13 +74,32 @@ class Issue < ApplicationRecord scope :issue_pull_request, ->{where(issue_classify: "pull_request")} scope :issue_index_includes, ->{includes(:tracker, :priority, :version, :issue_status, :journals,:issue_tags,user: :user_extension)} scope :closed, ->{where(status_id: 5)} + after_create :incre_project_common, :incre_user_statistic, :incre_platform_statistic after_update :change_versions_count - after_save :reset_cache_data - after_destroy :update_closed_issues_count_in_project!, :reset_cache_data + after_destroy :update_closed_issues_count_in_project!, :decre_project_common, :decre_user_statistic, :decre_platform_statistic - def reset_cache_data - CacheAsyncResetJob.perform_later("platform_statistic_service") - CacheAsyncResetJob.perform_later("user_statistic_service", self.author_id) + def incre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {issues: 1}, self.project_id) + end + + def decre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {issues: -1}, self.project_id) + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {issue_count: 1}, self.author_id) + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {issue_count: -1}, self.author_id) + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {issue_count: 1}) + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {issue_count: -1}) end def get_assign_user diff --git a/app/models/praise_tread.rb b/app/models/praise_tread.rb index 30d226735..5d4ae0d80 100644 --- a/app/models/praise_tread.rb +++ b/app/models/praise_tread.rb @@ -21,15 +21,31 @@ class PraiseTread < ApplicationRecord belongs_to :praise_tread_object, polymorphic: true, counter_cache: :praises_count has_many :tidings, :as => :container, :dependent => :destroy - after_create :send_tiding - after_save :reset_cache_data - after_destroy :reset_cache_data + after_create :send_tiding, :incre_project_common, :incre_user_statistic, :incre_platform_statistic + after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic - def reset_cache_data - CacheAsyncResetJob.perform_later("platform_statistic_service") - if self.praise_tread_object.is_a?(Project) - CacheAsyncResetJob.perform_later("user_statistic_service", self.praise_tread_object&.user_id) - end + def incre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {praises: 1}, self.praise_tread_object_id) if self.praise_tread_object_type == "Project" + end + + def decre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {praises: -1}, self.praise_tread_object_id) if self.praise_tread_object_type == "Project" + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {project_praise_count: 1}, self.praise_tread_object&.user_id) if self.praise_tread_object_type == "Project" + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {project_praise_count: -1}, self.praise_tread_object&.user_id) if self.praise_tread_object_type == "Project" + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_praise_count: 1}) if self.praise_tread_object_type == "Project" + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_praise_count: -1}) if self.praise_tread_object_type == "Project" end def send_tiding diff --git a/app/models/project.rb b/app/models/project.rb index d9933e13d..78c049ab2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -126,10 +126,10 @@ class Project < ApplicationRecord has_many :pinned_projects, dependent: :destroy has_many :has_pinned_users, through: :pinned_projects, source: :user has_many :webhooks, class_name: "Gitea::Webhook", primary_key: :gpid, foreign_key: :repo_id - + after_create :init_project_common, :incre_user_statistic, :incre_platform_statistic after_save :check_project_members before_save :set_invite_code, :reset_cache_data, :reset_unmember_followed - after_destroy :reset_cache_data + after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic scope :project_statics_select, -> {select(:id,:name, :is_public, :identifier, :status, :project_type, :user_id, :forked_count, :visits, :project_category_id, :project_language_id, :license_id, :ignore_id, :watchers_count, :created_on)} scope :no_anomory_projects, -> {where("projects.user_id is not null and projects.user_id != ?", 2)} scope :recommend, -> { visible.project_statics_select.where(recommend: true) } @@ -150,13 +150,46 @@ class Project < ApplicationRecord end def reset_cache_data + CacheAsyncResetJob.perform_later("project_common_service", self.id) if changes[:user_id].present? - CacheAsyncResetJob.perform_later("user_statistic_service", changes[:user_id].first) + CacheAsyncSetJob.perform_later("user_statistic_service", {project_count: -1}, changes[:user_id].first) + CacheAsyncSetJob.perform_later("user_statistic_service", {project_count: 1}, changes[:user_id].last) + end + if changes[:project_language_id].present? + first_language = ProjectLanguage.find_by_id(changes[:project_language_id].first) + last_language = ProjectLanguage.find_by_id(changes[:project_language_id].last) + CacheAsyncSetJob.perform_later("user_statistic_service", {project_language_count_key: first_language&.name, project_language_count: -1}, self.user_id) + CacheAsyncSetJob.perform_later("user_statistic_service", {project_language_count_key: last_language&.name, project_language_count: 1}, self.user_id) + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_language_count_key: first_language&.name, project_language_count: -1}) + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_language_count_key: last_language&.name, project_language_count: 1}) end - CacheAsyncResetJob.perform_later("platform_statistic_service") - CacheAsyncResetJob.perform_later("user_statistic_service", self.user_id) end + def init_project_common + CacheAsyncResetJob.perform_later("project_common_service", self.id) + end + + def decre_project_common + $redis_cache.del("v2-project-common:#{self.id}") + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {project_count: 1, project_language_count_key: self.project_language&.name, project_language_count: 1}, self.user_id) + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {project_count: -1, project_language_count_key: self.project_language&.name, project_language_count: -1}, self.user_id) + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_count: 1, project_language_count_key: self.project_language&.name, project_language_count: 1}) + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_count: -1, project_language_count_key: self.project_language&.name, project_language_count: -1}) + end + + def reset_unmember_followed if changes[:is_public].present? && changes[:is_public] == [true, false] self.watchers.where.not(user_id: self.all_collaborators).destroy_all diff --git a/app/models/pull_request.rb b/app/models/pull_request.rb index 9c6825f27..7338a1d72 100644 --- a/app/models/pull_request.rb +++ b/app/models/pull_request.rb @@ -42,12 +42,31 @@ class PullRequest < ApplicationRecord scope :merged_and_closed, ->{where.not(status: 0)} scope :opening, -> {where(status: 0)} - after_save :reset_cache_data - after_destroy :reset_cache_data + after_create :incre_project_common, :incre_user_statistic, :incre_platform_statistic + after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic - def reset_cache_data - CacheAsyncResetJob.perform_later("platform_statistic_service") - CacheAsyncResetJob.perform_later("user_statistic_service", self.user_id) + def incre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {pullrequests: 1}, self.project_id) + end + + def decre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {pullrequests: -1}, self.project_id) + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {pullrequest_count: 1}, self.user_id) + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {pullrequest_count: -1}, self.user_id) + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {pullrequest_count: 1}) + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {pullrequest_count: -1}) end def fork_project diff --git a/app/models/user.rb b/app/models/user.rb index fc4c33618..7a172020c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -188,7 +188,7 @@ class User < Owner :show_email, :show_location, :show_department, :technical_title, :province, :city, :custom_department, to: :user_extension, allow_nil: true - before_save :update_hashed_password, :set_lastname + before_save :update_hashed_password, :set_lastname, :reset_cache_data after_create do SyncTrustieJob.perform_later("user", 1) if allow_sync_to_trustie? end @@ -205,6 +205,10 @@ class User < Owner validate :validate_sensitive_string validate :validate_password_length + def reset_cache_data + Cache::V2::OwnerCommonService.new(self.login, self.mail).reset + end + # 用户参与的所有项目 def full_member_projects normal_projects = Project.members_projects(self.id).to_sql diff --git a/app/models/watcher.rb b/app/models/watcher.rb index abbcf07bc..5a2cd96fb 100644 --- a/app/models/watcher.rb +++ b/app/models/watcher.rb @@ -22,18 +22,36 @@ class Watcher < ApplicationRecord scope :watching_users, ->(watchable_id){ where("watchable_type = ? and user_id = ?",'User',watchable_id)} - after_save :reset_cache_data - after_destroy :reset_cache_data - after_create :send_create_message_to_notice_system + after_create :send_create_message_to_notice_system, :incre_project_common, :incre_user_statistic, :incre_platform_statistic + after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic - def reset_cache_data - if self.watchable.is_a?(User) - CacheAsyncResetJob.perform_later("user_statistic_service", self.watchable_id) - end - if self.watchable.is_a?(Project) - CacheAsyncResetJob.perform_later("user_statistic_service", self.watchable&.user_id) - end - CacheAsyncResetJob.perform_later("platform_statistic_service") + + def incre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {watchers: 1}, self.watchable_id) if self.watchable_type == "Project" + end + + def decre_project_common + CacheAsyncSetJob.perform_later("project_common_service", {watchers: -1}, self.watchable_id) if self.watchable_type == "Project" + end + + def incre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {follow_count: 1}, self.watchable_id) if self.watchable_type == "User" + CacheAsyncSetJob.perform_later("user_statistic_service", {project_watcher_count: 1}, self.watchable&.user_id) if self.watchable_type == "Project" + end + + def decre_user_statistic + CacheAsyncSetJob.perform_later("user_statistic_service", {follow_count: -1}, self.watchable_id) if self.watchable_type == "User" + CacheAsyncSetJob.perform_later("user_statistic_service", {project_watcher_count: -1}, self.watchable&.user_id) if self.watchable_type == "Project" + end + + def incre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {follow_count: 1}) if self.watchable_type == "User" + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_watcher_count: 1}) if self.watchable_type == "Project" + end + + def decre_platform_statistic + CacheAsyncSetJob.perform_later("platform_statistic_service", {follow_count: -1}) if self.watchable_type == "User" + CacheAsyncSetJob.perform_later("platform_statistic_service", {project_watcher_count: -1}) if self.watchable_type == "Project" end def send_create_message_to_notice_system diff --git a/app/services/cache/v2/project_common_service.rb b/app/services/cache/v2/project_common_service.rb index 365f07a6e..6ea4bb3a9 100644 --- a/app/services/cache/v2/project_common_service.rb +++ b/app/services/cache/v2/project_common_service.rb @@ -1,5 +1,5 @@ class Cache::V2::ProjectCommonService < ApplicationService - attr_reader :project_id, :owner_id, :name, :identifier, :visits, :watchers, :praises, :forks, :issues, :pullrequests + attr_reader :project_id, :owner_id, :name, :identifier, :description, :visits, :watchers, :praises, :forks, :issues, :pullrequests attr_accessor :project def initialize(project_id, params={}) @@ -7,6 +7,7 @@ class Cache::V2::ProjectCommonService < ApplicationService @owner_id = params[:owner_id] @name = params[:name] @identifier = params[:identifier] + @description = params[:description] @visits = params[:visits] @watchers = params[:watchers] @praises = params[:praises] @@ -48,6 +49,10 @@ class Cache::V2::ProjectCommonService < ApplicationService "identifier" end + def description_key + "description" + end + def visits_key "visits" end @@ -102,13 +107,22 @@ class Cache::V2::ProjectCommonService < ApplicationService $redis_cache.hset(project_common_key, identifier_key, @identifier) end end + if @description.present? + if $redis_cache.hget(project_common_key, description_key).nil? + reset_project_description + else + $redis_cache.hset(project_common_key, description_key, @description) + end + end if @visits.present? if $redis_cache.hget(project_common_key, visits_key).nil? reset_project_visits + Cache::V2::ProjectRankService.call(@project_id, {visits: @visits}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {visits: @visits}) else $redis_cache.hincrby(project_common_key, visits_key, @visits) Cache::V2::ProjectRankService.call(@project_id, {visits: @visits}) - Cache::V2::ProjectDateRankService.call(@project_id, {visits: @visits}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {visits: @visits}) end end if @watchers.present? @@ -121,37 +135,45 @@ class Cache::V2::ProjectCommonService < ApplicationService if @praises.present? if $redis_cache.hget(project_common_key, praises_key).nil? reset_project_praises + Cache::V2::ProjectRankService.call(@project_id, {praises: @praises}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {praises: @praises}) else $redis_cache.hincrby(project_common_key, praises_key, @praises) Cache::V2::ProjectRankService.call(@project_id, {praises: @praises}) - Cache::V2::ProjectDateRankService.call(@project_id, {praises: @praises}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {praises: @praises}) end end if @forks.present? if $redis_cache.hget(project_common_key, forks_key).nil? reset_project_forks + Cache::V2::ProjectRankService.call(@project_id, {forks: @forks}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {forks: @forks}) else $redis_cache.hincrby(project_common_key, forks_key, @forks) Cache::V2::ProjectRankService.call(@project_id, {forks: @forks}) - Cache::V2::ProjectDateRankService.call(@project_id, {forks: @forks}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {forks: @forks}) end end if @issues.present? if $redis_cache.hget(project_common_key, issues_key).nil? reset_project_issues + Cache::V2::ProjectRankService.call(@project_id, {issues: @issues}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {issues: @issues}) else $redis_cache.hincrby(project_common_key, issues_key, @issues) Cache::V2::ProjectRankService.call(@project_id, {issues: @issues}) - Cache::V2::ProjectDateRankService.call(@project_id, {issues: @issues}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {issues: @issues}) end end if @pullrequests.present? if $redis_cache.hget(project_common_key, pullrequests_key).nil? reset_project_pullrequests + Cache::V2::ProjectRankService.call(@project_id, {pullrequests: @pullrequests}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {pullrequests: @pullrequests}) else $redis_cache.hincrby(project_common_key, pullrequests_key, @pullrequests) Cache::V2::ProjectRankService.call(@project_id, {pullrequests: @pullrequests}) - Cache::V2::ProjectDateRankService.call(@project_id, {pullrequests: @pullrequests}) + Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {pullrequests: @pullrequests}) end end $redis_cache.hgetall(project_common_key) @@ -169,6 +191,10 @@ class Cache::V2::ProjectCommonService < ApplicationService $redis_cache.hset(project_common_key, identifier_key, @project&.identifier) end + def reset_project_description + $redis_cache.hset(project_common_key, description_key, @project&.description) + end + def reset_project_visits $redis_cache.hset(project_common_key, visits_key, @project&.visits) end @@ -186,7 +212,7 @@ class Cache::V2::ProjectCommonService < ApplicationService end def reset_project_issues - $redis_cache.hset(project_common_key, issues_key, Issue.where(project_id: @project_id).count) + $redis_cache.hset(project_common_key, issues_key, Issue.issue_issue.where(project_id: @project_id).count) end def reset_project_pullrequests @@ -199,6 +225,7 @@ class Cache::V2::ProjectCommonService < ApplicationService reset_project_owner_id reset_project_name reset_project_identifier + reset_project_description reset_project_visits reset_project_watchers reset_project_praises diff --git a/app/services/cache/v2/project_date_rank_service.rb b/app/services/cache/v2/project_date_rank_service.rb index 38c11c9b9..092aff796 100644 --- a/app/services/cache/v2/project_date_rank_service.rb +++ b/app/services/cache/v2/project_date_rank_service.rb @@ -4,6 +4,7 @@ class Cache::V2::ProjectDateRankService < ApplicationService def initialize(project_id, rank_date=Date.today, params={}) @project_id = project_id + @rank_date = rank_date @visits = params[:visits] @praises = params[:praises] @forks = params[:forks] @@ -21,7 +22,7 @@ class Cache::V2::ProjectDateRankService < ApplicationService private def project_rank_key - "v2-project-rank-#{rank_date.to_s}" + "v2-project-rank-#{@rank_date.to_s}" end def project_rank diff --git a/app/services/cache/v2/user_date_rank_service.rb b/app/services/cache/v2/user_date_rank_service.rb new file mode 100644 index 000000000..583f03a22 --- /dev/null +++ b/app/services/cache/v2/user_date_rank_service.rb @@ -0,0 +1,118 @@ +class Cache::V2::UserDateRankService < ApplicationService + attr_reader :user_id, :rank_date, :follow_count, :fork_count, :issue_count, :project_count, :project_language_count_key, :project_language_count, :project_praise_count, :project_watcher_count, :pullrequest_count + + def initialize(user_id, rank_date=Date.today, params={}) + @user_id = user_id + @rank_date = rank_date + @follow_count = params[:follow_count] + @fork_count = params[:fork_count] + @issue_count = params[:issue_count] + @project_count = params[:project_count] + @project_language_count_key = params[:project_language_count_key] + @project_language_count = params[:project_language_count] + @project_praise_count = params[:project_praise_count] + @project_watcher_count = params[:project_watcher_count] + @pullrequest_count = params[:pullrequest_count] + end + + def read_rank + user_rank + end + + def read_statistic + user_statistic + end + + def call + set_user_rank + end + + private + def user_rank_key + "v2-user-rank-#{@rank_date.to_s}" + end + + def user_date_statistic_key + "v2-user-statistic:#{@user_id}-#{@rank_date.to_s}" + end + + def user_rank + $redis_cache.zscore(user_rank_key, @user_id) + end + + def user_statistic + $redis_cache.hgetall(user_date_statistic_key) + end + + def set_user_statistic + if @follow_count.present? + $redis_cache.hincrby(user_date_statistic_key, "follow-count", @follow_count.to_i) + end + if @fork_count.present? + $redis_cache.hincrby(user_date_statistic_key, "fork-count", @fork_count.to_i) + end + if @issue_count.present? + $redis_cache.hincrby(user_date_statistic_key, "issue-count", @issue_count.to_i) + end + if @project_count.present? + $redis_cache.hincrby(user_date_statistic_key, "project-count", @project_count.to_i) + end + if project_language_count_key.present? && project_language_count.present? + if $redis_cache.hget(user_date_statistic_key, "project-language").nil? + result = {} + result[@project_language_count_key] = project_language_count.to_i + result.delete(@project_language_count_key) if result[@project_language_count_key] == 0 + $redis_cache.hset(user_date_statistic_key, "project-language", result.to_json) + else + result = JSON.parse($redis_cache.hget(user_date_statistic_key, "project-language")) + result[@project_language_count_key] ||= 0 + result[@project_language_count_key] += project_language_count.to_i + result.delete(@project_language_count_key) if result[@project_language_count_key] == 0 + $redis_cache.hset(user_date_statistic_key, "project-language", result.to_json) + end + end + if @project_praise_count.present? + $redis_cache.hincrby(user_date_statistic_key, "project-praise-count", @project_praise_count.to_i) + end + if @project_watcher_count.present? + $redis_cache.hincrby(user_date_statistic_key, "project-watcher-count", @project_watcher_count.to_i) + end + if @pullrequest_count.present? + $redis_cache.hincrby(user_date_statistic_key, "pullrequest-count", @pullrequest_count.to_i) + end + + $redis_cache.hgetall(user_date_statistic_key) + end + + def set_user_rank + set_user_statistic + follow_count = $redis_cache.hget(user_date_statistic_key, "follow-count") || 0 + pullrequest_count = $redis_cache.hget(user_date_statistic_key, "pullrequest-count") || 0 + issues_count = $redis_cache.hget(user_date_statistic_key, "issue-count") || 0 + project_count = $redis_cache.hget(user_date_statistic_key, "project-count") || 0 + fork_count = $redis_cache.hget(user_date_statistic_key, "fork-count") || 0 + project_watchers_count = $redis_cache.hget(user_date_statistic_key, "project-watcher-count") || 0 + project_praises_count = $redis_cache.hget(user_date_statistic_key, "project-praise-count") || 0 + project_languages_count = $redis_cache.hget(user_date_statistic_key, "project-language").nil? ? 0 : $redis_cache.hget(user_date_statistic_key, "project-language").length + # 影响力 + influence = (60.0 + follow_count.to_i / (follow_count.to_i + 20.0) * 40.0).to_i + + # 贡献度 + contribution = (60.0 + pullrequest_count.to_i / (pullrequest_count.to_i + 20.0) * 40.0).to_i + + # 活跃度 + activity = (60.0 + issues_count.to_i / (issues_count.to_i + 80.0) * 40.0).to_i + + # 项目经验 + experience = 10 * project_count.to_i + 5 * fork_count.to_i + project_watchers_count.to_i + project_praises_count.to_i + experience = (60.0 + experience / (experience + 100.0) * 40.0).to_i + # 语言能力 + language = (60.0 + project_languages_count.to_i / (project_languages_count.to_i + 5.0) * 40.0).to_i + + score = influence+ contribution + activity + experience + language + + $redis_cache.zadd(user_rank_key, score, @user_id) if score.to_i > 300 + + $redis_cache.zscore(user_rank_key, @user_id) + end +end \ No newline at end of file diff --git a/app/services/cache/v2/user_statistic_service.rb b/app/services/cache/v2/user_statistic_service.rb index 0df49c8ae..8caa66d88 100644 --- a/app/services/cache/v2/user_statistic_service.rb +++ b/app/services/cache/v2/user_statistic_service.rb @@ -76,60 +76,77 @@ class Cache::V2::UserStatisticService < ApplicationService if @follow_count.present? if $redis_cache.hget(user_statistic_key, follow_count_key).nil? reset_user_follow_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {follow_count: @follow_count}) else $redis_cache.hincrby(user_statistic_key, follow_count_key, @follow_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {follow_count: @follow_count}) end end if @fork_count.present? if $redis_cache.hget(user_statistic_key, fork_count_key).nil? reset_user_fork_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {fork_count: @fork_count}) else $redis_cache.hincrby(user_statistic_key, fork_count_key, @fork_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {fork_count: @fork_count}) end end if @issue_count.present? if $redis_cache.hget(user_statistic_key, issue_count_key).nil? reset_user_issue_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {issue_count: @issue_count}) else $redis_cache.hincrby(user_statistic_key, issue_count_key, @issue_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {issue_count: @issue_count}) end end if @project_count.present? if $redis_cache.hget(user_statistic_key, project_count_key).nil? reset_user_project_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_count: @project_count}) else $redis_cache.hincrby(user_statistic_key, project_count_key, @project_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_count: @project_count}) end end if @project_language_count_key.present? && project_language_count.present? if $redis_cache.hget(user_statistic_key, project_language_key).nil? reset_user_project_language + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_language_count_key: @project_language_count_key, project_language_count: @project_language_count}) else result = JSON.parse($redis_cache.hget(user_statistic_key, project_language_key)) result[@project_language_count_key] ||= 0 result[@project_language_count_key] += project_language_count.to_i + result.delete(@project_language_count_key) if result[@project_language_count_key] == 0 $redis_cache.hset(user_statistic_key, project_language_key, result.to_json) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_language_count_key: @project_language_count_key, project_language_count: @project_language_count}) end end if @project_praise_count.present? if $redis_cache.hget(user_statistic_key, project_praise_count_key).nil? reset_user_project_praise_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_praise_count: @project_praise_count}) else $redis_cache.hincrby(user_statistic_key, project_praise_count_key, @project_praise_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_praise_count: @project_praise_count}) end end if @project_watcher_count.present? if $redis_cache.hget(user_statistic_key, project_watcher_count_key).nil? reset_user_project_watcher_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_watcher_count: @project_watcher_count}) else $redis_cache.hincrby(user_statistic_key, project_watcher_count_key, @project_watcher_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {project_watcher_count: @project_watcher_count}) end end if @pullrequest_count.present? if $redis_cache.hget(user_statistic_key, pullrequest_count_key).nil? reset_user_pullrequest_count + Cache::V2::UserDateRankService.call(@user_id, Date.today, {pullrequest_count: @pullrequest_count}) else $redis_cache.hincrby(user_statistic_key, pullrequest_count_key, @pullrequest_count) + Cache::V2::UserDateRankService.call(@user_id, Date.today, {pullrequest_count: @pullrequest_count}) end end $redis_cache.hgetall(user_statistic_key) From 21fa609e05cfa2e3d7ada3b3cf8e7c58b09ccf44 Mon Sep 17 00:00:00 2001 From: yystopf Date: Tue, 26 Oct 2021 18:42:35 +0800 Subject: [PATCH 05/11] add: project rank and user rank api --- app/controllers/project_rank_controller.rb | 24 +++++++++++++++++++ app/controllers/user_rank_controller.rb | 24 +++++++++++++++++++ app/models/project.rb | 7 +++--- app/services/cache/v2/owner_common_service.rb | 14 +++++++++++ .../cache/v2/project_common_service.rb | 7 ++++-- app/services/cache/v2/project_rank_service.rb | 10 ++++++-- .../cache/v2/user_date_rank_service.rb | 7 +++--- app/views/project_rank/_detail.json.jbuilder | 19 +++++++++++++++ app/views/project_rank/index.json.jbuilder | 8 +++++++ app/views/user_rank/_detail.json.jbuilder | 15 ++++++++++++ app/views/user_rank/index.json.jbuilder | 8 +++++++ config/routes.rb | 3 +++ 12 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 app/controllers/project_rank_controller.rb create mode 100644 app/controllers/user_rank_controller.rb create mode 100644 app/views/project_rank/_detail.json.jbuilder create mode 100644 app/views/project_rank/index.json.jbuilder create mode 100644 app/views/user_rank/_detail.json.jbuilder create mode 100644 app/views/user_rank/index.json.jbuilder diff --git a/app/controllers/project_rank_controller.rb b/app/controllers/project_rank_controller.rb new file mode 100644 index 000000000..07bc2fb0f --- /dev/null +++ b/app/controllers/project_rank_controller.rb @@ -0,0 +1,24 @@ +class ProjectRankController < ApplicationController + # 根据时间获取热门项目 + def index + $redis_cache.zunionstore("recent-days-project-rank", get_timeable_key_names) + @project_rank = $redis_cache.zrevrange("recent-days-project-rank", 0, 5, withscores: true) + rescue Exception => e + @project_rack = [] + end + + private + # 默认显示7天的 + def time + params.fetch(:time, 7).to_i + end + + def get_timeable_key_names + names_array = [] + (0...time).to_a.each do |i| + date_time_string = (Date.today - i.days).to_s + names_array << "v2-project-rank-#{date_time_string}" + end + names_array + end +end \ No newline at end of file diff --git a/app/controllers/user_rank_controller.rb b/app/controllers/user_rank_controller.rb new file mode 100644 index 000000000..734b94c84 --- /dev/null +++ b/app/controllers/user_rank_controller.rb @@ -0,0 +1,24 @@ +class UserRankController < ApplicationController + # 根据时间获取热门开发者 + def index + $redis_cache.zunionstore("recent-days-user-rank", get_timeable_key_names) + @user_rank = $redis_cache.zrevrange("recent-days-user-rank", 0, 5, withscores: true) + rescue Exception => e + @user_rank = [] + end + + private + # 默认显示7天的 + def time + params.fetch(:time, 7).to_i + end + + def get_timeable_key_names + names_array = [] + (0...time).to_a.each do |i| + date_time_string = (Date.today - i.days).to_s + names_array << "v2-user-rank-#{date_time_string}" + end + names_array + end +end \ No newline at end of file diff --git a/app/models/project.rb b/app/models/project.rb index 78c049ab2..c4c085096 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -127,9 +127,10 @@ class Project < ApplicationRecord has_many :has_pinned_users, through: :pinned_projects, source: :user has_many :webhooks, class_name: "Gitea::Webhook", primary_key: :gpid, foreign_key: :repo_id after_create :init_project_common, :incre_user_statistic, :incre_platform_statistic - after_save :check_project_members - before_save :set_invite_code, :reset_cache_data, :reset_unmember_followed - after_destroy :decre_project_common, :decre_user_statistic, :decre_platform_statistic + after_save :check_project_members, :reset_cache_data + before_save :set_invite_code, :reset_unmember_followed + before_destroy :decre_project_common + after_destroy :decre_user_statistic, :decre_platform_statistic scope :project_statics_select, -> {select(:id,:name, :is_public, :identifier, :status, :project_type, :user_id, :forked_count, :visits, :project_category_id, :project_language_id, :license_id, :ignore_id, :watchers_count, :created_on)} scope :no_anomory_projects, -> {where("projects.user_id is not null and projects.user_id != ?", 2)} scope :recommend, -> { visible.project_statics_select.where(recommend: true) } diff --git a/app/services/cache/v2/owner_common_service.rb b/app/services/cache/v2/owner_common_service.rb index 07183f811..62322b49c 100644 --- a/app/services/cache/v2/owner_common_service.rb +++ b/app/services/cache/v2/owner_common_service.rb @@ -31,6 +31,10 @@ class Cache::V2::OwnerCommonService < ApplicationService "v2-owner-common:#{@login}-#{@email.to_s}" end + def owner_common_key_by_id + "v2-owner-common:#{@owner.id}" + end + def owner_common $redis_cache.hgetall(owner_common_key).blank? ? reset_owner_common : $redis_cache.hgetall(owner_common_key) end @@ -46,6 +50,9 @@ class Cache::V2::OwnerCommonService < ApplicationService else $redis_cache.hset(owner_common_key, "name", @name) $redis_cache.hset(owner_common_key, "avatar_url", url_to_avatar(owner)) + + $redis_cache.hset(owner_common_key_by_id, "name", @name) + $redis_cache.hset(owner_common_key_by_id, "avatar_url", url_to_avatar(owner)) end end if @email.present? @@ -53,6 +60,7 @@ class Cache::V2::OwnerCommonService < ApplicationService reset_owner_email else $redis_cache.hset(owner_common_key, "email", @email) + $redis_cache.hset(owner_common_key_by_id, "email", @email) end end @@ -60,23 +68,29 @@ class Cache::V2::OwnerCommonService < ApplicationService end def reset_owner_id $redis_cache.hset(owner_common_key, "id", owner&.id) + $redis_cache.hset(owner_common_key_by_id, "id", owner&.id) end def reset_owner_type $redis_cache.hset(owner_common_key, "type", owner&.type) + $redis_cache.hset(owner_common_key_by_id, "type", owner&.type) end def reset_owner_login $redis_cache.hset(owner_common_key, "login", owner&.login) + $redis_cache.hset(owner_common_key_by_id, "login", owner&.login) end def reset_owner_email $redis_cache.hset(owner_common_key, "email", owner&.mail) + $redis_cache.hset(owner_common_key_by_id, "email", owner&.mail) end def reset_owner_name $redis_cache.hset(owner_common_key, "name", owner&.real_name) $redis_cache.hset(owner_common_key, "avatar_url", url_to_avatar(owner)) + $redis_cache.hset(owner_common_key_by_id, "name", owner&.real_name) + $redis_cache.hset(owner_common_key_by_id, "avatar_url", url_to_avatar(owner)) end def reset_owner_common diff --git a/app/services/cache/v2/project_common_service.rb b/app/services/cache/v2/project_common_service.rb index 6ea4bb3a9..841a0bb55 100644 --- a/app/services/cache/v2/project_common_service.rb +++ b/app/services/cache/v2/project_common_service.rb @@ -120,7 +120,10 @@ class Cache::V2::ProjectCommonService < ApplicationService Cache::V2::ProjectRankService.call(@project_id, {visits: @visits}) Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {visits: @visits}) else - $redis_cache.hincrby(project_common_key, visits_key, @visits) + puts project_common_key + puts visits_key + puts @visits + $redis_cache.hincrby(project_common_key, visits_key, @visits.to_s) Cache::V2::ProjectRankService.call(@project_id, {visits: @visits}) Cache::V2::ProjectDateRankService.call(@project_id, Date.today, {visits: @visits}) end @@ -196,7 +199,7 @@ class Cache::V2::ProjectCommonService < ApplicationService end def reset_project_visits - $redis_cache.hset(project_common_key, visits_key, @project&.visits) + $redis_cache.hset(project_common_key, visits_key, @project&.visits || 0) end def reset_project_watchers diff --git a/app/services/cache/v2/project_rank_service.rb b/app/services/cache/v2/project_rank_service.rb index 089258d96..45484dc08 100644 --- a/app/services/cache/v2/project_rank_service.rb +++ b/app/services/cache/v2/project_rank_service.rb @@ -37,6 +37,7 @@ class Cache::V2::ProjectRankService < ApplicationService end def set_project_rank + load_project_common if $redis_cache.zscore(project_rank_key, @project_id).blank? reset_project_rank return @@ -56,16 +57,21 @@ class Cache::V2::ProjectRankService < ApplicationService if @pullrequests.present? $redis_cache.zincrby(project_rank_key, @pullrequests.to_i * 10, @project_id) end + reset_user_project_rank $redis_cache.zscore(project_rank_key, @project_id) end def reset_project_rank load_project_common - $redis_cache.zrem(project_rank_key, @project_id) score = @project_common["visits"].to_i * 1 + @project_common["praises"].to_i * 5 + @project_common["forks"].to_i * 5 + @project_common["issues"].to_i * 10 + @project_common["pullrequests"].to_i * 10 - $redis_cache.zincrby(project_rank_key, score, @project_id) + $redis_cache.zadd(project_rank_key, score, @project_id) + reset_user_project_rank $redis_cache.zscore(project_rank_key, @project_id) end + + def reset_user_project_rank + $redis_cache.zadd("v2-user-project-rank:#{@project_common["owner_id"]}", $redis_cache.zscore(project_rank_key, @project_id), @project_id) + end end \ No newline at end of file diff --git a/app/services/cache/v2/user_date_rank_service.rb b/app/services/cache/v2/user_date_rank_service.rb index 583f03a22..00073a8e8 100644 --- a/app/services/cache/v2/user_date_rank_service.rb +++ b/app/services/cache/v2/user_date_rank_service.rb @@ -93,7 +93,8 @@ class Cache::V2::UserDateRankService < ApplicationService fork_count = $redis_cache.hget(user_date_statistic_key, "fork-count") || 0 project_watchers_count = $redis_cache.hget(user_date_statistic_key, "project-watcher-count") || 0 project_praises_count = $redis_cache.hget(user_date_statistic_key, "project-praise-count") || 0 - project_languages_count = $redis_cache.hget(user_date_statistic_key, "project-language").nil? ? 0 : $redis_cache.hget(user_date_statistic_key, "project-language").length + project_language = $redis_cache.hget(user_date_statistic_key, "project-language") + project_languages_count = project_language.nil? || project_language == "{}" ? 0 : JSON.parse(project_language).length # 影响力 influence = (60.0 + follow_count.to_i / (follow_count.to_i + 20.0) * 40.0).to_i @@ -110,8 +111,8 @@ class Cache::V2::UserDateRankService < ApplicationService language = (60.0 + project_languages_count.to_i / (project_languages_count.to_i + 5.0) * 40.0).to_i score = influence+ contribution + activity + experience + language - - $redis_cache.zadd(user_rank_key, score, @user_id) if score.to_i > 300 + $redis_cache.zrem(user_rank_key, @user_id) + $redis_cache.zadd(user_rank_key, score-300, @user_id) if score > 300 $redis_cache.zscore(user_rank_key, @user_id) end diff --git a/app/views/project_rank/_detail.json.jbuilder b/app/views/project_rank/_detail.json.jbuilder new file mode 100644 index 000000000..454aee2cb --- /dev/null +++ b/app/views/project_rank/_detail.json.jbuilder @@ -0,0 +1,19 @@ +project_common = $redis_cache.hgetall("v2-project-common:#{item[0]}") +owner_common = $redis_cache.hgetall("v2-owner-common:#{project_common["owner_id"]}") +json.id item[0] +json.score item[1] +json.name project_common["name"] +json.identifier project_common["identifier"] +json.description project_common["description"] +json.owner do + json.id project_common["owner_id"] + json.name owner_common["name"] + json.login owner_common["login"] + json.avatar_url owner_common["avatar_url"] +end +json.visits project_common["visits"] +json.forks project_common["forks"] +json.watchers project_common["watchers"] +json.praises project_common["praises"] +json.issues project_common["issues"] +json.pulls project_common["pullrequests"] \ No newline at end of file diff --git a/app/views/project_rank/index.json.jbuilder b/app/views/project_rank/index.json.jbuilder new file mode 100644 index 000000000..84792ca7d --- /dev/null +++ b/app/views/project_rank/index.json.jbuilder @@ -0,0 +1,8 @@ +json.partial! "commons/success" +json.projects do + + json.array! @project_rank.each do |item| + json.partial! "detail", locals: {item: item} + end + +end \ No newline at end of file diff --git a/app/views/user_rank/_detail.json.jbuilder b/app/views/user_rank/_detail.json.jbuilder new file mode 100644 index 000000000..54afea5c4 --- /dev/null +++ b/app/views/user_rank/_detail.json.jbuilder @@ -0,0 +1,15 @@ +owner_common = $redis_cache.hgetall("v2-owner-common:#{item[0]}") +popular_project = $redis_cache.zrevrange("v2-user-project-rank:#{item[0]}", 0, 1, withscores: true)[0] +popular_project_common = $redis_cache.hgetall("v2-project-common:#{popular_project[0]}") +json.id item[0] +json.score item[1] +json.name owner_common["name"] +json.type owner_common["type"] +json.login owner_common["login"] +json.avatar_url owner_common["avatar_url"] +json.project do + json.id popular_project[0] + json.name popular_project_common["name"] + json.identifier popular_project_common["identifier"] + json.description popular_project_common["description"] +end \ No newline at end of file diff --git a/app/views/user_rank/index.json.jbuilder b/app/views/user_rank/index.json.jbuilder new file mode 100644 index 000000000..51ad153c4 --- /dev/null +++ b/app/views/user_rank/index.json.jbuilder @@ -0,0 +1,8 @@ +json.partial! "commons/success" +json.users do + + json.array! @user_rank.each do |item| + json.partial! "detail", locals: {item: item} + end + +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 35fea39c7..7f78cc8dd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -73,6 +73,9 @@ Rails.application.routes.draw do resources :public_keys, only: [:index, :create, :destroy] + resources :project_rank, only: [:index] + resources :user_rank, only: [:index] + resources :statistic, only: [:index] do collection do get :platform_profile From e1b0abbf310da699ac610ed5000f90aec2e56cae Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 09:20:46 +0800 Subject: [PATCH 06/11] fix --- app/controllers/projects_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index b1ab9d010..4f48e53dc 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -191,7 +191,7 @@ class ProjectsController < ApplicationController def simple # 为了缓存活跃项目的基本信息，后续删除 - Cache::V2::ProjectCommonService.new(@project.id).read + Cache::V2::ProjectCommonService.new(@project.id).reset json_response(@project, current_user) end From d0791ee6539781e79c054d87e6d36d974441ea7a Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 09:48:23 +0800 Subject: [PATCH 07/11] fix: privacy project not use cache --- app/models/project.rb | 8 ++++++++ app/services/cache/v2/project_common_service.rb | 3 +++ app/views/project_rank/_detail.json.jbuilder | 1 + 3 files changed, 12 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index c4c085096..03cad7ca7 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -190,6 +190,14 @@ class Project < ApplicationRecord CacheAsyncSetJob.perform_later("platform_statistic_service", {project_count: -1, project_language_count_key: self.project_language&.name, project_language_count: -1}) end + def is_full_public + owner = self.owner + if owner.is_a?(Organization) + return self.is_public && owner&.visibility == "common" + else + return self.is_public + end + end def reset_unmember_followed if changes[:is_public].present? && changes[:is_public] == [true, false] diff --git a/app/services/cache/v2/project_common_service.rb b/app/services/cache/v2/project_common_service.rb index 841a0bb55..77a7d27b7 100644 --- a/app/services/cache/v2/project_common_service.rb +++ b/app/services/cache/v2/project_common_service.rb @@ -86,6 +86,8 @@ class Cache::V2::ProjectCommonService < ApplicationService reset_project_common return end + load_project + return unless @project.is_full_public if @owner_id.present? if $redis_cache.hget(project_common_key, owner_id_key).nil? reset_project_owner_id @@ -224,6 +226,7 @@ class Cache::V2::ProjectCommonService < ApplicationService def reset_project_common load_project + return unless @project.is_full_public $redis_cache.del(project_common_key) reset_project_owner_id reset_project_name diff --git a/app/views/project_rank/_detail.json.jbuilder b/app/views/project_rank/_detail.json.jbuilder index 454aee2cb..ce703e77a 100644 --- a/app/views/project_rank/_detail.json.jbuilder +++ b/app/views/project_rank/_detail.json.jbuilder @@ -7,6 +7,7 @@ json.identifier project_common["identifier"] json.description project_common["description"] json.owner do json.id project_common["owner_id"] + json type owner_common["type"] json.name owner_common["name"] json.login owner_common["login"] json.avatar_url owner_common["avatar_url"] From e6c6e544fe0c53c46af1880493ef14287dc3ec64 Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 10:00:06 +0800 Subject: [PATCH 08/11] fix --- app/views/project_rank/_detail.json.jbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/project_rank/_detail.json.jbuilder b/app/views/project_rank/_detail.json.jbuilder index ce703e77a..b23403966 100644 --- a/app/views/project_rank/_detail.json.jbuilder +++ b/app/views/project_rank/_detail.json.jbuilder @@ -7,7 +7,7 @@ json.identifier project_common["identifier"] json.description project_common["description"] json.owner do json.id project_common["owner_id"] - json type owner_common["type"] + json.type owner_common["type"] json.name owner_common["name"] json.login owner_common["login"] json.avatar_url owner_common["avatar_url"] From 9232a284d859695ca1ae8b29d5ee44887ae2ff0e Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 10:46:09 +0800 Subject: [PATCH 09/11] fix --- app/controllers/organizations/organizations_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/organizations/organizations_controller.rb b/app/controllers/organizations/organizations_controller.rb index 218bc872d..f019e8d39 100644 --- a/app/controllers/organizations/organizations_controller.rb +++ b/app/controllers/organizations/organizations_controller.rb @@ -22,6 +22,7 @@ class Organizations::OrganizationsController < Organizations::BaseController @can_create_project = @organization.can_create_project?(current_user.id) @is_admin = can_edit_org? @is_member = @organization.is_member?(current_user.id) + Cache::V2::OwnerCommonService.new(@organization.login, @organization.mail).read end def create From c019766df20b0ac3072fbe73ba13648fa4db5b42 Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 10:48:45 +0800 Subject: [PATCH 10/11] fix --- app/services/cache/v2/owner_common_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/cache/v2/owner_common_service.rb b/app/services/cache/v2/owner_common_service.rb index 62322b49c..d8b86d097 100644 --- a/app/services/cache/v2/owner_common_service.rb +++ b/app/services/cache/v2/owner_common_service.rb @@ -24,7 +24,7 @@ class Cache::V2::OwnerCommonService < ApplicationService private def load_owner - @owner = User.find_by(login: @login) + @owner = Owner.find_by(login: @login) end def owner_common_key From 01f99a2f120d5834c36f9f97e74c81070b13c53d Mon Sep 17 00:00:00 2001 From: yystopf Date: Wed, 27 Oct 2021 11:03:34 +0800 Subject: [PATCH 11/11] fix --- app/models/organization.rb | 6 ++++++ app/models/user.rb | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/models/organization.rb b/app/models/organization.rb index 843c2b05f..4fd0886da 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -81,6 +81,12 @@ class Organization < Owner scope :with_visibility, ->(visibility) { joins(:organization_extension).where(organization_extensions: {visibility: visibility}) if visibility.present? } + after_save :reset_cache_data + + def reset_cache_data + Cache::V2::OwnerCommonService.new(self.login, self.mail).reset + end + def self.build(name, nickname, gitea_token=nil) self.create!(login: name, nickname: nickname, gitea_token: gitea_token) end diff --git a/app/models/user.rb b/app/models/user.rb index 7a172020c..ae20b83d3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -188,7 +188,8 @@ class User < Owner :show_email, :show_location, :show_department, :technical_title, :province, :city, :custom_department, to: :user_extension, allow_nil: true - before_save :update_hashed_password, :set_lastname, :reset_cache_data + before_save :update_hashed_password, :set_lastname + after_save :reset_cache_data after_create do SyncTrustieJob.perform_later("user", 1) if allow_sync_to_trustie? end