diff --git a/app/controllers/admins/dashboards_controller.rb b/app/controllers/admins/dashboards_controller.rb index 2c01c8bd..1e13621e 100644 --- a/app/controllers/admins/dashboards_controller.rb +++ b/app/controllers/admins/dashboards_controller.rb @@ -28,6 +28,41 @@ class Admins::DashboardsController < Admins::BaseController @day_new_project_count = Project.where(created_on: today).count @weekly_new_project_count = Project.where(created_on: current_week).count @month_new_project_count = Project.where(created_on: current_month).count + + # 总的平台用户数 + # 总的平台项目数 + # 总的平台组织数 + # 总的平台Issue数、评论数、PR数、Commit数 + @user_count = User.count + @project_count = Project.count + @organization_count = Organization.count + @issue_count = Issue.count + @comment_count = Journal.count + @pr_count = PullRequest.count + @commit_count = CommitLog.count + + @subject_name = ["用户数", "项目数", "组织数", "Issue数", "Issue评论数", "PR数", "Commit数"] + @subject_icon = ["fa-user","fa-git", "fa-sitemap", "fa-warning", "fa-comments", "fa-share-alt", "fa-upload"] + @subject_data = [@user_count, @project_count, @organization_count, @issue_count, @comment_count, @pr_count, @commit_count] + + + tongji_service = Baidu::TongjiService.new + @access_token = tongji_service.access_token + Rails.logger.info "baidu_tongji_auth access_token ===== #{@access_token}" + # @overview_data = tongji_service.api_overview + last_date = DailyPlatformStatistic.order(:date).last + start_date = last_date.date + end_date = Time.now + if @access_token.present? + @overview_data = tongji_service.overview_batch_add(start_date, end_date) + tongji_service.source_from_batch_add(start_date, end_date) + end + + @current_week_statistic = DailyPlatformStatistic.where(date: current_week) + @pre_week_statistic = DailyPlatformStatistic.where(date: pre_week) + + + end def month_active_user @@ -42,6 +77,19 @@ class Admins::DashboardsController < Admins::BaseController render_ok(data: data) end + def baidu_tongji + tongji_service = Baidu::TongjiService.new + redirect_to tongji_service.code_url + end + + def baidu_tongji_auth + if params[:code].present? + tongji_service = Baidu::TongjiService.new + tongji_service.get_access_token(params[:code]) + end + redirect_to "/admins/" + end + def evaluate names = [] data = [] @@ -63,8 +111,12 @@ class Admins::DashboardsController < Admins::BaseController Time.now.beginning_of_day..Time.now.end_of_day end - def current_week + def pre_7_days 7.days.ago.end_of_day..Time.now.end_of_day + end + + def current_week + Time.now.beginning_of_week..Time.now.end_of_day end def current_month @@ -72,6 +124,7 @@ class Admins::DashboardsController < Admins::BaseController end def pre_week - 14.days.ago.end_of_day..7.days.ago.end_of_day + # 14.days.ago.end_of_day..7.days.ago.end_of_day + Time.now.prev_week..Time.now.prev_week.end_of_week end end \ No newline at end of file diff --git a/app/jobs/daily_platform_statistics_job.rb b/app/jobs/daily_platform_statistics_job.rb new file mode 100644 index 00000000..5610f304 --- /dev/null +++ b/app/jobs/daily_platform_statistics_job.rb @@ -0,0 +1,44 @@ +# 按天获取百度统计数据，pv，访问，ip和来源分类占比 +# 其他统计：前一周用户留存率 +class DailyPlatformStatisticsJob < ApplicationJob + queue_as :default + + def perform(*args) + Rails.logger.info("*********开始统计*********") + + tongji_service = Baidu::TongjiService.new + access_token = tongji_service.access_token + Rails.logger.info "job baidu_tongji_auth access_token ===== #{access_token}" + ActiveJob::Base.logger.info "job baidu_tongji_auth access_token ===== #{access_token}" + # 从最后一个记录日期开始，如果遗漏日期数据可以补充数据 + last_date = DailyPlatformStatistic.order(:date).last + start_date = last_date.date + end_date = Time.now + if access_token.present? + tongji_service.overview_batch_add(start_date, end_date) + + # 本周访问来源占比，每天记录一次，如果遗漏日期数据可以补充数据 + tongji_service.source_from_batch_add(start_date, end_date) + end + # 周用户留存率 + pre_week_user_ids = User.where(created_on: pre_week).pluck(:id).uniq + weekly_keep_user_count = User.where(id: pre_week_user_ids).where(last_login_on: current_week).count + weekly_keep_rate = format("%.2f", pre_week_user_ids.size > 0 ? weekly_keep_user_count.to_f / pre_week_user_ids.size : 0) + + job_date = 1.days.ago + daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: job_date) + daily_statistic.weekly_keep_rate = weekly_keep_rate + daily_statistic.save + end + + private + + def current_week + Time.now.beginning_of_week..Time.now.end_of_day + end + + def pre_week + # 7.days.ago.beginning_of_week..7.days.ago.beginning_of_week.end_of_week + Time.now.prev_week..Time.now.prev_week.end_of_week + end +end diff --git a/app/models/daily_platform_statistic.rb b/app/models/daily_platform_statistic.rb new file mode 100644 index 00000000..a904d9b1 --- /dev/null +++ b/app/models/daily_platform_statistic.rb @@ -0,0 +1,24 @@ +# == Schema Information +# +# Table name: daily_platform_statistics +# +# id :integer not null, primary key +# date :date +# pv :integer default("0") +# visitor :integer default("0") +# ip :integer default("0") +# weekly_keep_rate :float(24) default("0") +# source_through :float(24) default("0") +# source_link :float(24) default("0") +# source_search :float(24) default("0") +# source_custom :float(24) default("0") +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_daily_platform_statistics_on_date (date) UNIQUE +# + +class DailyPlatformStatistic < ApplicationRecord +end diff --git a/app/services/baidu/tongji_service.rb b/app/services/baidu/tongji_service.rb new file mode 100644 index 00000000..8a611cd0 --- /dev/null +++ b/app/services/baidu/tongji_service.rb @@ -0,0 +1,221 @@ +module Baidu + class TongjiService < ApplicationService + attr_reader :client_id, :client_secret, :site_id + # login、code、password、password_confirmation + def initialize + @client_id = "6dMO2kqKUaMZkBrMaUMxQSNAT49v0Mjq" + @client_secret = "qvWqF33AOmGs1tPCgsROvis9EQCuNmd3" + @site_id = 18657013 + end + + def call + + end + + + def init_overview_data_by(start_date = nil, end_date = nil) + start_date = Time.now.prev_year.beginning_of_year if start_date.nil? + end_date = Time.now + Rails.logger.info("*********开始百度统计-概览:#{start_date}-#{end_date}*********") + sql_connection = ActiveRecord::Base.connection + sql_connection.begin_db_transaction + + # 如果存在数据 先清空 + # sql_connection.execute("delete from daily_platform_statistics where date between '#{start_date}' and '#{end_date}'") + multiple_days_data = overview_multiple_days_data(start_date, end_date) + if multiple_days_data.present? + sql = "replace into daily_platform_statistics (date,pv,visitor,ip,created_at,updated_at) values #{multiple_days_data.join(",")}" + sql_connection.execute(sql) + end + sql_connection.commit_db_transaction + Rails.logger.info("*********结束百度统计-概览:#{start_date}-#{end_date}*********") + end + + def init_source_from_data_by(start_date = nil, end_date = nil) + start_date = Time.now.prev_year.beginning_of_year if start_date.nil? + end_date = Time.now + Rails.logger.info("*********开始百度统计-来源:#{start_date}-#{end_date}*********") + source_from_batch_add(start_date, end_date) + Rails.logger.info("*********结束百度统计-来源:#{start_date}-#{end_date}*********") + end + + # 按日期获取来源数据 + def source_from_batch_add(start_date,end_date) + # 补充更新开始时间的当天数据 + source_from_by_date(start_date) + diff_days(start_date, end_date).times.each do |t| + new_start_date = start_date + (t + 1).days + source_from_by_date(new_start_date) + end + # 补充更新最后时间一天数据 + source_from_by_date(end_date) + end + + # 按天获取来源数据 + def source_from_by_date(start_date) + return [] unless access_token.present? && start_date.present? + source_from_data = api("source/all/a", start_date, start_date, "pv_count,visitor_count,ip_count") + source_from = [] + source_from_data['items'][1].each_with_index do |source, index| + source_from.push(((source[0].to_f / source_from_data['sum'][0][0].to_f) * 100).round(2)) + end + daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: start_date) + daily_statistic.source_through = source_from[0] + daily_statistic.source_link = source_from[1] + daily_statistic.source_search = source_from[2] + daily_statistic.source_custom = source_from[3] + daily_statistic.save + end + + def diff_days(start_date, end_date) + (end_date.beginning_of_day.to_i - start_date.beginning_of_day.to_i) / (24 * 3600) + end + + def overview_batch_add(start_date, end_date) + return [] unless access_token.present? && start_date.present? && end_date.present? + start_date = Time.now - 1.days if start_date.strftime("%Y%m%d") == end_date.strftime("%Y%m%d") + overview_data = api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count") + overview_data['items'][0].each_with_index do |date, index| + pv = overview_data['items'][1][index][0] + visitor = overview_data['items'][1][index][1] + ip = overview_data['items'][1][index][2] + job_date = date[0].to_s.gsub("/", "-") + daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: job_date) + daily_statistic.date = job_date + daily_statistic.pv = pv + daily_statistic.visitor = visitor + daily_statistic.ip = ip + daily_statistic.save + end + overview_data + end + + def overview_multiple_days_data(start_date, end_date) + return [] unless access_token.present? && start_date.present? && end_date.present? + overview_data = api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count") + data = [] + created_at = Time.now.strftime("%Y-%m-%d 00:00:00") + overview_data['items'][0].each_with_index do |date, index| + pv = overview_data['items'][1][index][0] + visitor = overview_data['items'][1][index][1] + ip = overview_data['items'][1][index][2] + data.push("('#{date[0].to_s.gsub("/", "-")}', #{pv.to_s.gsub("--","0")}, #{visitor.to_s.gsub("--","0")}, #{ip.to_s.gsub("--","0")},\"#{created_at}\",\"#{created_at}\")") + end + data + end + + def code_url + "http://openapi.baidu.com/oauth/2.0/authorize?response_type=code&client_id=#{client_id}&redirect_uri=oob&scope=basic&display=popup" + end + + def oauth_url(code) + "http://openapi.baidu.com/oauth/2.0/token?grant_type=authorization_code&code=#{code}&client_id=#{client_id}&client_secret=#{client_secret}&redirect_uri=oob" + end + + def get_access_token(code) + uri = URI.parse(oauth_url(code)) + response = Net::HTTP.get_response(uri) + Rails.logger.info "baidu_tongji_auth response.body ===== #{response.body}" + if response.code.to_i == 200 + data = JSON.parse(response.body) + access_token = data['access_token'] + refresh_token = data['refresh_token'] + expires_in = data['expires_in'] + if access_token.present? + Rails.cache.write("baidu_tongji_auth/access_token", access_token, expires_in: expires_in) + Rails.cache.write("baidu_tongji_auth/refresh_token", refresh_token, expires_in: 1.year) + end + end + end + + def refresh_access_token + url = "http://openapi.baidu.com/oauth/2.0/token?grant_type=refresh_token&refresh_token=#{refresh_token}&client_id=#{client_id}&client_secret=#{client_secret}" + uri = URI.parse(url) + response = Net::HTTP.get_response(uri) + Rails.logger.info "baidu_tongji_auth response.body ===== #{response.body}" + if response.code.to_i == 200 + data = JSON.parse(response.body) + access_token = data['access_token'] + refresh_token = data['refresh_token'] + expires_in = data['expires_in'] + if access_token.present? + Rails.cache.write("baidu_tongji_auth/access_token", access_token, expires_in: expires_in) + Rails.cache.write("baidu_tongji_auth/refresh_token", refresh_token, expires_in: 1.year) + end + end + end + + def access_token + access_token = Rails.cache.read("baidu_tongji_auth/access_token") + if access_token.blank? + refresh_access_token + access_token = Rails.cache.read("baidu_tongji_auth/access_token") + end + access_token + end + + def refresh_token + refresh_token = Rails.cache.read("baidu_tongji_auth/refresh_token") + # 如果刷新token失效，access_token也重置 + if refresh_token.blank? + Rails.cache.delete("baidu_tongji_auth/access_token") + end + refresh_token + end + + # 网站概况(趋势数据) + def api_overview + start_date = Time.now.beginning_of_week + end_date = Time.now + start_date = Time.now - 1.days if start_date.strftime("%Y%m%d") == end_date.strftime("%Y%m%d") + api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count") + end + + # 网站概况(来源网站、搜索词、入口页面、受访页面) + def api_overview_getCommonTrackRpt + start_date = Time.now.beginning_of_week + end_date = Time.now + api("overview/getCommonTrackRpt", start_date, end_date, "pv_count") + end + + # 全部来源 + def source_from + start_date = Time.now.beginning_of_week + end_date = Time.now + api("source/all/a", start_date, end_date, "pv_count,visitor_count,ip_count") + end + + def api(api_method, start_date, end_date, metrics = nil) + start_date_fmt = start_date.strftime("%Y%m%d") + end_date_fmt = end_date.strftime("%Y%m%d") + api_url = "https://openapi.baidu.com/rest/2.0/tongji/report/getData?access_token=#{access_token}&site_id=#{site_id}&method=#{api_method}&start_date=#{start_date_fmt}&end_date=#{end_date_fmt}&metrics=#{metrics}" + data = url_http_post(api_url, {}) + data['result'] + end + + def url_http_post(api_url, params) + Rails.logger.info "api_url==#{api_url}" + uri = URI.parse(api_url) + http = Net::HTTP.new uri.host, uri.port + http.open_timeout = 60 + http.read_timeout = 60 + if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.use_ssl = true + end + begin + request = Net::HTTP::Post.new(uri) + request.set_form_data(params) if params.present? + request['Content-Type'] = 'application/json;charset=utf-8' + # request['Content-Type'] = 'application/x-www-form-urlencoded;charset=utf-8' + response = http.start { |http| http.request(request) } + Rails.logger.info "api response.body==#{response.body}" + JSON.parse response.body + rescue => err + Rails.logger.error("#############api_url:#{api_url},error:#{err.message.size}") + # Rails.logger.error("#############api_url:#{api_url},error:#{err.message}") + return {} + end + end + end +end diff --git a/app/views/admins/dashboards/_baidu_tongji.html.erb b/app/views/admins/dashboards/_baidu_tongji.html.erb new file mode 100644 index 00000000..c59e1aca --- /dev/null +++ b/app/views/admins/dashboards/_baidu_tongji.html.erb @@ -0,0 +1,50 @@ +
+ 数据来源百度统计，本周 [<%= @current_week_statistic.first&.date %> / <%= @current_week_statistic.last&.date %>] +
+ + + + + + + + + + + + + + + <% @current_week_statistic.each_with_index do |week, index| %> + + + + + + + + + + + + <% end %> + + + + + + + + +
日期访问量访客数IP数直接访问占比外部链接占比搜索引擎占比自定义
<%= week.date %> <%= week.pv %><%= week.visitor %><%= week.ip %><%= week.source_through %>%<%= week.source_link %>%<%= week.source_search %>%<%= week.source_custom.to_f %>%
+ + +<% unless @access_token.present? && @overview_data.present? %> +
+ 1.百度统计需人工授权,点击去授权 +
+
+ 2.获取百度统计授权码 + +
+<% end %> \ No newline at end of file diff --git a/app/views/admins/dashboards/_baidu_tongji_api.html.erb b/app/views/admins/dashboards/_baidu_tongji_api.html.erb new file mode 100644 index 00000000..3a18dc40 --- /dev/null +++ b/app/views/admins/dashboards/_baidu_tongji_api.html.erb @@ -0,0 +1,67 @@ +<% if @access_token.present? && @overview_data.present? %> +
+ 数据来源百度统计，本周 <%= @overview_data['timeSpan'] %> +
+ + + + + + + + + + + <% pv_count = [] %> + <% visitor_count = [] %> + <% ip_count = [] %> + <% @overview_data['items'][0].each_with_index do |date, index| %> + <% pv = @overview_data['items'][1][index][0] %> + <% visitor = @overview_data['items'][1][index][1] %> + <% ip = @overview_data['items'][1][index][2] %> + + + + + + + + <% pv_count.push(pv) %> + <% visitor_count.push(visitor) %> + <% ip_count.push(ip) %> + <% end %> + + + + + + + + +
日期访问量访客数IP数
<%= date[0] %> <%= pv %><%= visitor %><%= ip %>
合计<%= pv_count %><%= visitor_count %><%= ip_count %>
+ + + + + + + + + + + + <% @source_from_data['items'][1].each_with_index do |source, index| %> + + <% end %> + + +
直接访问占比外部链接占比搜索引擎占比自定义
<%= ((source[0].to_f / @source_from_data['sum'][0][0].to_f) * 100).round(2).to_s %>%
+<% else %> +
+ 1.百度统计需人工授权,点击去授权 +
+
+ 2.获取百度统计授权码 + +
+<% end %> \ No newline at end of file diff --git a/app/views/admins/dashboards/index.html.erb b/app/views/admins/dashboards/index.html.erb index 2d86b4b6..35593f16 100644 --- a/app/views/admins/dashboards/index.html.erb +++ b/app/views/admins/dashboards/index.html.erb @@ -1,6 +1,35 @@ <% define_admin_breadcrumbs do %> <% add_admin_breadcrumb('概览', admins_path) %> <% end %> +
+
+
+ +
+ <%@subject_name.each_with_index do |subject, index| %> +
+
+
+
+
+
<%=subject %>
+ <%= @subject_data[index] %> +
+
+
+ +
+
+
+
+
+
+ <% end %> + +
+
+
+
@@ -53,6 +82,7 @@
+ <%= render partial: 'admins/dashboards/baidu_tongji' %>
\ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 7694a77c..6138028d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -795,6 +795,8 @@ Rails.application.routes.draw do namespace :admins do mount Sidekiq::Web => '/sidekiq' get '/', to: 'dashboards#index' + get '/baidu_tongji', to: 'dashboards#baidu_tongji' + get '/baidu_tongji_auth', to: 'dashboards#baidu_tongji_auth' namespace :topic do resources :activity_forums resources :banners diff --git a/config/sidekiq_cron.yml b/config/sidekiq_cron.yml index 0ec8f997..21b8f05f 100644 --- a/config/sidekiq_cron.yml +++ b/config/sidekiq_cron.yml @@ -11,4 +11,9 @@ delay_expired_issue: create_daily_project_statistics: cron: "0 1 * * *" class: "DailyProjectStatisticsJob" - queue: cache \ No newline at end of file + queue: cache + +daily_platform_statistics: + cron: "0 1 * * *" + class: "DailyPlatformStatisticsJob" + queue: default \ No newline at end of file diff --git a/db/migrate/202401321314370_create_daily_platform_statistics.rb b/db/migrate/202401321314370_create_daily_platform_statistics.rb new file mode 100644 index 00000000..d658979c --- /dev/null +++ b/db/migrate/202401321314370_create_daily_platform_statistics.rb @@ -0,0 +1,18 @@ +class CreateDailyPlatformStatistics < ActiveRecord::Migration[5.2] + def change + create_table :daily_platform_statistics do |t| + t.date :date + t.index :date, unique: true + t.bigint :pv, default: 0 + t.bigint :visitor, default: 0 + t.bigint :ip, default: 0 + t.float :weekly_keep_rate, default: 0 + t.float :source_through, default: 0 + t.float :source_link, default: 0 + t.float :source_search, default: 0 + t.float :source_custom, default: 0 + + t.timestamps + end + end +end diff --git a/spec/models/daily_platform_statistic_spec.rb b/spec/models/daily_platform_statistic_spec.rb new file mode 100644 index 00000000..0f6d5fcd --- /dev/null +++ b/spec/models/daily_platform_statistic_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe DailyPlatformStatistic, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end