diff --git a/.gitignore b/.gitignore index fbed31a8e..2eac1d277 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,5 @@ redis_data/ Dockerfile dump.rdb .tags* -ceshi_user.xlsx \ No newline at end of file +ceshi_user.xlsx +public/trace_task_results \ No newline at end of file diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 8bc2fb476..68c4e59fb 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -22,6 +22,7 @@ class ProjectsController < ApplicationController menu.append(menu_hash_by_name("devops")) if @project.has_menu_permission("devops") && @project.forge? menu.append(menu_hash_by_name("versions")) if @project.has_menu_permission("versions") menu.append(menu_hash_by_name("wiki")) if @project.has_menu_permission("wiki") && @project.forge? + menu.append(menu_hash_by_name("services")) if @project.has_menu_permission("services") && @project.forge? && (current_user.admin? || @project.member?(current_user.id)) menu.append(menu_hash_by_name("resources")) if @project.has_menu_permission("resources") && @project.forge? menu.append(menu_hash_by_name("activity")) menu.append(menu_hash_by_name("settings")) if user_is_admin && @project.forge? diff --git a/app/controllers/traces/base_controller.rb b/app/controllers/traces/base_controller.rb new file mode 100644 index 000000000..2b857d232 --- /dev/null +++ b/app/controllers/traces/base_controller.rb @@ -0,0 +1,18 @@ +class Traces::BaseController < ApplicationController + + helper_method :observed_logged_user?, :observed_user + + + def observed_user + @_observed_user ||= (User.find_by_login(params[:user_id]) || User.find_by_id(params[:user_id])) + end + + def observed_logged_user? + observed_user.id == User.current&.id + end + + protected + def check_auth + return render_forbidden unless current_user.admin? || observed_logged_user? + end +end \ No newline at end of file diff --git a/app/controllers/traces/projects_controller.rb b/app/controllers/traces/projects_controller.rb new file mode 100644 index 000000000..62573a1d8 --- /dev/null +++ b/app/controllers/traces/projects_controller.rb @@ -0,0 +1,83 @@ +class Traces::ProjectsController < Traces::BaseController + include OperateProjectAbilityAble + + before_action :require_login + before_action :load_project + before_action :authorizate_user_can_edit_project!, except: [:task_results] + + def tasks + branch_name = params[:branch_name] + return render_error("无可用检测次数") if @project.user_trace_tasks.size >= 5 + return render_error("分支名不能为空！") if branch_name.blank? + @all_branches = Gitea::Repository::Branches::ListNameService.call(@project&.owner, @project.identifier) + return render_error("请输入正确的分支名！") unless @all_branches["branch_name"].include?(branch_name) + code, data, error = Trace::CheckService.call(current_user.trace_token, @project, "1", branch_name) + if code == 200 + UserTraceTask.create!( + user_id: current_user.id, + project_id: @project.id, + branch_tag: branch_name, + task_id: data["task_id"] + ) + render_ok + else + render_error("检测失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + def task_results + limit = params[:limit] || params[:per_page] + limit = (limit.to_i.zero? || limit.to_i > 15) ? 15 : limit.to_i + page = params[:page].to_i.zero? ? 1 : params[:page].to_i + return render :json => {left_tasks_count: 5, data: []} if current_user.trace_user.nil? + code, data, error = Trace::CheckResultService.call(current_user.trace_token, @project, nil, page, limit) + if code == 200 + render :json => {left_tasks_count: 5 - @project.user_trace_tasks.size, data: data} + else + render_error("获取检测记录失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + def reload_task + return render_error("project_id错误") if params[:project_id].blank? + branch_name = params[:branch_name] + return render_error("分支名不能为空！") if branch_name.blank? + @all_branches = Gitea::Repository::Branches::ListNameService.call(@project&.owner, @project.identifier) + return render_error("请输入正确的分支名！") unless @all_branches["branch_name"].include?(branch_name) + code, data, error = Trace::ReloadCheckService.call(current_user.trace_token, params[:project_id]) + if code == 200 + UserTraceTask.create!( + user_id: current_user.id, + project_id: @project.id, + branch_tag: branch_name, + task_id: data["task_id"] + ) + render_ok + else + render_error("重新检测失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + + def task_pdf + return render_error("task_id错误") if params[:task_id].blank? + result = Trace::PdfReportService.call(current_user.trace_token, params[:task_id]) + if result.is_a?(Hash) && result[:code] == 200 + redirect_to result[:download_url] + else + render_error("下载报告失败！") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end +end \ No newline at end of file diff --git a/app/controllers/traces/trace_users_controller.rb b/app/controllers/traces/trace_users_controller.rb new file mode 100644 index 000000000..0b28e905a --- /dev/null +++ b/app/controllers/traces/trace_users_controller.rb @@ -0,0 +1,14 @@ +class Traces::TraceUsersController < Traces::BaseController + before_action :require_login + + def create + if current_user.trace_token.present? + render_ok + else + render_error(-1, "代码溯源用户初始化失败") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end +end \ No newline at end of file diff --git a/app/docs/slate/source/api.html.md b/app/docs/slate/source/api.html.md index a846317c4..0031a99f4 100644 --- a/app/docs/slate/source/api.html.md +++ b/app/docs/slate/source/api.html.md @@ -16,6 +16,7 @@ includes: - users - projects - repositories + - traces - pulls - issues - organizations diff --git a/app/docs/slate/source/includes/_projects.md b/app/docs/slate/source/includes/_projects.md index d4899a710..08ad6c234 100644 --- a/app/docs/slate/source/includes/_projects.md +++ b/app/docs/slate/source/includes/_projects.md @@ -280,7 +280,7 @@ repo |是| |string |项目标识identifier ### 返回字段说明 参数 | 类型 | 字段说明 --------- | ----------- | ----------- -menu_name |string|导航名称， home:主页,code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,activity:动态,setting:仓库设置 +menu_name |string|导航名称， home:主页,code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,wiki:维基,services:服务,activity:动态,setting:仓库设置 > 返回的JSON示例: @@ -408,7 +408,7 @@ await octokit.request('POST /api/yystopf/ceshi/project_units') ### 请求参数 参数 | 必选 | 默认 | 类型 | 字段说明 --------- | ------- | ------- | -------- | ---------- -|unit_types |是| |array | 项目模块内容， 支持以下参数:code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑 | +|unit_types |是| |array | 项目模块内容， 支持以下参数:code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,wiki:维基,resources:资源库,services:服务 | ### 返回字段说明: diff --git a/app/docs/slate/source/includes/_traces.md b/app/docs/slate/source/includes/_traces.md new file mode 100644 index 000000000..e6aa24b48 --- /dev/null +++ b/app/docs/slate/source/includes/_traces.md @@ -0,0 +1,217 @@ +# Traces + +## 代码溯源初始化 +用户同意协议后请求的接口，创建代码溯源的账号 + +> 示例: + +```shell +curl -X POST \ +http://localhost:3000/api/traces/trace_users.json +``` + +```javascript +await octokit.request('POST /api/traces/trace_users.json') +``` + +### HTTP 请求 +`POST api/traces/trace_users.json` + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + +## 代码分析结果列表 +查询项目下代码分析的结果 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/task_results.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/task_results.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/task_results.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner|是|否|string | 项目所有者标识| +repo|是 | 否|string | 项目标识 | +page |否| 1 | int | 页码 | +limit |否| 15 | int | 每页数量 | + +### 返回字段说明（暂缺） + + +> 返回的JSON示例: + +```json +{ + "data": [ + { + "accuracy": "20", + "code_type": "C", + "depth": "2", + "detect_flag": "快速-组件级", + "detect_rule": "快速-组件级,2,20,,开源软件,50,10", + "detect_startdate": "2022-05-10 15:59:46 ", + "detect_status": "fail", + "detectflag": "快速-组件级", + "fail_reason": "Invalid package type", + "file_name": "many_branch.zip", + "license_process": "100", + "licenseparam": "开源软件", + "package_type": "", + "product_name": "84727546110", + "project_id": "6dbc3e42-5857-4ca4-a54d-58fd9dbf6dc5", + "sim_process": "100", + "similarity_process": "2", + "task_id": "15139171-091b-4316-98b1-6068970efa44", + "totalsize": 5, + "uid": "78", + "vuln_process": "", + "vulnlevel": "" + } + ] +} +``` + + + + +## 新建分析 +用户选择仓库分支进行代码分析的接口 + +> 示例: + +```shell +curl -X POST \ +http://localhost:3000/api/traces/yystopf/many_branch/tasks.json +``` + +```javascript +await octokit.request('POST /api/traces/:owner/:repo/tasks.json') +``` + +### HTTP 请求 +`POST api/traces/:owner/:repo/tasks.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +branch_name|是 | 否| string | 分支名称 | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + +## 重新扫描 +对代码分析结果进行再次分析 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/reload_task.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/reload_task.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/reload_task.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +project_id|是 | 否| string | 代码分析结果里的project_id | +branch_name|是 | 否| string | 分支名称 | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + + +## 下载报告 +把代码分析的结果下载到本地 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/task_pdf.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/task_pdf.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/task_pdf.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +task_id|是 | 否| string | 代码分析结果里的task_id | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + diff --git a/app/models/message_template/project_setting_changed.rb b/app/models/message_template/project_setting_changed.rb index 0920dfe7a..ab597122a 100644 --- a/app/models/message_template/project_setting_changed.rb +++ b/app/models/message_template/project_setting_changed.rb @@ -141,6 +141,7 @@ class MessageTemplate::ProjectSettingChanged < MessageTemplate navbar.gsub!('devops', '工作流') navbar.gsub!('versions', '里程碑') navbar.gsub!('resources', '资源库') + navbar.gsub!('services', '服务') if change_count > 1 content.sub!('{ifnavbar}', '
') else @@ -290,6 +291,7 @@ class MessageTemplate::ProjectSettingChanged < MessageTemplate navbar.gsub!('devops', '工作流') navbar.gsub!('versions', '里程碑') navbar.gsub!('resources', '资源库') + navbar.gsub!('services', '服务') if change_count > 1 content.sub!('{ifnavbar}', '
') else diff --git a/app/models/project.rb b/app/models/project.rb index a35fbf387..1edc088f6 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -124,6 +124,7 @@ class Project < ApplicationRecord has_many :pinned_projects, dependent: :destroy has_many :has_pinned_users, through: :pinned_projects, source: :user has_many :webhooks, class_name: "Gitea::Webhook", primary_key: :gpid, foreign_key: :repo_id + has_many :user_trace_tasks, dependent: :destroy after_create :incre_user_statistic, :incre_platform_statistic after_save :check_project_members before_save :set_invite_code, :reset_unmember_followed, :set_recommend_and_is_pinned, :reset_cache_data diff --git a/app/models/project_unit.rb b/app/models/project_unit.rb index cc35a6b28..6ee0f2a8b 100644 --- a/app/models/project_unit.rb +++ b/app/models/project_unit.rb @@ -16,7 +16,7 @@ class ProjectUnit < ApplicationRecord belongs_to :project - enum unit_type: {code: 1, issues: 2, pulls: 3, wiki:4, devops: 5, versions: 6, resources: 7} + enum unit_type: {code: 1, issues: 2, pulls: 3, wiki:4, devops: 5, versions: 6, resources: 7, services: 8} validates :unit_type, uniqueness: { scope: :project_id} diff --git a/app/models/user.rb b/app/models/user.rb index 66aeb6164..bf764c148 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -175,6 +175,7 @@ class User < Owner has_many :system_notification_histories has_many :system_notifications, through: :system_notification_histories has_one :trace_user, dependent: :destroy + has_many :user_trace_tasks, dependent: :destroy # Groups and active users scope :active, lambda { where(status: [STATUS_ACTIVE, STATUS_EDIT_INFO]) } diff --git a/app/models/user_trace_task.rb b/app/models/user_trace_task.rb new file mode 100644 index 000000000..328cb7c0b --- /dev/null +++ b/app/models/user_trace_task.rb @@ -0,0 +1,25 @@ +# == Schema Information +# +# Table name: user_trace_tasks +# +# id :integer not null, primary key +# user_id :integer +# project_id :integer +# branch_tag :string(255) +# task_id :string(255) +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_user_trace_tasks_on_project_id (project_id) +# index_user_trace_tasks_on_user_id (user_id) +# + +class UserTraceTask < ApplicationRecord + + belongs_to :user + belongs_to :project + + +end diff --git a/app/services/trace/check_result_service.rb b/app/services/trace/check_result_service.rb index f1dd61ab0..ff47d9a39 100644 --- a/app/services/trace/check_result_service.rb +++ b/app/services/trace/check_result_service.rb @@ -1,11 +1,11 @@ # 代码溯源 查询检测结果 class Trace::CheckResultService < Trace::ClientService - attr_accessor :token, :project_name, :file_name, :page_num, :page_size + attr_accessor :token, :project, :file_name, :page_num, :page_size - def initialize(token, project_name=nil, file_name=nil, page_num=1, page_size=15) + def initialize(token, project, file_name=nil, page_num=1, page_size=15) @token = token - @project_name = project_name + @project = project @file_name = file_name @page_num = page_num @page_size = page_size @@ -19,7 +19,7 @@ class Trace::CheckResultService < Trace::ClientService private def request_params { - product_name: project_name, + product_name: Digest::MD5.hexdigest(project&.id.to_s)[0...20], file_name: file_name, pageNum: page_num, pageSize: page_size, diff --git a/app/services/trace/check_service.rb b/app/services/trace/check_service.rb index d31bbcf09..d1623445b 100644 --- a/app/services/trace/check_service.rb +++ b/app/services/trace/check_service.rb @@ -11,26 +11,25 @@ class Trace::CheckService < Trace::ClientService end def call - result = authed_post(token, url, {data: request_params}) + result = http_authed_post(token, url, {data: request_params}) reponse = render_response(result) end private def request_params - repo = Gitea::Repository::GetService.call(project&.owner&.login, project&.identifier) + repo = Gitea::Repository::GetService.call(project&.owner, project&.identifier) { - product_name: project&.name, - product_type: project&.category&.name, - code_type: project&.language&.name, + product_name: Digest::MD5.hexdigest(project&.id.to_s)[0...20], + product_type: project&.project_category&.name || '其他', + code_type: project&.project_language&.name || '其他', product_desc: project&.description, git_url: repo['clone_url'], if_branch: if_branch, branch_tag: branch_tag - } + }.compact end def url "/user/check".freeze end -end - +end \ No newline at end of file diff --git a/app/services/trace/client_service.rb b/app/services/trace/client_service.rb index 72ffa8ca2..0f1449225 100644 --- a/app/services/trace/client_service.rb +++ b/app/services/trace/client_service.rb @@ -12,6 +12,19 @@ class Trace::ClientService < ApplicationService conn.post(full_url(url), params[:data]) end + def http_authed_post(token, url, params={}) + puts "[trace][POST] request params: #{params}" + puts "[trace][POST] request token: #{token}" + url = URI("#{full_url(url)}") + http = Net::HTTP.new(url.host, url.port) + http.read_timeout = 1200 + request = Net::HTTP::Post.new(url) + request["Authorization"] = token + form_data = params[:data].stringify_keys.to_a + request.set_form form_data, 'multipart/form-data' + http.request(request) + end + def get(url, params={}) puts "[trace][GET] request params: #{params}" conn.get do |req| @@ -100,11 +113,22 @@ class Trace::ClientService < ApplicationService end def render_response(response) - status = response.status - body = JSON.parse(response&.body) + if response.is_a?(Faraday::Response) + status = response.status + body = JSON.parse(response&.body) - log_error(status, body) + log_error(status, body) - return [body["code"], body["data"], body["error"]] + return [body["code"], body["data"], body["error"]] + end + + if response.is_a?(Net::HTTPOK) + status = 200 + body = JSON.parse(response&.body) + + log_error(status, body) + + return [body["code"], body["data"], body["error"]] + end end end \ No newline at end of file diff --git a/app/services/trace/pdf_report_service.rb b/app/services/trace/pdf_report_service.rb index e91a78b30..aa7312739 100644 --- a/app/services/trace/pdf_report_service.rb +++ b/app/services/trace/pdf_report_service.rb @@ -1,4 +1,7 @@ # 代码溯源 导出pdf +require 'open-uri' +require 'fileutils' + class Trace::PdfReportService < Trace::ClientService attr_accessor :token, :task_id @@ -9,15 +12,23 @@ class Trace::PdfReportService < Trace::ClientService end def call - result = authed_get(token, url, request_params) - response = render_response(result) + content = open("#{domain}#{base_url}#{url}?task_id=#{task_id}", "Authorization" => token) + if content.is_a?(Tempfile) + check_file_path + IO.copy_stream(content, "#{save_path}/#{task_id}.pdf") + return {code: 200, download_url: "/trace_task_results/#{task_id}.pdf"} + else + return {code: 404} + end end private - def request_params - { - task_id: task_id - } + def check_file_path + FileUtils.mkdir_p save_path + end + + def save_path + "public/trace_task_results" end def url diff --git a/app/views/users/get_user_info.json.jbuilder b/app/views/users/get_user_info.json.jbuilder index 95085580e..d60f46ade 100644 --- a/app/views/users/get_user_info.json.jbuilder +++ b/app/views/users/get_user_info.json.jbuilder @@ -25,3 +25,4 @@ json.description @user.description json.super_description @user.super_description json.(@user, :show_email, :show_department, :show_location, :show_super_description) json.message_unread_total @message_unread_total +json.has_trace_user @user.trace_user.present? \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 64dce0e60..097aaf2a7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -427,6 +427,20 @@ Rails.application.routes.draw do end end + namespace :traces do + resources :trace_users, only: [:create] + scope "/:owner/:repo" do + resource :projects, path: '/', only: [:index] do + member do + post :tasks + get :task_results + get :reload_task + get :task_pdf + end + end + end + end + # Project Area START scope "/:owner/:repo" do scope do diff --git a/db/migrate/20220513061129_create_user_trace_tasks.rb b/db/migrate/20220513061129_create_user_trace_tasks.rb new file mode 100644 index 000000000..db96ea402 --- /dev/null +++ b/db/migrate/20220513061129_create_user_trace_tasks.rb @@ -0,0 +1,12 @@ +class CreateUserTraceTasks < ActiveRecord::Migration[5.2] + def change + create_table :user_trace_tasks do |t| + t.references :user + t.references :project + t.string :branch_tag + t.string :task_id + + t.timestamps + end + end +end diff --git a/public/docs/api.html b/public/docs/api.html index 45da64400..1fe7bc08e 100644 --- a/public/docs/api.html +++ b/public/docs/api.html @@ -543,6 +543,26 @@ +
  • + Traces + +
  • Pulls