diff --git a/.gitignore b/.gitignore index fbed31a8e..2eac1d277 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,5 @@ redis_data/ Dockerfile dump.rdb .tags* -ceshi_user.xlsx \ No newline at end of file +ceshi_user.xlsx +public/trace_task_results \ No newline at end of file diff --git a/app/controllers/admins/import_users_controller.rb b/app/controllers/admins/import_users_controller.rb index 5d8a60ce6..4575b08d5 100644 --- a/app/controllers/admins/import_users_controller.rb +++ b/app/controllers/admins/import_users_controller.rb @@ -2,7 +2,7 @@ class Admins::ImportUsersController < Admins::BaseController def create return render_error('请上传正确的文件') if params[:file].blank? || !params[:file].is_a?(ActionDispatch::Http::UploadedFile) - result = Admins::ImportUserService.call(params[:file].to_io) + result = Admins::ImportUserFromExcelService.call(params[:file].to_io) render_ok(result) rescue Admins::ImportUserService::Error => ex render_error(ex) diff --git a/app/controllers/owners_controller.rb b/app/controllers/owners_controller.rb index bc4be96c8..28a5210d5 100644 --- a/app/controllers/owners_controller.rb +++ b/app/controllers/owners_controller.rb @@ -12,7 +12,7 @@ class OwnersController < ApplicationController def show @owner = Owner.find_by(login: params[:id]) || Owner.find_by(id: params[:id]) - return render_ok(type: 'User') unless @owner.present? + return render_not_found unless @owner.present? # 组织 if @owner.is_a?(Organization) return render_forbidden("没有查看组织的权限") if org_limited_condition || org_privacy_condition diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 8bc2fb476..68c4e59fb 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -22,6 +22,7 @@ class ProjectsController < ApplicationController menu.append(menu_hash_by_name("devops")) if @project.has_menu_permission("devops") && @project.forge? menu.append(menu_hash_by_name("versions")) if @project.has_menu_permission("versions") menu.append(menu_hash_by_name("wiki")) if @project.has_menu_permission("wiki") && @project.forge? + menu.append(menu_hash_by_name("services")) if @project.has_menu_permission("services") && @project.forge? && (current_user.admin? || @project.member?(current_user.id)) menu.append(menu_hash_by_name("resources")) if @project.has_menu_permission("resources") && @project.forge? menu.append(menu_hash_by_name("activity")) menu.append(menu_hash_by_name("settings")) if user_is_admin && @project.forge? diff --git a/app/controllers/repositories_controller.rb b/app/controllers/repositories_controller.rb index 9fef22fc5..86cef7f32 100644 --- a/app/controllers/repositories_controller.rb +++ b/app/controllers/repositories_controller.rb @@ -225,7 +225,8 @@ class RepositoriesController < ApplicationController @path = Gitea.gitea_config[:domain]+"/#{@owner.login}/#{@repository.identifier}/raw/branch/#{params[:ref]}/" @readme = result[:status] === :success ? result[:body] : nil @readme['content'] = decode64_content(@readme, @owner, @repository, params[:ref], @path) - render json: @readme.slice("type", "encoding", "size", "name", "path", "content", "sha") + @readme['replace_content'] = readme_decode64_content(@readme, @owner, @repository, params[:ref], @path) + render json: @readme.slice("type", "encoding", "size", "name", "path", "content", "sha", "replace_content") rescue render json: nil end @@ -387,4 +388,4 @@ class RepositoriesController < ApplicationController end end -end +end \ No newline at end of file diff --git a/app/controllers/traces/base_controller.rb b/app/controllers/traces/base_controller.rb new file mode 100644 index 000000000..2b857d232 --- /dev/null +++ b/app/controllers/traces/base_controller.rb @@ -0,0 +1,18 @@ +class Traces::BaseController < ApplicationController + + helper_method :observed_logged_user?, :observed_user + + + def observed_user + @_observed_user ||= (User.find_by_login(params[:user_id]) || User.find_by_id(params[:user_id])) + end + + def observed_logged_user? + observed_user.id == User.current&.id + end + + protected + def check_auth + return render_forbidden unless current_user.admin? || observed_logged_user? + end +end \ No newline at end of file diff --git a/app/controllers/traces/projects_controller.rb b/app/controllers/traces/projects_controller.rb new file mode 100644 index 000000000..62573a1d8 --- /dev/null +++ b/app/controllers/traces/projects_controller.rb @@ -0,0 +1,83 @@ +class Traces::ProjectsController < Traces::BaseController + include OperateProjectAbilityAble + + before_action :require_login + before_action :load_project + before_action :authorizate_user_can_edit_project!, except: [:task_results] + + def tasks + branch_name = params[:branch_name] + return render_error("无可用检测次数") if @project.user_trace_tasks.size >= 5 + return render_error("分支名不能为空！") if branch_name.blank? + @all_branches = Gitea::Repository::Branches::ListNameService.call(@project&.owner, @project.identifier) + return render_error("请输入正确的分支名！") unless @all_branches["branch_name"].include?(branch_name) + code, data, error = Trace::CheckService.call(current_user.trace_token, @project, "1", branch_name) + if code == 200 + UserTraceTask.create!( + user_id: current_user.id, + project_id: @project.id, + branch_tag: branch_name, + task_id: data["task_id"] + ) + render_ok + else + render_error("检测失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + def task_results + limit = params[:limit] || params[:per_page] + limit = (limit.to_i.zero? || limit.to_i > 15) ? 15 : limit.to_i + page = params[:page].to_i.zero? ? 1 : params[:page].to_i + return render :json => {left_tasks_count: 5, data: []} if current_user.trace_user.nil? + code, data, error = Trace::CheckResultService.call(current_user.trace_token, @project, nil, page, limit) + if code == 200 + render :json => {left_tasks_count: 5 - @project.user_trace_tasks.size, data: data} + else + render_error("获取检测记录失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + def reload_task + return render_error("project_id错误") if params[:project_id].blank? + branch_name = params[:branch_name] + return render_error("分支名不能为空！") if branch_name.blank? + @all_branches = Gitea::Repository::Branches::ListNameService.call(@project&.owner, @project.identifier) + return render_error("请输入正确的分支名！") unless @all_branches["branch_name"].include?(branch_name) + code, data, error = Trace::ReloadCheckService.call(current_user.trace_token, params[:project_id]) + if code == 200 + UserTraceTask.create!( + user_id: current_user.id, + project_id: @project.id, + branch_tag: branch_name, + task_id: data["task_id"] + ) + render_ok + else + render_error("重新检测失败 Error:#{error}") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end + + + def task_pdf + return render_error("task_id错误") if params[:task_id].blank? + result = Trace::PdfReportService.call(current_user.trace_token, params[:task_id]) + if result.is_a?(Hash) && result[:code] == 200 + redirect_to result[:download_url] + else + render_error("下载报告失败！") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end +end \ No newline at end of file diff --git a/app/controllers/traces/trace_users_controller.rb b/app/controllers/traces/trace_users_controller.rb new file mode 100644 index 000000000..0b28e905a --- /dev/null +++ b/app/controllers/traces/trace_users_controller.rb @@ -0,0 +1,14 @@ +class Traces::TraceUsersController < Traces::BaseController + before_action :require_login + + def create + if current_user.trace_token.present? + render_ok + else + render_error(-1, "代码溯源用户初始化失败") + end + rescue Exception => exception + puts exception.message + normal_status(-1, exception.message) + end +end \ No newline at end of file diff --git a/app/docs/slate/source/api.html.md b/app/docs/slate/source/api.html.md index a846317c4..0031a99f4 100644 --- a/app/docs/slate/source/api.html.md +++ b/app/docs/slate/source/api.html.md @@ -16,6 +16,7 @@ includes: - users - projects - repositories + - traces - pulls - issues - organizations diff --git a/app/docs/slate/source/includes/_projects.md b/app/docs/slate/source/includes/_projects.md index d4899a710..08ad6c234 100644 --- a/app/docs/slate/source/includes/_projects.md +++ b/app/docs/slate/source/includes/_projects.md @@ -280,7 +280,7 @@ repo |是| |string |项目标识identifier ### 返回字段说明 参数 | 类型 | 字段说明 --------- | ----------- | ----------- -menu_name |string|导航名称， home:主页,code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,activity:动态,setting:仓库设置 +menu_name |string|导航名称， home:主页,code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,wiki:维基,services:服务,activity:动态,setting:仓库设置 > 返回的JSON示例: @@ -408,7 +408,7 @@ await octokit.request('POST /api/yystopf/ceshi/project_units') ### 请求参数 参数 | 必选 | 默认 | 类型 | 字段说明 --------- | ------- | ------- | -------- | ---------- -|unit_types |是| |array | 项目模块内容， 支持以下参数:code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑 | +|unit_types |是| |array | 项目模块内容， 支持以下参数:code:代码库,issues:疑修,pulls:合并请求,devops:工作流,versions:里程碑,wiki:维基,resources:资源库,services:服务 | ### 返回字段说明: diff --git a/app/docs/slate/source/includes/_traces.md b/app/docs/slate/source/includes/_traces.md new file mode 100644 index 000000000..e6aa24b48 --- /dev/null +++ b/app/docs/slate/source/includes/_traces.md @@ -0,0 +1,217 @@ +# Traces + +## 代码溯源初始化 +用户同意协议后请求的接口，创建代码溯源的账号 + +> 示例: + +```shell +curl -X POST \ +http://localhost:3000/api/traces/trace_users.json +``` + +```javascript +await octokit.request('POST /api/traces/trace_users.json') +``` + +### HTTP 请求 +`POST api/traces/trace_users.json` + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + +## 代码分析结果列表 +查询项目下代码分析的结果 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/task_results.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/task_results.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/task_results.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner|是|否|string | 项目所有者标识| +repo|是 | 否|string | 项目标识 | +page |否| 1 | int | 页码 | +limit |否| 15 | int | 每页数量 | + +### 返回字段说明（暂缺） + + +> 返回的JSON示例: + +```json +{ + "data": [ + { + "accuracy": "20", + "code_type": "C", + "depth": "2", + "detect_flag": "快速-组件级", + "detect_rule": "快速-组件级,2,20,,开源软件,50,10", + "detect_startdate": "2022-05-10 15:59:46 ", + "detect_status": "fail", + "detectflag": "快速-组件级", + "fail_reason": "Invalid package type", + "file_name": "many_branch.zip", + "license_process": "100", + "licenseparam": "开源软件", + "package_type": "", + "product_name": "84727546110", + "project_id": "6dbc3e42-5857-4ca4-a54d-58fd9dbf6dc5", + "sim_process": "100", + "similarity_process": "2", + "task_id": "15139171-091b-4316-98b1-6068970efa44", + "totalsize": 5, + "uid": "78", + "vuln_process": "", + "vulnlevel": "" + } + ] +} +``` + + + + +## 新建分析 +用户选择仓库分支进行代码分析的接口 + +> 示例: + +```shell +curl -X POST \ +http://localhost:3000/api/traces/yystopf/many_branch/tasks.json +``` + +```javascript +await octokit.request('POST /api/traces/:owner/:repo/tasks.json') +``` + +### HTTP 请求 +`POST api/traces/:owner/:repo/tasks.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +branch_name|是 | 否| string | 分支名称 | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + +## 重新扫描 +对代码分析结果进行再次分析 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/reload_task.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/reload_task.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/reload_task.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +project_id|是 | 否| string | 代码分析结果里的project_id | +branch_name|是 | 否| string | 分支名称 | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + + + +## 下载报告 +把代码分析的结果下载到本地 + +> 示例: + +```shell +curl -X GET \ +http://localhost:3000/api/traces/yystopf/many_branch/task_pdf.json +``` + +```javascript +await octokit.request('GET /api/traces/:owner/:repo/task_pdf.json') +``` + +### HTTP 请求 +`GET api/traces/:owner/:repo/task_pdf.json` + +### 请求参数 +参数 | 必选 | 默认 | 类型 | 字段说明 +--------- | ------- | ------- | -------- | ---------- +owner |是 | 否 | string | 项目所有者标识 | +repo |是 | 否 | string | 项目标识 | +task_id|是 | 否| string | 代码分析结果里的task_id | + + +> 返回的JSON示例: + +```json +{ + "status": 0, + "message": "success" +} +``` + diff --git a/app/helpers/repositories_helper.rb b/app/helpers/repositories_helper.rb index 3af3ef140..8b847a493 100644 --- a/app/helpers/repositories_helper.rb +++ b/app/helpers/repositories_helper.rb @@ -1,169 +1,218 @@ -module RepositoriesHelper - def render_permission(user, project) - return "Admin" if user&.admin? - project.get_premission(user) - end - - def render_decode64_content(str) - return nil if str.blank? - Base64.decode64(str).force_encoding("UTF-8").encode("UTF-8", invalid: :replace) - end - - def download_type(str) - default_type = %w(xlsx xls ppt pptx pdf zip 7z rar exe pdb obj idb RData rdata doc docx mpp vsdx dot otf eot ttf woff woff2 mp4 mov wmv flv mpeg avi avchd webm mkv) - default_type.include?(str&.downcase) || str.blank? - end - - def image_type?(str) - default_type = %w(png jpg gif tif psd svg bmp webp jpeg ico psd) - default_type.include?(str&.downcase) - end - - def is_readme?(type, str) - return false if type != 'file' || str.blank? - readme_types = ["readme.md", "readme", "readme_en.md", "readme_zh.md", "readme_en", "readme_zh"] - readme_types.include?(str.to_s.downcase) - end - - def render_commit_author(author_json) - return nil if author_json.blank? || (author_json["id"].blank? && author_json['name'].blank?) - if author_json["id"].present? - return find_user_by_gitea_uid author_json['id'] - end - if author_json["id"].nil? && (author_json["name"].present? && author_json["email"].present?) - return find_user_by_login_and_mail(author_json['name'], author_json["email"]) - end - end - - def render_cache_commit_author(author_json) - if author_json["name"].present? && author_json["email"].present? - return find_user_in_redis_cache(author_json['name'], author_json['email']) - end - if author_json["Name"].present? && author_json["Email"].present? - return find_user_in_redis_cache(author_json['Name'], author_json['Email']) - end - end - - def readme_render_decode64_content(str, owner, repo, ref, path) - return nil if str.blank? - begin - content = Base64.decode64(str).force_encoding('UTF-8') - - c_regex = /\!\[.*?\]\((.*?)\)/ - src_regex = /src=\"(.*?)\"/ - src2_regex = /src='(.*?)'/ - ss = content.to_s.scan(c_regex) - ss_src = content.scan(src_regex) - ss_src2 = content.scan(src2_regex) - total_images = ss + ss_src + ss_src2 - if total_images.length > 0 - total_images.each do |s| - begin - image_title = /\"(.*?)\"/ - r_content = s[0] - remove_title = r_content.to_s.scan(image_title) - # if remove_title.length > 0 - # r_content = r_content.gsub(/#{remove_title[0]}/, "").strip - # end - path_last = r_content - path_current = "" - # 相对路径处理 - if r_content.start_with?("../") - relative_path_length = r_content.split("../").size - 1 - path_pre = path.split("/").size - 1 - relative_path_length - path_pre = 0 if path_pre < 0 - path_current = path_pre == 0 ? "" : path.split("/")[0..path_pre].join("/") - path_last = r_content.split("../").last - elsif r_content.start_with?("/") # 根路径处理 - path_last = r_content[1..r_content.size] - else - path_current = path - end - # if r_content.include?("?") - # new_r_content = r_content + "&raw=true" - # else - # new_r_content = r_content + "?raw=true" - # end - new_r_content = r_content - - unless r_content.include?("http://") || r_content.include?("https://") || r_content.include?("mailto:") - # new_r_content = "#{path}" + new_r_content - new_r_content = [base_url, "/api/#{owner&.login}/#{repo.identifier}/raw?filepath=#{path_current}/#{path_last}&ref=#{ref}"].join - end - content = content.gsub(/src=\"#{r_content}\"/, "src=\"#{new_r_content}\"").gsub(/src='#{r_content}'/, "src=\"#{new_r_content}\"") - rescue - next - end - end - end - - return content - rescue - return str - end - end - - # unix_time values for example: 1604382982 - def render_format_time_with_unix(unix_time) - Time.at(unix_time).strftime("%Y-%m-%d %H:%M") - end - - # date for example: 2020-11-01T19:57:27+08:00 - def render_format_time_with_date(date) - date.to_time.strftime("%Y-%m-%d %H:%M") - end - - def decode64_content(entry, owner, repo, ref, path=nil) - if is_readme?(entry['type'], entry['name']) - # content = Gitea::Repository::Entries::GetService.call(owner, repo.identifier, URI.escape(entry['path']), ref: ref)['content'] - content = entry['content'] - path = URI.escape(entry['path']).to_s.downcase.gsub("/readme.md","") - readme_render_decode64_content(content, owner, repo, ref, path) - else - file_type = File.extname(entry['name'].to_s)[1..-1] - if image_type?(file_type) - return entry['content'].nil? ? Gitea::Repository::Entries::GetService.call(owner, repo.identifier, URI.escape(entry['path']), ref: ref)['content'] : entry['content'] - end - if download_type(file_type) - return entry['content'] - end - render_decode64_content(entry['content']) - end - end - - def base64_to_image(path, content) - # generate to https://git.trusite.net/pawm36ozq/-/raw/branch/master/entrn.png" - content = Base64.decode64(content) - File.open(path, 'wb') { |f| f.write(content) } - end - - def render_download_image_url(dir_path, file_path, content) - full_path = file_path.starts_with?("/") ? [dir_path, file_path].join("") : [dir_path, file_path].join("/") - file_name = full_path.split("/")[-1] - # 用户名/项目标识/文件路径 - dir_path = generate_dir_path(full_path.split("/"+file_name)[0]) - - file_path = [dir_path, file_name].join('/') - - puts "##### render_download_image_url file_path: #{file_path}" - base64_to_image(file_path, content) - file_path = file_path[6..-1] - File.join(base_url, file_path) - end - - def generate_dir_path(dir_path) - # tmp_dir_path - # eg: jasder/forgeplus/raw/branch/ref - dir_path = ["public", tmp_dir, dir_path].join('/') - puts "#### dir_path: #{dir_path}" - unless Dir.exists?(dir_path) - FileUtils.mkdir_p(dir_path) ##不成功这里会抛异常 - end - dir_path - end - - def tmp_dir - "repo" - end - -end +module RepositoriesHelper + def render_permission(user, project) + return "Admin" if user&.admin? + project.get_premission(user) + end + + def render_decode64_content(str) + return nil if str.blank? + Base64.decode64(str).force_encoding("UTF-8").encode("UTF-8", invalid: :replace) + end + + def download_type(str) + default_type = %w(xlsx xls ppt pptx pdf zip 7z rar exe pdb obj idb RData rdata doc docx mpp vsdx dot otf eot ttf woff woff2 mp4 mov wmv flv mpeg avi avchd webm mkv) + default_type.include?(str&.downcase) || str.blank? + end + + def image_type?(str) + default_type = %w(png jpg gif tif psd svg bmp webp jpeg ico psd) + default_type.include?(str&.downcase) + end + + def is_readme?(type, str) + return false if type != 'file' || str.blank? + readme_types = ["readme.md", "readme", "readme_en.md", "readme_zh.md", "readme_en", "readme_zh"] + readme_types.include?(str.to_s.downcase) + end + + def render_commit_author(author_json) + return nil if author_json.blank? || (author_json["id"].blank? && author_json['name'].blank?) + if author_json["id"].present? + return find_user_by_gitea_uid author_json['id'] + end + if author_json["id"].nil? && (author_json["name"].present? && author_json["email"].present?) + return find_user_by_login_and_mail(author_json['name'], author_json["email"]) + end + end + + def render_cache_commit_author(author_json) + if author_json["name"].present? && author_json["email"].present? + return find_user_in_redis_cache(author_json['name'], author_json['email']) + end + if author_json["Name"].present? && author_json["Email"].present? + return find_user_in_redis_cache(author_json['Name'], author_json['Email']) + end + end + + def readme_render_decode64_content(str, owner, repo, ref, path) + return nil if str.blank? + begin + content = Base64.decode64(str).force_encoding('UTF-8') + + c_regex = /\!\[.*?\]\((.*?)\)/ + src_regex = /src=\"(.*?)\"/ + src2_regex = /src='(.*?)'/ + ss = content.to_s.scan(c_regex) + ss_src = content.scan(src_regex) + ss_src2 = content.scan(src2_regex) + total_images = ss + ss_src + ss_src2 + if total_images.length > 0 + total_images.each do |s| + begin + image_title = /\"(.*?)\"/ + r_content = s[0] + remove_title = r_content.to_s.scan(image_title) + # if remove_title.length > 0 + # r_content = r_content.gsub(/#{remove_title[0]}/, "").strip + # end + path_last = r_content + path_current = "" + # 相对路径处理 + if r_content.start_with?("../") + relative_path_length = r_content.split("../").size - 1 + path_pre = path.split("/").size - 1 - relative_path_length + path_pre = 0 if path_pre < 0 + path_current = path_pre == 0 ? "" : path.split("/")[0..path_pre].join("/") + path_last = r_content.split("../").last + elsif r_content.start_with?("/") # 根路径处理 + path_last = r_content[1..r_content.size] + else + path_current = path + end + # if r_content.include?("?") + # new_r_content = r_content + "&raw=true" + # else + # new_r_content = r_content + "?raw=true" + # end + new_r_content = r_content + + unless r_content.include?("http://") || r_content.include?("https://") || r_content.include?("mailto:") + # new_r_content = "#{path}" + new_r_content + new_r_content = [base_url, "/api/#{owner&.login}/#{repo.identifier}/raw?filepath=#{path_current}/#{path_last}&ref=#{ref}"].join + end + content = content.gsub(/src=\"#{r_content}\"/, "src=\"#{new_r_content}\"").gsub(/src='#{r_content}'/, "src=\"#{new_r_content}\"") + rescue + next + end + end + end + + return content + rescue + return str + end + end + + # author hui.he + def new_readme_render_decode64_content(str, owner, repo, ref, readme_path, readme_name) + file_path = readme_path.include?('/') ? readme_path.gsub("/#{readme_name}", '') : readme_path.gsub("#{readme_name}", '') + return nil if str.blank? + content = Base64.decode64(str).force_encoding('UTF-8') + s_regex = /\[.*?\]\((.*?)\)/ + src_regex = /src=\"(.*?)\"/ + ss = content.to_s.scan(s_regex) + ss_src = content.to_s.scan(src_regex) + total_sources = ss + ss_src + total_sources.uniq! + total_sources.each do |s| + begin + s_content = s[0] + # 链接直接跳过不做替换 + next if s_content.starts_with?('http://') || s_content.starts_with?('https://') || s_content.starts_with?('mailto:') + ext = File.extname(s_content)[1..-1] + + if image_type?(ext) || download_type(ext) + s_content = File.expand_path(s_content, file_path) + s_content = s_content.split("#{Rails.root}/")[1] + # content = content.gsub(s[0], "/#{s_content}") + s_content = [base_url, "/api/#{owner&.login}/#{repo.identifier}/raw?filepath=#{s_content}&ref=#{ref}"].join + content = content.gsub(s[0], s_content) + else + path = [owner&.login, repo&.identifier, 'tree', ref, file_path].join("/") + s_content = File.expand_path(s_content, path) + s_content = s_content.split("#{Rails.root}/")[1] + content = content.gsub(s[0], "/#{s_content}") + end + rescue + next + end + end + + return content + rescue + return str + end + + # unix_time values for example: 1604382982 + def render_format_time_with_unix(unix_time) + Time.at(unix_time).strftime("%Y-%m-%d %H:%M") + end + + # date for example: 2020-11-01T19:57:27+08:00 + def render_format_time_with_date(date) + date.to_time.strftime("%Y-%m-%d %H:%M") + end + + def readme_decode64_content(entry, owner, repo, ref, path=nil) + Rails.logger.info("entry===#{entry["type"]} #{entry["name"]}") + content = Gitea::Repository::Entries::GetService.call(owner, repo.identifier, URI.escape(entry['path']), ref: ref)['content'] + Rails.logger.info("content===#{content}") + # readme_render_decode64_content(content, owner, repo, ref) + new_readme_render_decode64_content(content, owner, repo, ref, entry['path'], entry['name']) + end + + def decode64_content(entry, owner, repo, ref, path=nil) + if is_readme?(entry['type'], entry['name']) + Rails.logger.info("entry===#{entry["type"]} #{entry["name"]}") + content = Gitea::Repository::Entries::GetService.call(owner, repo.identifier, URI.escape(entry['path']), ref: ref)['content'] + Rails.logger.info("content===#{content}") + # readme_render_decode64_content(content, owner, repo, ref) + return Base64.decode64(content).force_encoding('UTF-8') + else + file_type = File.extname(entry['name'].to_s)[1..-1] + if image_type?(file_type) + return entry['content'].nil? ? Gitea::Repository::Entries::GetService.call(owner, repo.identifier, URI.escape(entry['path']), ref: ref)['content'] : entry['content'] + end + if download_type(file_type) + return entry['content'] + end + render_decode64_content(entry['content']) + end + end + + def base64_to_image(path, content) + # generate to https://git.trusite.net/pawm36ozq/-/raw/branch/master/entrn.png" + content = Base64.decode64(content) + File.open(path, 'wb') { |f| f.write(content) } + end + + def render_download_image_url(dir_path, file_path, content) + full_path = file_path.starts_with?("/") ? [dir_path, file_path].join("") : [dir_path, file_path].join("/") + file_name = full_path.split("/")[-1] + # 用户名/项目标识/文件路径 + dir_path = generate_dir_path(full_path.split("/"+file_name)[0]) + + file_path = [dir_path, file_name].join('/') + + puts "##### render_download_image_url file_path: #{file_path}" + base64_to_image(file_path, content) + file_path = file_path[6..-1] + File.join(base_url, file_path) + end + + def generate_dir_path(dir_path) + # tmp_dir_path + # eg: jasder/forgeplus/raw/branch/ref + dir_path = ["public", tmp_dir, dir_path].join('/') + puts "#### dir_path: #{dir_path}" + unless Dir.exists?(dir_path) + FileUtils.mkdir_p(dir_path) ##不成功这里会抛异常 + end + dir_path + end + + def tmp_dir + "repo" + end + +end diff --git a/app/imports/admins/new_import_user_from_excel.rb b/app/imports/admins/new_import_user_from_excel.rb new file mode 100644 index 000000000..b9a452cf1 --- /dev/null +++ b/app/imports/admins/new_import_user_from_excel.rb @@ -0,0 +1,15 @@ +class Admins::NewImportUserFromExcel < BaseImportXlsx + UserData = Struct.new(:login, :email, :password, :nickname) + + def read_each(&block) + sheet.each_row_streaming(pad_cells: true, offset: 1) do |row| + data = row.map(&method(:cell_value))[0..3] + block.call UserData.new(*data) + end + end + + private + def cell_value(obj) + obj&.cell_value + end +end \ No newline at end of file diff --git a/app/libs/custom_regexp.rb b/app/libs/custom_regexp.rb index 25c3ae988..72184630a 100644 --- a/app/libs/custom_regexp.rb +++ b/app/libs/custom_regexp.rb @@ -1,7 +1,7 @@ module CustomRegexp PHONE = /1\d{10}/ EMAIL = /\A[a-zA-Z0-9]+([._\\]*[a-zA-Z0-9])*@([a-z0-9]+[-a-z0-9]*[a-z0-9]+.){1,63}[a-z0-9]+\z/ - LOGIN = /^(?!_)(?!.*?_$)[a-zA-Z0-9_-]+$/ #只含有数字、字母、下划线不能以下划线开头和结尾 + LOGIN = /^(?!_)(?!.*?_$)[a-zA-Z0-9_-]{4,15}/ #只含有数字、字母、下划线不能以下划线开头和结尾 LASTNAME = /\A[a-zA-Z0-9\u4e00-\u9fa5]+\z/ NICKNAME = /\A[\u4e00-\u9fa5_a-zA-Z0-9]+\z/ PASSWORD = /\A[a-z_A-Z0-9\-\.!@#\$%\\\^&\*\)\(\+=\{\}\[\]\/",'_<>~\·`\?:;|]{8,16}\z/ diff --git a/app/models/message_template/project_setting_changed.rb b/app/models/message_template/project_setting_changed.rb index 0920dfe7a..ab597122a 100644 --- a/app/models/message_template/project_setting_changed.rb +++ b/app/models/message_template/project_setting_changed.rb @@ -141,6 +141,7 @@ class MessageTemplate::ProjectSettingChanged < MessageTemplate navbar.gsub!('devops', '工作流') navbar.gsub!('versions', '里程碑') navbar.gsub!('resources', '资源库') + navbar.gsub!('services', '服务') if change_count > 1 content.sub!('{ifnavbar}', '
') else @@ -290,6 +291,7 @@ class MessageTemplate::ProjectSettingChanged < MessageTemplate navbar.gsub!('devops', '工作流') navbar.gsub!('versions', '里程碑') navbar.gsub!('resources', '资源库') + navbar.gsub!('services', '服务') if change_count > 1 content.sub!('{ifnavbar}', '
') else diff --git a/app/models/project.rb b/app/models/project.rb index a35fbf387..1edc088f6 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -124,6 +124,7 @@ class Project < ApplicationRecord has_many :pinned_projects, dependent: :destroy has_many :has_pinned_users, through: :pinned_projects, source: :user has_many :webhooks, class_name: "Gitea::Webhook", primary_key: :gpid, foreign_key: :repo_id + has_many :user_trace_tasks, dependent: :destroy after_create :incre_user_statistic, :incre_platform_statistic after_save :check_project_members before_save :set_invite_code, :reset_unmember_followed, :set_recommend_and_is_pinned, :reset_cache_data diff --git a/app/models/project_unit.rb b/app/models/project_unit.rb index cc35a6b28..6ee0f2a8b 100644 --- a/app/models/project_unit.rb +++ b/app/models/project_unit.rb @@ -16,7 +16,7 @@ class ProjectUnit < ApplicationRecord belongs_to :project - enum unit_type: {code: 1, issues: 2, pulls: 3, wiki:4, devops: 5, versions: 6, resources: 7} + enum unit_type: {code: 1, issues: 2, pulls: 3, wiki:4, devops: 5, versions: 6, resources: 7, services: 8} validates :unit_type, uniqueness: { scope: :project_id} diff --git a/app/models/user.rb b/app/models/user.rb index 66aeb6164..bf764c148 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -175,6 +175,7 @@ class User < Owner has_many :system_notification_histories has_many :system_notifications, through: :system_notification_histories has_one :trace_user, dependent: :destroy + has_many :user_trace_tasks, dependent: :destroy # Groups and active users scope :active, lambda { where(status: [STATUS_ACTIVE, STATUS_EDIT_INFO]) } diff --git a/app/models/user_trace_task.rb b/app/models/user_trace_task.rb new file mode 100644 index 000000000..328cb7c0b --- /dev/null +++ b/app/models/user_trace_task.rb @@ -0,0 +1,25 @@ +# == Schema Information +# +# Table name: user_trace_tasks +# +# id :integer not null, primary key +# user_id :integer +# project_id :integer +# branch_tag :string(255) +# task_id :string(255) +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_user_trace_tasks_on_project_id (project_id) +# index_user_trace_tasks_on_user_id (user_id) +# + +class UserTraceTask < ApplicationRecord + + belongs_to :user + belongs_to :project + + +end diff --git a/app/services/admins/import_user_from_excel_service.rb b/app/services/admins/import_user_from_excel_service.rb new file mode 100644 index 000000000..709551c23 --- /dev/null +++ b/app/services/admins/import_user_from_excel_service.rb @@ -0,0 +1,71 @@ +class Admins::ImportUserFromExcelService < ApplicationService + Error = Class.new(StandardError) + + attr_reader :file, :result + + def initialize(file) + @file = file + @result = { success: 0, fail: [] } + end + + def call + raise Error, '文件不存在' if file.blank? + excel = Admins::NewImportUserFromExcel.new(file) + + excel.read_each(&method(:save_user)) + result + rescue ApplicationImport::Error => ex + raise Error, ex.message + end + + private + def save_user(data) + user = find_user(data) + if user.blank? + create_user(data) + result[:success] +=1 + else + fail_data = data.as_json + fail_data[:data] = fail_data.values.join(",") + fail_data[:message] = '用户已存在' + result[:fail] << fail_data + end + + rescue Exception => ex + fail_data = data.as_json + fail_data[:data] = fail_data.values.join(",") + fail_data[:message] = ex.message + result[:fail] << fail_data + end + + def create_user(data) + ActiveRecord::Base.transaction do + username = data.login&.gsub(/\s+/, "") + email = data.email&.gsub(/\s+/, "") + password = data.password + nickname = data.nickname&.gsub(/\s+/, "") + raise Error, "无法使用以下关键词：#{username}，请重新命名" if ReversedKeyword.check_exists?(data.login) + Register::RemoteForm.new({username: username, email: email, password: password, platform: 'forge'}).validate! + user = User.new(admin: false, login: username, mail: email, nickname: nickname, platform: 'forge' , type: "User") + user.password = password + user.activate + raise Error, user.errors.full_messages.join(",") unless user.valid? + interactor = Gitea::RegisterInteractor.call({username: username, email: email, password: password}) + if interactor.success? + gitea_user = interactor.result + result = Gitea::User::GenerateTokenService.call(username, password) + user.gitea_token = result['sha1'] + user.gitea_uid = gitea_user[:body]['id'] + UserExtension.create!(user_id: user.id) if user.save! + else + raise interactor.error, 'gitea user create error' + end + + user + end + end + + def find_user(data) + User.find_by(login: data.login) + end +end \ No newline at end of file diff --git a/app/services/trace/check_result_service.rb b/app/services/trace/check_result_service.rb index f1dd61ab0..ff47d9a39 100644 --- a/app/services/trace/check_result_service.rb +++ b/app/services/trace/check_result_service.rb @@ -1,11 +1,11 @@ # 代码溯源 查询检测结果 class Trace::CheckResultService < Trace::ClientService - attr_accessor :token, :project_name, :file_name, :page_num, :page_size + attr_accessor :token, :project, :file_name, :page_num, :page_size - def initialize(token, project_name=nil, file_name=nil, page_num=1, page_size=15) + def initialize(token, project, file_name=nil, page_num=1, page_size=15) @token = token - @project_name = project_name + @project = project @file_name = file_name @page_num = page_num @page_size = page_size @@ -19,7 +19,7 @@ class Trace::CheckResultService < Trace::ClientService private def request_params { - product_name: project_name, + product_name: Digest::MD5.hexdigest(project&.id.to_s)[0...20], file_name: file_name, pageNum: page_num, pageSize: page_size, diff --git a/app/services/trace/check_service.rb b/app/services/trace/check_service.rb index d31bbcf09..d1623445b 100644 --- a/app/services/trace/check_service.rb +++ b/app/services/trace/check_service.rb @@ -11,26 +11,25 @@ class Trace::CheckService < Trace::ClientService end def call - result = authed_post(token, url, {data: request_params}) + result = http_authed_post(token, url, {data: request_params}) reponse = render_response(result) end private def request_params - repo = Gitea::Repository::GetService.call(project&.owner&.login, project&.identifier) + repo = Gitea::Repository::GetService.call(project&.owner, project&.identifier) { - product_name: project&.name, - product_type: project&.category&.name, - code_type: project&.language&.name, + product_name: Digest::MD5.hexdigest(project&.id.to_s)[0...20], + product_type: project&.project_category&.name || '其他', + code_type: project&.project_language&.name || '其他', product_desc: project&.description, git_url: repo['clone_url'], if_branch: if_branch, branch_tag: branch_tag - } + }.compact end def url "/user/check".freeze end -end - +end \ No newline at end of file diff --git a/app/services/trace/client_service.rb b/app/services/trace/client_service.rb index 72ffa8ca2..0f1449225 100644 --- a/app/services/trace/client_service.rb +++ b/app/services/trace/client_service.rb @@ -12,6 +12,19 @@ class Trace::ClientService < ApplicationService conn.post(full_url(url), params[:data]) end + def http_authed_post(token, url, params={}) + puts "[trace][POST] request params: #{params}" + puts "[trace][POST] request token: #{token}" + url = URI("#{full_url(url)}") + http = Net::HTTP.new(url.host, url.port) + http.read_timeout = 1200 + request = Net::HTTP::Post.new(url) + request["Authorization"] = token + form_data = params[:data].stringify_keys.to_a + request.set_form form_data, 'multipart/form-data' + http.request(request) + end + def get(url, params={}) puts "[trace][GET] request params: #{params}" conn.get do |req| @@ -100,11 +113,22 @@ class Trace::ClientService < ApplicationService end def render_response(response) - status = response.status - body = JSON.parse(response&.body) + if response.is_a?(Faraday::Response) + status = response.status + body = JSON.parse(response&.body) - log_error(status, body) + log_error(status, body) - return [body["code"], body["data"], body["error"]] + return [body["code"], body["data"], body["error"]] + end + + if response.is_a?(Net::HTTPOK) + status = 200 + body = JSON.parse(response&.body) + + log_error(status, body) + + return [body["code"], body["data"], body["error"]] + end end end \ No newline at end of file diff --git a/app/services/trace/pdf_report_service.rb b/app/services/trace/pdf_report_service.rb index e91a78b30..aa7312739 100644 --- a/app/services/trace/pdf_report_service.rb +++ b/app/services/trace/pdf_report_service.rb @@ -1,4 +1,7 @@ # 代码溯源 导出pdf +require 'open-uri' +require 'fileutils' + class Trace::PdfReportService < Trace::ClientService attr_accessor :token, :task_id @@ -9,15 +12,23 @@ class Trace::PdfReportService < Trace::ClientService end def call - result = authed_get(token, url, request_params) - response = render_response(result) + content = open("#{domain}#{base_url}#{url}?task_id=#{task_id}", "Authorization" => token) + if content.is_a?(Tempfile) + check_file_path + IO.copy_stream(content, "#{save_path}/#{task_id}.pdf") + return {code: 200, download_url: "/trace_task_results/#{task_id}.pdf"} + else + return {code: 404} + end end private - def request_params - { - task_id: task_id - } + def check_file_path + FileUtils.mkdir_p save_path + end + + def save_path + "public/trace_task_results" end def url diff --git a/app/views/admins/users/index.html.erb b/app/views/admins/users/index.html.erb index e21a3d665..2f41ffdb6 100644 --- a/app/views/admins/users/index.html.erb +++ b/app/views/admins/users/index.html.erb @@ -31,7 +31,9 @@ <%= submit_tag('搜索', class: 'btn btn-primary ml-3', 'data-disable-with': '搜索中...') %> <% end %> - <%= javascript_void_link '导入用户', class: 'btn btn-secondary btn-sm', data: { toggle: 'modal', target: '.admin-import-user-modal'} %> + <%= link_to '下载导入模板', "/导入用户模板.xlsx", class: 'btn btn-secondary mr-3' %> + + <%= javascript_void_link '导入用户', class: 'btn btn-secondary', data: { toggle: 'modal', target: '.admin-import-user-modal'} %> diff --git a/app/views/repositories/_simple_entry.json.jbuilder b/app/views/repositories/_simple_entry.json.jbuilder index 261eb4d22..d11d9af43 100644 --- a/app/views/repositories/_simple_entry.json.jbuilder +++ b/app/views/repositories/_simple_entry.json.jbuilder @@ -9,7 +9,10 @@ if @project.forge? json.path entry['path'] json.type entry['type'] json.size entry['size'] - + is_readme = is_readme?(entry['type'], entry['name']) + if is_readme + json.replace_content readme_decode64_content(entry, @owner, @repository, @ref, @path) + end json.content (direct_download || image_type || is_dir) ? nil : decode64_content(entry, @owner, @repository, @ref, @path) json.target entry['target'] @@ -25,7 +28,7 @@ if @project.forge? json.direct_download direct_download json.image_type image_type - json.is_readme_file is_readme?(entry['type'], entry['name']) + json.is_readme_file is_readme json.commit do json.partial! 'last_commit', latest_commit: entry['latest_commit'] end diff --git a/app/views/users/get_user_info.json.jbuilder b/app/views/users/get_user_info.json.jbuilder index 95085580e..d60f46ade 100644 --- a/app/views/users/get_user_info.json.jbuilder +++ b/app/views/users/get_user_info.json.jbuilder @@ -25,3 +25,4 @@ json.description @user.description json.super_description @user.super_description json.(@user, :show_email, :show_department, :show_location, :show_super_description) json.message_unread_total @message_unread_total +json.has_trace_user @user.trace_user.present? \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 64dce0e60..097aaf2a7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -427,6 +427,20 @@ Rails.application.routes.draw do end end + namespace :traces do + resources :trace_users, only: [:create] + scope "/:owner/:repo" do + resource :projects, path: '/', only: [:index] do + member do + post :tasks + get :task_results + get :reload_task + get :task_pdf + end + end + end + end + # Project Area START scope "/:owner/:repo" do scope do diff --git a/db/migrate/20220513061129_create_user_trace_tasks.rb b/db/migrate/20220513061129_create_user_trace_tasks.rb new file mode 100644 index 000000000..db96ea402 --- /dev/null +++ b/db/migrate/20220513061129_create_user_trace_tasks.rb @@ -0,0 +1,12 @@ +class CreateUserTraceTasks < ActiveRecord::Migration[5.2] + def change + create_table :user_trace_tasks do |t| + t.references :user + t.references :project + t.string :branch_tag + t.string :task_id + + t.timestamps + end + end +end diff --git a/public/docs/api.html b/public/docs/api.html index 45da64400..1fe7bc08e 100644 --- a/public/docs/api.html +++ b/public/docs/api.html @@ -543,6 +543,26 @@ +
  • + Traces + +
  • Pulls