From c2484119f93cf70c8535d16a4278a6b5937c63be Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 13:53:02 +0800 Subject: [PATCH 1/8] rake commit_log_to_db --- lib/tasks/commit_log_to_db.rake | 76 +++++++++++++++++++++++++++++++ lib/tasks/total_commit_to_db.rake | 65 ++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 lib/tasks/commit_log_to_db.rake create mode 100644 lib/tasks/total_commit_to_db.rake diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake new file mode 100644 index 000000000..5cb0003f3 --- /dev/null +++ b/lib/tasks/commit_log_to_db.rake @@ -0,0 +1,76 @@ +namespace :commit_log_to_db do + desc "commit_log_to_db" + task done: :environment do + puts "project_id=================#{ENV['project_id']}" + return if ENV['project_id'].blank? + projects = Project.where(id: ENV['project_id']) + projects.each_with_index do |project, index| + result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 200, token: project.owner.gitea_token) + next if result.blank? || result[:total_count].blank? + total_count = result[:total_count] + # next if total_count > 2000 + puts "#{index} total_count==========#{total_count}" + if total_count > 200 + total_page = (total_count / 200) + 1 + total_page.times do |i| + add_commit_by_page(project, i + 1) + end + else + # add_commit_to_index(project, 1) + data = "" + result[:body].each do |commit| + # puts "commit==========#{commit}" + commiter = commit['commit']['committer'] + # "luoyuan " + commit_author = "#{commiter['name']} <#{commiter['email']}>" + commit_sha = commit['sha'] + ref = "master" + commit_message = commit['commit']['message'] + user = User.find_by(mail: commiter['email']) + commit_date = Time.parse(commit['commit']['author']['date']) + commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") + + data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}','#{commit_message}','#{commit_date_str}','#{commit_date_str}')," + end + data = data[0,data.length-1] + sql_connection = ActiveRecord::Base.connection + sql_connection.begin_db_transaction + sql = "INSERT INTO commit_logs (`user_id`, `project_id`, `repository_id`, `name`, `full_name`, `commit_id`, `ref`, `message`, `created_at`, `updated_at`) VALUES #{data}" + sql_connection.execute(sql) + end + end + + # Time.now + # Wed Mar 15 14:12:09 2023 +0800 + # Time.now.strftime("%a %b %d %H:%M:%S %Y") + # Time.now.strftime("%a %b %d %H:%M:%S %Y +0800") + Time.parse("2023-03-15 14:12:09").strftime("%a %b %d %H:%M:%S %Y +0800") + + end + + def add_commit_by_page(project, page) + # Gitea::Repository::Commits::ListSliceService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 1000, token: "a9244ecac647dd33fee3b480c5898baab1d3fe7d") + result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: page, limit: 200, token: project.owner.gitea_token) + data = "" + result[:body].each do |commit| + # puts "commit==========#{commit}" + commiter = commit['commit']['committer'] + # "luoyuan " + commit_author = "#{commiter['name']} <#{commiter['email']}>" + commit_sha = commit['sha'] + ref = "master" + commit_message = commit['commit']['message'] + user = User.find_by(mail: commiter['email']) + commit_date = Time.parse(commit['commit']['author']['date']) + commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") + + data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}','#{commit_message}','#{commit_date_str}','#{commit_date_str}')," + end + data = data[0,data.length-1] + sql_connection = ActiveRecord::Base.connection + sql_connection.begin_db_transaction + sql = "INSERT INTO commit_logs (`user_id`, `project_id`, `repository_id`, `name`, `full_name`, `commit_id`, `ref`, `message`, `created_at`, `updated_at`) VALUES #{data}" + sql_connection.execute(sql) + end + +end \ No newline at end of file diff --git a/lib/tasks/total_commit_to_db.rake b/lib/tasks/total_commit_to_db.rake new file mode 100644 index 000000000..cdb29e108 --- /dev/null +++ b/lib/tasks/total_commit_to_db.rake @@ -0,0 +1,65 @@ +namespace :total_commit_to_db do + desc "total_commit_to_db" + task done: :environment do + project_name = ENV['name'] || "mindspore" + puts "project_id=================#{project_name}" + projects = Project.where(identifier: ['MindSpore-first-experience', ' MindSpore-install', 'MindSpore-Application-practice', 'MindSpore-Model-Development', 'MindSpore-Data-preprocessing', 'Mindspore-Data-storage-use', 'MindSpore-Data-storage-kunpeng', 'MindSpore-LeNet-jzx3', 'MindSpore-competition'] ) + + projects.each_with_index do |project, index| + result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 5, token: project.owner.gitea_token) + next if result.blank? || result[:total_count].blank? + total_count = result[:total_count] + # next if total_count > 2000 + puts "#{index} total_count==========#{total_count}" + if total_count > 200 + total_page = (total_count / 200) + 1 + total_page.times do |i| + add_commit_to_index(project, i + 1) + end + else + # add_commit_to_index(project, 1) + data = "" + result[:body].each do |commit| + commit_date = Time.parse(commit['commit']['author']['date']) + commit_date_str = commit_date.strftime("%Y-%m-%d") + data += "(\"#{commit_date_str}\",1)," + end + data = data[0,data.length-1] + sql_connection = ActiveRecord::Base.connection + sql_connection.begin_db_transaction + sql = "insert into mindspore_commit(week_date,num) values #{data}" + sql_connection.execute(sql) + end + puts "#{index} date_count_hash===========#{@date_count_hash.to_json}" + + end + puts "@date_count_hash===========#{@date_count_hash.to_json}" + + + + + # Time.now + # Wed Mar 15 14:12:09 2023 +0800 + # Time.now.strftime("%a %b %d %H:%M:%S %Y") + # Time.now.strftime("%a %b %d %H:%M:%S %Y +0800") + Time.parse("2023-03-15 14:12:09").strftime("%a %b %d %H:%M:%S %Y +0800") + + end + + def add_commit_to_index(project, page) + # Gitea::Repository::Commits::ListSliceService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 1000, token: "a9244ecac647dd33fee3b480c5898baab1d3fe7d") + result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: page, limit: 200, token: project.owner.gitea_token) + data = "" + result[:body].each do |commit| + commit_date = Time.parse(commit['commit']['author']['date']) + commit_date_str = commit_date.strftime("%Y-%m-%d") + data += "(\"#{commit_date_str}\",1)," + end + data = data[0,data.length-1] + sql_connection = ActiveRecord::Base.connection + sql_connection.begin_db_transaction + sql = "insert into mindspore_commit(week_date,num) values #{data}" + sql_connection.execute(sql) + end + +end \ No newline at end of file From 0fa0cc492e8d34ee9ad6170333dc47528932e2ab Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 13:59:09 +0800 Subject: [PATCH 2/8] =?UTF-8?q?rake=20commit=5Flog=5Fto=5Fdb=20=E6=97=A5?= =?UTF-8?q?=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/commit_log_to_db.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake index 5cb0003f3..bde59a337 100644 --- a/lib/tasks/commit_log_to_db.rake +++ b/lib/tasks/commit_log_to_db.rake @@ -62,7 +62,7 @@ namespace :commit_log_to_db do commit_message = commit['commit']['message'] user = User.find_by(mail: commiter['email']) commit_date = Time.parse(commit['commit']['author']['date']) - commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") + commit_date_str = commit_date.strftime("%Y-%m-%d %H:%M:%S") data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}','#{commit_message}','#{commit_date_str}','#{commit_date_str}')," end From 15100689c3f8a592d5c7c0e359dabf67276dcad8 Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:01:48 +0800 Subject: [PATCH 3/8] =?UTF-8?q?rake=20commit=5Flog=5Fto=5Fdb=20sql?= =?UTF-8?q?=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/commit_log_to_db.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake index bde59a337..bd45d91ce 100644 --- a/lib/tasks/commit_log_to_db.rake +++ b/lib/tasks/commit_log_to_db.rake @@ -30,7 +30,7 @@ namespace :commit_log_to_db do commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") - data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}','#{commit_message}','#{commit_date_str}','#{commit_date_str}')," + data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," end data = data[0,data.length-1] sql_connection = ActiveRecord::Base.connection @@ -64,7 +64,7 @@ namespace :commit_log_to_db do commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%Y-%m-%d %H:%M:%S") - data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}','#{commit_message}','#{commit_date_str}','#{commit_date_str}')," + data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," end data = data[0,data.length-1] sql_connection = ActiveRecord::Base.connection From ffea28596222b13e926588a2e7cd432ebe6db835 Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:08:20 +0800 Subject: [PATCH 4/8] =?UTF-8?q?rake=20commit=5Flog=5Fto=5Fdb=20=E6=94=BE?= =?UTF-8?q?=E8=A1=8C=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/commit_log_to_db.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake index bd45d91ce..a85084c74 100644 --- a/lib/tasks/commit_log_to_db.rake +++ b/lib/tasks/commit_log_to_db.rake @@ -25,7 +25,7 @@ namespace :commit_log_to_db do commit_author = "#{commiter['name']} <#{commiter['email']}>" commit_sha = commit['sha'] ref = "master" - commit_message = commit['commit']['message'] + commit_message = commit['commit']['message'].to_s.gsub("/n","") user = User.find_by(mail: commiter['email']) commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") @@ -59,7 +59,7 @@ namespace :commit_log_to_db do commit_author = "#{commiter['name']} <#{commiter['email']}>" commit_sha = commit['sha'] ref = "master" - commit_message = commit['commit']['message'] + commit_message = commit['commit']['message'].to_s.gsub("/n","") user = User.find_by(mail: commiter['email']) commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%Y-%m-%d %H:%M:%S") From b8e31b8d0bce624f0814722b4171a17fa48e05ae Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:15:45 +0800 Subject: [PATCH 5/8] =?UTF-8?q?rake=20commit=5Flog=5Fto=5Fdb=20=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=B8=8D=E5=AD=98=E5=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/commit_log_to_db.rake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake index a85084c74..6d1c854ea 100644 --- a/lib/tasks/commit_log_to_db.rake +++ b/lib/tasks/commit_log_to_db.rake @@ -27,10 +27,11 @@ namespace :commit_log_to_db do ref = "master" commit_message = commit['commit']['message'].to_s.gsub("/n","") user = User.find_by(mail: commiter['email']) + user_id = user&.id || 1 commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") - data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," + data += "(#{user_id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," end data = data[0,data.length-1] sql_connection = ActiveRecord::Base.connection @@ -61,10 +62,11 @@ namespace :commit_log_to_db do ref = "master" commit_message = commit['commit']['message'].to_s.gsub("/n","") user = User.find_by(mail: commiter['email']) + user_id = user&.id || 1 commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%Y-%m-%d %H:%M:%S") - data += "(#{user&.id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," + data += "(#{user_id},#{project.id},#{project.repository&.id},'#{project.identifier}','#{project.owner.name}/#{project.identifier}','#{commit_sha}','#{ref}',\"#{commit_message}\",'#{commit_date_str}','#{commit_date_str}')," end data = data[0,data.length-1] sql_connection = ActiveRecord::Base.connection From cb848c5b45e40c50cda961df29a7834cdc92840b Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:21:12 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E4=BF=AE=E6=94=B9commit=5Flog=20message?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/migrate/20230417141788_update_commit_log_utf8.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 db/migrate/20230417141788_update_commit_log_utf8.rb diff --git a/db/migrate/20230417141788_update_commit_log_utf8.rb b/db/migrate/20230417141788_update_commit_log_utf8.rb new file mode 100644 index 000000000..8e3af9528 --- /dev/null +++ b/db/migrate/20230417141788_update_commit_log_utf8.rb @@ -0,0 +1,5 @@ +class UpdateCommitLogUtf8 < ActiveRecord::Migration[5.2] + def change + execute("ALTER TABLE `commit_logs` MODIFY `message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;") + end +end From aaf3b381fbfaba53d13176ca3b156a86afd3ebc1 Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:22:15 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E4=BF=AE=E6=94=B9commit=5Flog=20message?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ..._log_utf8.rb => 20230417141788_update_commit_log_message.rb} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename db/migrate/{20230417141788_update_commit_log_utf8.rb => 20230417141788_update_commit_log_message.rb} (69%) diff --git a/db/migrate/20230417141788_update_commit_log_utf8.rb b/db/migrate/20230417141788_update_commit_log_message.rb similarity index 69% rename from db/migrate/20230417141788_update_commit_log_utf8.rb rename to db/migrate/20230417141788_update_commit_log_message.rb index 8e3af9528..ee5d1f1c4 100644 --- a/db/migrate/20230417141788_update_commit_log_utf8.rb +++ b/db/migrate/20230417141788_update_commit_log_message.rb @@ -1,4 +1,4 @@ -class UpdateCommitLogUtf8 < ActiveRecord::Migration[5.2] +class UpdateCommitLogMessage < ActiveRecord::Migration[5.2] def change execute("ALTER TABLE `commit_logs` MODIFY `message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;") end From f370e1906e35f2356757f73be5365e4e577d9cc5 Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 17 Apr 2023 14:28:58 +0800 Subject: [PATCH 8/8] =?UTF-8?q?rake=20commit=5Flog=5Fto=5Fdb=20=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=B8=8D=E5=AD=98=E5=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/commit_log_to_db.rake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tasks/commit_log_to_db.rake b/lib/tasks/commit_log_to_db.rake index 6d1c854ea..ae80f9424 100644 --- a/lib/tasks/commit_log_to_db.rake +++ b/lib/tasks/commit_log_to_db.rake @@ -25,9 +25,9 @@ namespace :commit_log_to_db do commit_author = "#{commiter['name']} <#{commiter['email']}>" commit_sha = commit['sha'] ref = "master" - commit_message = commit['commit']['message'].to_s.gsub("/n","") + commit_message = commit['commit']['message'].to_s.gsub("\"","") user = User.find_by(mail: commiter['email']) - user_id = user&.id || 1 + user_id = user&.id || project.user_id commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S") @@ -60,9 +60,9 @@ namespace :commit_log_to_db do commit_author = "#{commiter['name']} <#{commiter['email']}>" commit_sha = commit['sha'] ref = "master" - commit_message = commit['commit']['message'].to_s.gsub("/n","") + commit_message = commit['commit']['message'].to_s.gsub("/n","").gsub("\"","") user = User.find_by(mail: commiter['email']) - user_id = user&.id || 1 + user_id = user&.id || project.user_id commit_date = Time.parse(commit['commit']['author']['date']) commit_date_str = commit_date.strftime("%Y-%m-%d %H:%M:%S")