Merge branch 'fix/github-importer' into 'master'
Refactoring rake task to import GitHub repositories See merge request !10695
This commit is contained in:
Sean McGivern
commit
d255425b8c
22 changed files with 930 additions and 101 deletions
6
Gemfile
6
Gemfile
|
|
@ -17,6 +17,8 @@ gem 'pg', '~> 0.18.2', group: :postgres
|
|||
|
||||
gem 'rugged', '~> 0.25.1.1'
|
||||
|
||||
gem 'faraday', '~> 0.11.0'
|
||||
|
||||
# Authentication libraries
|
||||
gem 'devise', '~> 4.2'
|
||||
gem 'doorkeeper', '~> 4.2.0'
|
||||
|
|
@ -186,7 +188,7 @@ gem 'gemnasium-gitlab-service', '~> 0.2'
|
|||
gem 'slack-notifier', '~> 1.5.1'
|
||||
|
||||
# Asana integration
|
||||
gem 'asana', '~> 0.4.0'
|
||||
gem 'asana', '~> 0.6.0'
|
||||
|
||||
# FogBugz integration
|
||||
gem 'ruby-fogbugz', '~> 0.2.1'
|
||||
|
|
@ -345,7 +347,7 @@ gem 'html2text'
|
|||
gem 'ruby-prof', '~> 0.16.2'
|
||||
|
||||
# OAuth
|
||||
gem 'oauth2', '~> 1.2.0'
|
||||
gem 'oauth2', '~> 1.3.0'
|
||||
|
||||
# Soft deletion
|
||||
gem 'paranoia', '~> 2.2'
|
||||
|
|
|
|||
19
Gemfile.lock
19
Gemfile.lock
|
|
@ -47,7 +47,7 @@ GEM
|
|||
akismet (2.0.0)
|
||||
allocations (1.0.5)
|
||||
arel (6.0.4)
|
||||
asana (0.4.0)
|
||||
asana (0.6.0)
|
||||
faraday (~> 0.9)
|
||||
faraday_middleware (~> 0.9)
|
||||
faraday_middleware-multi_json (~> 0.0)
|
||||
|
|
@ -193,10 +193,10 @@ GEM
|
|||
factory_girl_rails (4.7.0)
|
||||
factory_girl (~> 4.7.0)
|
||||
railties (>= 3.0.0)
|
||||
faraday (0.9.2)
|
||||
faraday (0.11.0)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
faraday_middleware (0.10.0)
|
||||
faraday (>= 0.7.4, < 0.10)
|
||||
faraday_middleware (0.11.0.1)
|
||||
faraday (>= 0.7.4, < 1.0)
|
||||
faraday_middleware-multi_json (0.0.6)
|
||||
faraday_middleware
|
||||
multi_json
|
||||
|
|
@ -454,15 +454,15 @@ GEM
|
|||
mini_portile2 (~> 2.1.0)
|
||||
numerizer (0.1.1)
|
||||
oauth (0.5.1)
|
||||
oauth2 (1.2.0)
|
||||
faraday (>= 0.8, < 0.10)
|
||||
oauth2 (1.3.1)
|
||||
faraday (>= 0.8, < 0.12)
|
||||
jwt (~> 1.0)
|
||||
multi_json (~> 1.3)
|
||||
multi_xml (~> 0.5)
|
||||
rack (>= 1.2, < 3)
|
||||
octokit (4.6.2)
|
||||
sawyer (~> 0.8.0, >= 0.5.3)
|
||||
oj (2.17.4)
|
||||
oj (2.17.5)
|
||||
omniauth (1.4.2)
|
||||
hashie (>= 1.2, < 4)
|
||||
rack (>= 1.0, < 3)
|
||||
|
|
@ -853,7 +853,7 @@ DEPENDENCIES
|
|||
after_commit_queue (~> 1.3.0)
|
||||
akismet (~> 2.0)
|
||||
allocations (~> 1.0)
|
||||
asana (~> 0.4.0)
|
||||
asana (~> 0.6.0)
|
||||
asciidoctor (~> 1.5.2)
|
||||
asciidoctor-plantuml (= 0.0.7)
|
||||
attr_encrypted (~> 3.0.0)
|
||||
|
|
@ -891,6 +891,7 @@ DEPENDENCIES
|
|||
email_reply_trimmer (~> 0.1)
|
||||
email_spec (~> 1.6.0)
|
||||
factory_girl_rails (~> 4.7.0)
|
||||
faraday (~> 0.11.0)
|
||||
ffaker (~> 2.4)
|
||||
flay (~> 2.8.0)
|
||||
fog-aws (~> 0.9)
|
||||
|
|
@ -943,7 +944,7 @@ DEPENDENCIES
|
|||
mysql2 (~> 0.3.16)
|
||||
net-ssh (~> 3.0.1)
|
||||
nokogiri (~> 1.6.7, >= 1.6.7.2)
|
||||
oauth2 (~> 1.2.0)
|
||||
oauth2 (~> 1.3.0)
|
||||
octokit (~> 4.6.2)
|
||||
oj (~> 2.17.4)
|
||||
omniauth (~> 1.4.2)
|
||||
|
|
|
|||
23
lib/github/client.rb
Normal file
23
lib/github/client.rb
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
module Github
|
||||
class Client
|
||||
attr_reader :connection, :rate_limit
|
||||
|
||||
def initialize(options)
|
||||
@connection = Faraday.new(url: options.fetch(:url)) do |faraday|
|
||||
faraday.options.open_timeout = options.fetch(:timeout, 60)
|
||||
faraday.options.timeout = options.fetch(:timeout, 60)
|
||||
faraday.authorization 'token', options.fetch(:token)
|
||||
faraday.adapter :net_http
|
||||
end
|
||||
|
||||
@rate_limit = RateLimit.new(connection)
|
||||
end
|
||||
|
||||
def get(url, query = {})
|
||||
exceed, reset_in = rate_limit.get
|
||||
sleep reset_in if exceed
|
||||
|
||||
Github::Response.new(connection.get(url, query))
|
||||
end
|
||||
end
|
||||
end
|
||||
29
lib/github/collection.rb
Normal file
29
lib/github/collection.rb
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
module Github
|
||||
class Collection
|
||||
attr_reader :options
|
||||
|
||||
def initialize(options)
|
||||
@options = options
|
||||
end
|
||||
|
||||
def fetch(url, query = {})
|
||||
return [] if url.blank?
|
||||
|
||||
Enumerator.new do |yielder|
|
||||
loop do
|
||||
response = client.get(url, query)
|
||||
response.body.each { |item| yielder << item }
|
||||
|
||||
raise StopIteration unless response.rels.key?(:next)
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end.lazy
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def client
|
||||
@client ||= Github::Client.new(options)
|
||||
end
|
||||
end
|
||||
end
|
||||
3
lib/github/error.rb
Normal file
3
lib/github/error.rb
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module Github
|
||||
RepositoryFetchError = Class.new(StandardError)
|
||||
end
|
||||
405
lib/github/import.rb
Normal file
405
lib/github/import.rb
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
require_relative 'error'
|
||||
module Github
|
||||
class Import
|
||||
include Gitlab::ShellAdapter
|
||||
|
||||
class MergeRequest < ::MergeRequest
|
||||
self.table_name = 'merge_requests'
|
||||
|
||||
self.reset_callbacks :save
|
||||
self.reset_callbacks :commit
|
||||
self.reset_callbacks :update
|
||||
self.reset_callbacks :validate
|
||||
end
|
||||
|
||||
class Issue < ::Issue
|
||||
self.table_name = 'issues'
|
||||
|
||||
self.reset_callbacks :save
|
||||
self.reset_callbacks :commit
|
||||
self.reset_callbacks :update
|
||||
self.reset_callbacks :validate
|
||||
end
|
||||
|
||||
class Note < ::Note
|
||||
self.table_name = 'notes'
|
||||
|
||||
self.reset_callbacks :save
|
||||
self.reset_callbacks :commit
|
||||
self.reset_callbacks :update
|
||||
self.reset_callbacks :validate
|
||||
end
|
||||
|
||||
class LegacyDiffNote < ::LegacyDiffNote
|
||||
self.table_name = 'notes'
|
||||
|
||||
self.reset_callbacks :commit
|
||||
self.reset_callbacks :update
|
||||
self.reset_callbacks :validate
|
||||
end
|
||||
|
||||
attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose
|
||||
|
||||
def initialize(project, options)
|
||||
@project = project
|
||||
@repository = project.repository
|
||||
@repo = project.import_source
|
||||
@options = options
|
||||
@verbose = options.fetch(:verbose, false)
|
||||
@cached = Hash.new { |hash, key| hash[key] = Hash.new }
|
||||
@errors = []
|
||||
end
|
||||
|
||||
# rubocop: disable Rails/Output
|
||||
def execute
|
||||
puts 'Fetching repository...'.color(:aqua) if verbose
|
||||
fetch_repository
|
||||
puts 'Fetching labels...'.color(:aqua) if verbose
|
||||
fetch_labels
|
||||
puts 'Fetching milestones...'.color(:aqua) if verbose
|
||||
fetch_milestones
|
||||
puts 'Fetching pull requests...'.color(:aqua) if verbose
|
||||
fetch_pull_requests
|
||||
puts 'Fetching issues...'.color(:aqua) if verbose
|
||||
fetch_issues
|
||||
puts 'Cloning wiki repository...'.color(:aqua) if verbose
|
||||
fetch_wiki_repository
|
||||
puts 'Expiring repository cache...'.color(:aqua) if verbose
|
||||
expire_repository_cache
|
||||
|
||||
true
|
||||
rescue Github::RepositoryFetchError
|
||||
false
|
||||
ensure
|
||||
keep_track_of_errors
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fetch_repository
|
||||
begin
|
||||
project.create_repository unless project.repository.exists?
|
||||
project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git")
|
||||
project.repository.set_remote_as_mirror('github')
|
||||
project.repository.fetch_remote('github', forced: true)
|
||||
rescue Gitlab::Shell::Error => e
|
||||
error(:project, "https://github.com/#{repo}.git", e.message)
|
||||
raise Github::RepositoryFetchError
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_wiki_repository
|
||||
wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git"
|
||||
wiki_path = "#{project.path_with_namespace}.wiki"
|
||||
|
||||
unless project.wiki.repository_exists?
|
||||
gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
|
||||
end
|
||||
rescue Gitlab::Shell::Error => e
|
||||
# GitHub error message when the wiki repo has not been created,
|
||||
# this means that repo has wiki enabled, but have no pages. So,
|
||||
# we can skip the import.
|
||||
if e.message !~ /repository not exported/
|
||||
errors(:wiki, wiki_url, e.message)
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_labels
|
||||
url = "/repos/#{repo}/labels"
|
||||
|
||||
while url
|
||||
response = Github::Client.new(options).get(url)
|
||||
|
||||
response.body.each do |raw|
|
||||
begin
|
||||
representation = Github::Representation::Label.new(raw)
|
||||
|
||||
label = project.labels.find_or_create_by!(title: representation.title) do |label|
|
||||
label.color = representation.color
|
||||
end
|
||||
|
||||
cached[:label_ids][label.title] = label.id
|
||||
rescue => e
|
||||
error(:label, representation.url, e.message)
|
||||
end
|
||||
end
|
||||
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_milestones
|
||||
url = "/repos/#{repo}/milestones"
|
||||
|
||||
while url
|
||||
response = Github::Client.new(options).get(url, state: :all)
|
||||
|
||||
response.body.each do |raw|
|
||||
begin
|
||||
milestone = Github::Representation::Milestone.new(raw)
|
||||
next if project.milestones.where(iid: milestone.iid).exists?
|
||||
|
||||
project.milestones.create!(
|
||||
iid: milestone.iid,
|
||||
title: milestone.title,
|
||||
description: milestone.description,
|
||||
due_date: milestone.due_date,
|
||||
state: milestone.state,
|
||||
created_at: milestone.created_at,
|
||||
updated_at: milestone.updated_at
|
||||
)
|
||||
rescue => e
|
||||
error(:milestone, milestone.url, e.message)
|
||||
end
|
||||
end
|
||||
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_pull_requests
|
||||
url = "/repos/#{repo}/pulls"
|
||||
|
||||
while url
|
||||
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
|
||||
|
||||
response.body.each do |raw|
|
||||
pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project))
|
||||
merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id)
|
||||
next unless merge_request.new_record? && pull_request.valid?
|
||||
|
||||
begin
|
||||
restore_branches(pull_request)
|
||||
|
||||
author_id = user_id(pull_request.author, project.creator_id)
|
||||
merge_request.iid = pull_request.iid
|
||||
merge_request.title = pull_request.title
|
||||
merge_request.description = format_description(pull_request.description, pull_request.author)
|
||||
merge_request.source_project = pull_request.source_project
|
||||
merge_request.source_branch = pull_request.source_branch_name
|
||||
merge_request.source_branch_sha = pull_request.source_branch_sha
|
||||
merge_request.target_project = pull_request.target_project
|
||||
merge_request.target_branch = pull_request.target_branch_name
|
||||
merge_request.target_branch_sha = pull_request.target_branch_sha
|
||||
merge_request.state = pull_request.state
|
||||
merge_request.milestone_id = milestone_id(pull_request.milestone)
|
||||
merge_request.author_id = author_id
|
||||
merge_request.assignee_id = user_id(pull_request.assignee)
|
||||
merge_request.created_at = pull_request.created_at
|
||||
merge_request.updated_at = pull_request.updated_at
|
||||
merge_request.save!(validate: false)
|
||||
|
||||
merge_request.merge_request_diffs.create
|
||||
|
||||
# Fetch review comments
|
||||
review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments"
|
||||
fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote)
|
||||
|
||||
# Fetch comments
|
||||
comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments"
|
||||
fetch_comments(merge_request, :comment, comments_url)
|
||||
rescue => e
|
||||
error(:pull_request, pull_request.url, e.message)
|
||||
ensure
|
||||
clean_up_restored_branches(pull_request)
|
||||
end
|
||||
end
|
||||
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_issues
|
||||
url = "/repos/#{repo}/issues"
|
||||
|
||||
while url
|
||||
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
|
||||
|
||||
response.body.each do |raw|
|
||||
representation = Github::Representation::Issue.new(raw, options)
|
||||
|
||||
begin
|
||||
# Every pull request is an issue, but not every issue
|
||||
# is a pull request. For this reason, "shared" actions
|
||||
# for both features, like manipulating assignees, labels
|
||||
# and milestones, are provided within the Issues API.
|
||||
if representation.pull_request?
|
||||
next unless representation.has_labels?
|
||||
|
||||
merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid)
|
||||
merge_request.update_attribute(:label_ids, label_ids(representation.labels))
|
||||
else
|
||||
next if Issue.where(iid: representation.iid, project_id: project.id).exists?
|
||||
|
||||
author_id = user_id(representation.author, project.creator_id)
|
||||
issue = Issue.new
|
||||
issue.iid = representation.iid
|
||||
issue.project_id = project.id
|
||||
issue.title = representation.title
|
||||
issue.description = format_description(representation.description, representation.author)
|
||||
issue.state = representation.state
|
||||
issue.label_ids = label_ids(representation.labels)
|
||||
issue.milestone_id = milestone_id(representation.milestone)
|
||||
issue.author_id = author_id
|
||||
issue.assignee_id = user_id(representation.assignee)
|
||||
issue.created_at = representation.created_at
|
||||
issue.updated_at = representation.updated_at
|
||||
issue.save!(validate: false)
|
||||
|
||||
# Fetch comments
|
||||
if representation.has_comments?
|
||||
comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments"
|
||||
fetch_comments(issue, :comment, comments_url)
|
||||
end
|
||||
end
|
||||
rescue => e
|
||||
error(:issue, representation.url, e.message)
|
||||
end
|
||||
end
|
||||
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_comments(noteable, type, url, klass = Note)
|
||||
while url
|
||||
comments = Github::Client.new(options).get(url)
|
||||
|
||||
ActiveRecord::Base.no_touching do
|
||||
comments.body.each do |raw|
|
||||
begin
|
||||
representation = Github::Representation::Comment.new(raw, options)
|
||||
author_id = user_id(representation.author, project.creator_id)
|
||||
|
||||
note = klass.new
|
||||
note.project_id = project.id
|
||||
note.noteable = noteable
|
||||
note.note = format_description(representation.note, representation.author)
|
||||
note.commit_id = representation.commit_id
|
||||
note.line_code = representation.line_code
|
||||
note.author_id = author_id
|
||||
note.created_at = representation.created_at
|
||||
note.updated_at = representation.updated_at
|
||||
note.save!(validate: false)
|
||||
rescue => e
|
||||
error(type, representation.url, e.message)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
url = comments.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_releases
|
||||
url = "/repos/#{repo}/releases"
|
||||
|
||||
while url
|
||||
response = Github::Client.new(options).get(url)
|
||||
|
||||
response.body.each do |raw|
|
||||
representation = Github::Representation::Release.new(raw)
|
||||
next unless representation.valid?
|
||||
|
||||
release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
|
||||
next unless relese.new_record?
|
||||
|
||||
begin
|
||||
release.description = representation.description
|
||||
release.created_at = representation.created_at
|
||||
release.updated_at = representation.updated_at
|
||||
release.save!(validate: false)
|
||||
rescue => e
|
||||
error(:release, representation.url, e.message)
|
||||
end
|
||||
end
|
||||
|
||||
url = response.rels[:next]
|
||||
end
|
||||
end
|
||||
|
||||
def restore_branches(pull_request)
|
||||
restore_source_branch(pull_request) unless pull_request.source_branch_exists?
|
||||
restore_target_branch(pull_request) unless pull_request.target_branch_exists?
|
||||
end
|
||||
|
||||
def restore_source_branch(pull_request)
|
||||
repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha)
|
||||
end
|
||||
|
||||
def restore_target_branch(pull_request)
|
||||
repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha)
|
||||
end
|
||||
|
||||
def remove_branch(name)
|
||||
repository.delete_branch(name)
|
||||
rescue Rugged::ReferenceError
|
||||
errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" }
|
||||
end
|
||||
|
||||
def clean_up_restored_branches(pull_request)
|
||||
return if pull_request.opened?
|
||||
|
||||
remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists?
|
||||
remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists?
|
||||
end
|
||||
|
||||
def label_ids(labels)
|
||||
labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact
|
||||
end
|
||||
|
||||
def milestone_id(milestone)
|
||||
return unless milestone.present?
|
||||
|
||||
project.milestones.select(:id).find_by(iid: milestone.iid)&.id
|
||||
end
|
||||
|
||||
def user_id(user, fallback_id = nil)
|
||||
return unless user.present?
|
||||
return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id)
|
||||
|
||||
gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
|
||||
|
||||
cached[:gitlab_user_ids][user.id] = gitlab_user_id.present?
|
||||
cached[:user_ids][user.id] = gitlab_user_id || fallback_id
|
||||
end
|
||||
|
||||
def user_id_by_email(email)
|
||||
return nil unless email
|
||||
|
||||
::User.find_by_any_email(email)&.id
|
||||
end
|
||||
|
||||
def user_id_by_external_uid(id)
|
||||
return nil unless id
|
||||
|
||||
::User.select(:id)
|
||||
.joins(:identities)
|
||||
.merge(::Identity.where(provider: :github, extern_uid: id))
|
||||
.first&.id
|
||||
end
|
||||
|
||||
def format_description(body, author)
|
||||
return body if cached[:gitlab_user_ids][author.id]
|
||||
|
||||
"*Created by: #{author.username}*\n\n#{body}"
|
||||
end
|
||||
|
||||
def expire_repository_cache
|
||||
repository.expire_content_cache
|
||||
end
|
||||
|
||||
def keep_track_of_errors
|
||||
return unless errors.any?
|
||||
|
||||
project.update_column(:import_error, {
|
||||
message: 'The remote data could not be fully imported.',
|
||||
errors: errors
|
||||
}.to_json)
|
||||
end
|
||||
|
||||
def error(type, url, message)
|
||||
errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message }
|
||||
end
|
||||
end
|
||||
end
|
||||
27
lib/github/rate_limit.rb
Normal file
27
lib/github/rate_limit.rb
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
module Github
|
||||
class RateLimit
|
||||
SAFE_REMAINING_REQUESTS = 100
|
||||
SAFE_RESET_TIME = 500
|
||||
RATE_LIMIT_URL = '/rate_limit'.freeze
|
||||
|
||||
attr_reader :connection
|
||||
|
||||
def initialize(connection)
|
||||
@connection = connection
|
||||
end
|
||||
|
||||
def get
|
||||
response = connection.get(RATE_LIMIT_URL)
|
||||
|
||||
# GitHub Rate Limit API returns 404 when the rate limit is disabled
|
||||
return false unless response.status != 404
|
||||
|
||||
body = Oj.load(response.body, class_cache: false, mode: :compat)
|
||||
remaining = body.dig('rate', 'remaining').to_i
|
||||
reset_in = body.dig('rate', 'reset').to_i
|
||||
exceed = remaining <= SAFE_REMAINING_REQUESTS
|
||||
|
||||
[exceed, reset_in]
|
||||
end
|
||||
end
|
||||
end
|
||||
19
lib/github/repositories.rb
Normal file
19
lib/github/repositories.rb
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
module Github
|
||||
class Repositories
|
||||
attr_reader :options
|
||||
|
||||
def initialize(options)
|
||||
@options = options
|
||||
end
|
||||
|
||||
def fetch
|
||||
Collection.new(options).fetch(repos_url)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def repos_url
|
||||
'/user/repos'
|
||||
end
|
||||
end
|
||||
end
|
||||
30
lib/github/representation/base.rb
Normal file
30
lib/github/representation/base.rb
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Base
|
||||
def initialize(raw, options = {})
|
||||
@raw = raw
|
||||
@options = options
|
||||
end
|
||||
|
||||
def id
|
||||
raw['id']
|
||||
end
|
||||
|
||||
def url
|
||||
raw['url']
|
||||
end
|
||||
|
||||
def created_at
|
||||
raw['created_at']
|
||||
end
|
||||
|
||||
def updated_at
|
||||
raw['updated_at']
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :raw, :options
|
||||
end
|
||||
end
|
||||
end
|
||||
51
lib/github/representation/branch.rb
Normal file
51
lib/github/representation/branch.rb
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Branch < Representation::Base
|
||||
attr_reader :repository
|
||||
|
||||
def user
|
||||
raw.dig('user', 'login') || 'unknown'
|
||||
end
|
||||
|
||||
def repo
|
||||
return @repo if defined?(@repo)
|
||||
|
||||
@repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present?
|
||||
end
|
||||
|
||||
def ref
|
||||
raw['ref']
|
||||
end
|
||||
|
||||
def sha
|
||||
raw['sha']
|
||||
end
|
||||
|
||||
def short_sha
|
||||
Commit.truncate_sha(sha)
|
||||
end
|
||||
|
||||
def exists?
|
||||
branch_exists? && commit_exists?
|
||||
end
|
||||
|
||||
def valid?
|
||||
sha.present? && ref.present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def branch_exists?
|
||||
repository.branch_exists?(ref)
|
||||
end
|
||||
|
||||
def commit_exists?
|
||||
repository.branch_names_contains(sha).include?(ref)
|
||||
end
|
||||
|
||||
def repository
|
||||
@repository ||= options.fetch(:repository)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
42
lib/github/representation/comment.rb
Normal file
42
lib/github/representation/comment.rb
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Comment < Representation::Base
|
||||
def note
|
||||
raw['body'] || ''
|
||||
end
|
||||
|
||||
def author
|
||||
@author ||= Github::Representation::User.new(raw['user'], options)
|
||||
end
|
||||
|
||||
def commit_id
|
||||
raw['commit_id']
|
||||
end
|
||||
|
||||
def line_code
|
||||
return unless on_diff?
|
||||
|
||||
parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines)
|
||||
generate_line_code(parsed_lines.to_a.last)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_line_code(line)
|
||||
Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos)
|
||||
end
|
||||
|
||||
def on_diff?
|
||||
diff_hunk.present?
|
||||
end
|
||||
|
||||
def diff_hunk
|
||||
raw['diff_hunk']
|
||||
end
|
||||
|
||||
def file_path
|
||||
raw['path']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
37
lib/github/representation/issuable.rb
Normal file
37
lib/github/representation/issuable.rb
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Issuable < Representation::Base
|
||||
def iid
|
||||
raw['number']
|
||||
end
|
||||
|
||||
def title
|
||||
raw['title']
|
||||
end
|
||||
|
||||
def description
|
||||
raw['body'] || ''
|
||||
end
|
||||
|
||||
def milestone
|
||||
return unless raw['milestone'].present?
|
||||
|
||||
@milestone ||= Github::Representation::Milestone.new(raw['milestone'])
|
||||
end
|
||||
|
||||
def author
|
||||
@author ||= Github::Representation::User.new(raw['user'], options)
|
||||
end
|
||||
|
||||
def assignee
|
||||
return unless assigned?
|
||||
|
||||
@assignee ||= Github::Representation::User.new(raw['assignee'], options)
|
||||
end
|
||||
|
||||
def assigned?
|
||||
raw['assignee'].present?
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/github/representation/issue.rb
Normal file
25
lib/github/representation/issue.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Issue < Representation::Issuable
|
||||
def labels
|
||||
raw['labels']
|
||||
end
|
||||
|
||||
def state
|
||||
raw['state'] == 'closed' ? 'closed' : 'opened'
|
||||
end
|
||||
|
||||
def has_comments?
|
||||
raw['comments'] > 0
|
||||
end
|
||||
|
||||
def has_labels?
|
||||
labels.count > 0
|
||||
end
|
||||
|
||||
def pull_request?
|
||||
raw['pull_request'].present?
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
13
lib/github/representation/label.rb
Normal file
13
lib/github/representation/label.rb
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Label < Representation::Base
|
||||
def color
|
||||
"##{raw['color']}"
|
||||
end
|
||||
|
||||
def title
|
||||
raw['name']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/github/representation/milestone.rb
Normal file
25
lib/github/representation/milestone.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Milestone < Representation::Base
|
||||
def iid
|
||||
raw['number']
|
||||
end
|
||||
|
||||
def title
|
||||
raw['title']
|
||||
end
|
||||
|
||||
def description
|
||||
raw['description']
|
||||
end
|
||||
|
||||
def due_date
|
||||
raw['due_on']
|
||||
end
|
||||
|
||||
def state
|
||||
raw['state'] == 'closed' ? 'closed' : 'active'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
78
lib/github/representation/pull_request.rb
Normal file
78
lib/github/representation/pull_request.rb
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
module Github
|
||||
module Representation
|
||||
class PullRequest < Representation::Issuable
|
||||
attr_reader :project
|
||||
|
||||
delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true
|
||||
delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true
|
||||
|
||||
def source_project
|
||||
project
|
||||
end
|
||||
|
||||
def source_branch_exists?
|
||||
!cross_project? && source_branch.exists?
|
||||
end
|
||||
|
||||
def source_branch_name
|
||||
@source_branch_name ||=
|
||||
if cross_project? || !source_branch_exists?
|
||||
source_branch_name_prefixed
|
||||
else
|
||||
source_branch_ref
|
||||
end
|
||||
end
|
||||
|
||||
def target_project
|
||||
project
|
||||
end
|
||||
|
||||
def target_branch_name
|
||||
@target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed
|
||||
end
|
||||
|
||||
def state
|
||||
return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present?
|
||||
return 'closed' if raw['state'] == 'closed'
|
||||
|
||||
'opened'
|
||||
end
|
||||
|
||||
def opened?
|
||||
state == 'opened'
|
||||
end
|
||||
|
||||
def valid?
|
||||
source_branch.valid? && target_branch.valid?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def project
|
||||
@project ||= options.fetch(:project)
|
||||
end
|
||||
|
||||
def source_branch
|
||||
@source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository)
|
||||
end
|
||||
|
||||
def source_branch_name_prefixed
|
||||
"gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}"
|
||||
end
|
||||
|
||||
def target_branch
|
||||
@target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository)
|
||||
end
|
||||
|
||||
def target_branch_name_prefixed
|
||||
"gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}"
|
||||
end
|
||||
|
||||
def cross_project?
|
||||
return true if source_branch_repo.nil?
|
||||
|
||||
source_branch_repo.id != target_branch_repo.id
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
17
lib/github/representation/release.rb
Normal file
17
lib/github/representation/release.rb
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Release < Representation::Base
|
||||
def description
|
||||
raw['body']
|
||||
end
|
||||
|
||||
def tag
|
||||
raw['tag_name']
|
||||
end
|
||||
|
||||
def valid?
|
||||
!raw['draft']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
6
lib/github/representation/repo.rb
Normal file
6
lib/github/representation/repo.rb
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
module Github
|
||||
module Representation
|
||||
class Repo < Representation::Base
|
||||
end
|
||||
end
|
||||
end
|
||||
15
lib/github/representation/user.rb
Normal file
15
lib/github/representation/user.rb
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
module Github
|
||||
module Representation
|
||||
class User < Representation::Base
|
||||
def email
|
||||
return @email if defined?(@email)
|
||||
|
||||
@email = Github::User.new(username, options).get.fetch('email', nil)
|
||||
end
|
||||
|
||||
def username
|
||||
raw['login']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/github/response.rb
Normal file
25
lib/github/response.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
module Github
|
||||
class Response
|
||||
attr_reader :raw, :headers, :status
|
||||
|
||||
def initialize(response)
|
||||
@raw = response
|
||||
@headers = response.headers
|
||||
@status = response.status
|
||||
end
|
||||
|
||||
def body
|
||||
Oj.load(raw.body, class_cache: false, mode: :compat)
|
||||
end
|
||||
|
||||
def rels
|
||||
links = headers['Link'].to_s.split(', ').map do |link|
|
||||
href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures
|
||||
|
||||
[name.to_sym, href]
|
||||
end
|
||||
|
||||
Hash[*links.flatten]
|
||||
end
|
||||
end
|
||||
end
|
||||
24
lib/github/user.rb
Normal file
24
lib/github/user.rb
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
module Github
|
||||
class User
|
||||
attr_reader :username, :options
|
||||
|
||||
def initialize(username, options)
|
||||
@username = username
|
||||
@options = options
|
||||
end
|
||||
|
||||
def get
|
||||
client.get(user_url).body
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def client
|
||||
@client ||= Github::Client.new(options)
|
||||
end
|
||||
|
||||
def user_url
|
||||
"/users/#{username}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,67 +1,5 @@
|
|||
require 'benchmark'
|
||||
require 'rainbow/ext/string'
|
||||
require_relative '../gitlab/shell_adapter'
|
||||
require_relative '../gitlab/github_import/importer'
|
||||
|
||||
class NewImporter < ::Gitlab::GithubImport::Importer
|
||||
def execute
|
||||
# Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress.
|
||||
puts 'Importing repository...'.color(:aqua)
|
||||
import_repository unless project.repository_exists?
|
||||
|
||||
puts 'Importing labels...'.color(:aqua)
|
||||
import_labels
|
||||
|
||||
puts 'Importing milestones...'.color(:aqua)
|
||||
import_milestones
|
||||
|
||||
puts 'Importing pull requests...'.color(:aqua)
|
||||
import_pull_requests
|
||||
|
||||
puts 'Importing issues...'.color(:aqua)
|
||||
import_issues
|
||||
|
||||
puts 'Importing issue comments...'.color(:aqua)
|
||||
import_comments(:issues)
|
||||
|
||||
puts 'Importing pull request comments...'.color(:aqua)
|
||||
import_comments(:pull_requests)
|
||||
|
||||
puts 'Importing wiki...'.color(:aqua)
|
||||
import_wiki
|
||||
|
||||
# Gitea doesn't have a Release API yet
|
||||
# See https://github.com/go-gitea/gitea/issues/330
|
||||
unless project.gitea_import?
|
||||
import_releases
|
||||
end
|
||||
|
||||
handle_errors
|
||||
|
||||
project.repository.after_import
|
||||
project.import_finish
|
||||
|
||||
true
|
||||
end
|
||||
|
||||
def import_repository
|
||||
begin
|
||||
raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
|
||||
|
||||
project.create_repository
|
||||
project.repository.add_remote(project.import_type, project.import_url)
|
||||
project.repository.set_remote_as_mirror(project.import_type)
|
||||
project.repository.fetch_remote(project.import_type, forced: true)
|
||||
rescue => e
|
||||
# Expire cache to prevent scenarios such as:
|
||||
# 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
|
||||
# 2. Retried import, repo is broken or not imported but +exists?+ still returns true
|
||||
project.repository.expire_content_cache if project.repository_exists?
|
||||
|
||||
raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class GithubImport
|
||||
def self.run!(*args)
|
||||
|
|
@ -69,14 +7,14 @@ class GithubImport
|
|||
end
|
||||
|
||||
def initialize(token, gitlab_username, project_path, extras)
|
||||
@token = token
|
||||
@options = { url: 'https://api.github.com', token: token, verbose: true }
|
||||
@project_path = project_path
|
||||
@current_user = User.find_by_username(gitlab_username)
|
||||
@github_repo = extras.empty? ? nil : extras.first
|
||||
end
|
||||
|
||||
def run!
|
||||
@repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one!
|
||||
@repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one!
|
||||
|
||||
raise 'No repo found!' unless @repo
|
||||
|
||||
|
|
@ -90,25 +28,24 @@ class GithubImport
|
|||
private
|
||||
|
||||
def show_warning!
|
||||
puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}"
|
||||
puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
|
||||
puts "Permission checks are ignored. Press any key to continue.".color(:red)
|
||||
|
||||
STDIN.getch
|
||||
|
||||
puts 'Starting the import...'.color(:green)
|
||||
puts 'Starting the import (this could take a while)'.color(:green)
|
||||
end
|
||||
|
||||
def import!
|
||||
import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@")
|
||||
@project.update(import_url: import_url)
|
||||
|
||||
@project.import_start
|
||||
|
||||
timings = Benchmark.measure do
|
||||
NewImporter.new(@project).execute
|
||||
Github::Import.new(@project, @options).execute
|
||||
end
|
||||
|
||||
puts "Import finished. Timings: #{timings}".color(:green)
|
||||
|
||||
@project.import_finish
|
||||
end
|
||||
|
||||
def new_project
|
||||
|
|
@ -116,17 +53,17 @@ class GithubImport
|
|||
namespace_path, _sep, name = @project_path.rpartition('/')
|
||||
namespace = find_or_create_namespace(namespace_path)
|
||||
|
||||
Project.create!(
|
||||
import_url: "https://#{@token}@github.com/#{@repo.full_name}.git",
|
||||
Projects::CreateService.new(
|
||||
@current_user,
|
||||
name: name,
|
||||
path: name,
|
||||
description: @repo.description,
|
||||
namespace: namespace,
|
||||
description: @repo['description'],
|
||||
namespace_id: namespace.id,
|
||||
visibility_level: visibility_level,
|
||||
import_type: 'github',
|
||||
import_source: @repo.full_name,
|
||||
creator: @current_user
|
||||
)
|
||||
import_source: @repo['full_name'],
|
||||
skip_wiki: @repo['has_wiki']
|
||||
).execute
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -134,7 +71,6 @@ class GithubImport
|
|||
return @current_user.namespace if names == @current_user.namespace_path
|
||||
return @current_user.namespace unless @current_user.can_create_group?
|
||||
|
||||
names = params[:target_namespace].presence || names
|
||||
full_path_namespace = Namespace.find_by_full_path(names)
|
||||
|
||||
return full_path_namespace if full_path_namespace
|
||||
|
|
@ -159,13 +95,13 @@ class GithubImport
|
|||
end
|
||||
|
||||
def visibility_level
|
||||
@repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
|
||||
@repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
|
||||
end
|
||||
end
|
||||
|
||||
class GithubRepos
|
||||
def initialize(token, current_user, github_repo)
|
||||
@token = token
|
||||
def initialize(options, current_user, github_repo)
|
||||
@options = options
|
||||
@current_user = current_user
|
||||
@github_repo = github_repo
|
||||
end
|
||||
|
|
@ -174,17 +110,17 @@ class GithubRepos
|
|||
return found_github_repo if @github_repo
|
||||
|
||||
repos.each do |repo|
|
||||
print "ID: #{repo[:id].to_s.bright} ".color(:green)
|
||||
puts "- Name: #{repo[:full_name]}".color(:green)
|
||||
print "ID: #{repo['id'].to_s.bright}".color(:green)
|
||||
print "\tName: #{repo['full_name']}\n".color(:green)
|
||||
end
|
||||
|
||||
print 'ID? '.bright
|
||||
|
||||
repos.find { |repo| repo[:id] == repo_id }
|
||||
repos.find { |repo| repo['id'] == repo_id }
|
||||
end
|
||||
|
||||
def found_github_repo
|
||||
repos.find { |repo| repo[:full_name] == @github_repo }
|
||||
repos.find { |repo| repo['full_name'] == @github_repo }
|
||||
end
|
||||
|
||||
def repo_id
|
||||
|
|
@ -192,11 +128,7 @@ class GithubRepos
|
|||
end
|
||||
|
||||
def repos
|
||||
@repos ||= client.repos
|
||||
end
|
||||
|
||||
def client
|
||||
@client ||= Gitlab::GithubImport::Client.new(@token, {})
|
||||
Github::Repositories.new(@options).fetch
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue