module Github class Import class MergeRequest < ::MergeRequest self.table_name = 'merge_requests' self.reset_callbacks :save self.reset_callbacks :commit self.reset_callbacks :update self.reset_callbacks :validate end class Issue < ::Issue self.table_name = 'issues' self.reset_callbacks :save self.reset_callbacks :commit self.reset_callbacks :update self.reset_callbacks :validate end class Note < ::Note self.table_name = 'notes' self.reset_callbacks :save self.reset_callbacks :commit self.reset_callbacks :update self.reset_callbacks :validate end attr_reader :project, :repository, :options, :cached_label_ids, :cached_gitlab_users, :cached_user_ids, :errors def initialize(project, options) @project = project @repository = project.repository @options = options @cached_label_ids = {} @cached_user_ids = {} @cached_gitlab_users = {} @errors = [] end def execute(owner, repo) # Fetch repository begin project.create_repository project.repository.add_remote('github', "https://{token}@github.com/#{owner}/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e error(:project, "https://github.com/#{owner}/#{repo}.git", e.message) end # Fetch labels url = "/repos/#{owner}/#{repo}/labels" loop do response = Github::Client.new(options).get(url) response.body.each do |raw| begin label = Github::Representation::Label.new(raw) # TODO: we should take group labels in account next if project.labels.where(title: label.title).exists? project.labels.create!(title: label.title, color: label.color) rescue => e error(:label, label.url, e.message) end end break unless url = response.rels[:next] end # Cache labels # TODO: we should take group labels in account project.labels.select(:id, :title).find_each do |label| @cached_label_ids[label.title] = label.id end # Fetch milestones url = "/repos/#{owner}/#{repo}/milestones" loop do response = Github::Client.new(options).get(url, state: :all) response.body.each do |raw| begin milestone = Github::Representation::Milestone.new(raw) next if project.milestones.where(iid: milestone.iid).exists? project.milestones.create!( iid: milestone.iid, title: milestone.title, description: milestone.description, due_date: milestone.due_date, state: milestone.state, created_at: milestone.created_at, updated_at: milestone.updated_at ) rescue => e error(:milestone, milestone.url, e.message) end end break unless url = response.rels[:next] end # Fetch pull requests url = "/repos/#{owner}/#{repo}/pulls" loop do response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| pull_request = Github::Representation::PullRequest.new(project, raw, options) merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) next unless merge_request.new_record? && pull_request.valid? begin restore_source_branch(pull_request) unless pull_request.source_branch_exists? restore_target_branch(pull_request) unless pull_request.target_branch_exists? author_id = user_id(pull_request.author, project.creator_id) merge_request.iid = pull_request.iid merge_request.title = pull_request.title merge_request.description = format_description(pull_request.description, pull_request.author) merge_request.source_project = pull_request.source_project merge_request.source_branch = pull_request.source_branch_name merge_request.source_branch_sha = pull_request.source_branch_sha merge_request.target_project = pull_request.target_project merge_request.target_branch = pull_request.target_branch_name merge_request.target_branch_sha = pull_request.target_branch_sha merge_request.state = pull_request.state merge_request.milestone_id = milestone_id(pull_request.milestone) merge_request.author_id = author_id merge_request.assignee_id = user_id(pull_request.assignee) merge_request.created_at = pull_request.created_at merge_request.updated_at = pull_request.updated_at merge_request.save!(validate: false) merge_request.merge_request_diffs.create # Fetch review comments review_comments_url = "/repos/#{owner}/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url) # Fetch comments comments_url = "/repos/#{owner}/#{repo}/issues/#{pull_request.iid}/comments" fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) ensure clean_up_restored_branches(pull_request) end end break unless url = response.rels[:next] end # Fetch issues url = "/repos/#{owner}/#{repo}/issues" loop do response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| representation = Github::Representation::Issue.new(raw, options) begin # Every pull request is an issue, but not every issue # is a pull request. For this reason, "shared" actions # for both features, like manipulating assignees, labels # and milestones, are provided within the Issues API. if representation.pull_request? next unless representation.has_labels? merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) merge_request.update_attribute(:label_ids, label_ids(representation.labels)) else next if Issue.where(iid: representation.iid, project_id: project.id).exists? author_id = user_id(representation.author, project.creator_id) issue = Issue.new issue.iid = representation.iid issue.project_id = project.id issue.title = representation.title issue.description = format_description(representation.description, representation.author) issue.state = representation.state issue.label_ids = label_ids(representation.labels) issue.milestone_id = milestone_id(representation.milestone) issue.author_id = author_id issue.assignee_id = user_id(representation.assignee) issue.created_at = representation.created_at issue.updated_at = representation.updated_at issue.save!(validate: false) if representation.has_comments? # Fetch comments comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" fetch_comments(issue, :comment, comments_url) end end rescue => e error(:issue, representation.url, e.message) end end break unless url = response.rels[:next] end repository.expire_content_cache errors end private def fetch_comments(noteable, type, url) loop do comments = Github::Client.new(options).get(url) ActiveRecord::Base.no_touching do comments.body.each do |raw| begin representation = Github::Representation::Comment.new(raw, options) author_id = user_id(representation.author, project.creator_id) note = Note.new note.project_id = project.id note.noteable = noteable note.note = format_description(representation.note, representation.author) note.commit_id = representation.commit_id note.line_code = representation.line_code note.author_id = author_id note.type = representation.type note.created_at = representation.created_at note.updated_at = representation.updated_at note.save!(validate: false) rescue => e error(type, representation.url, e.message) end end end break unless url = comments.rels[:next] end end def restore_source_branch(pull_request) repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end def restore_target_branch(pull_request) repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) end def remove_branch(name) repository.delete_branch(name) rescue Rugged::ReferenceError errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } end def clean_up_restored_branches(pull_request) return if pull_request.opened? remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end def label_ids(issuable) issuable.map { |attrs| cached_label_ids[attrs.fetch('name')] }.compact end def milestone_id(milestone) return unless milestone.present? project.milestones.select(:id).find_by(iid: milestone.iid)&.id end def user_id(user, fallback_id = nil) return unless user.present? return cached_user_ids[user.id] if cached_user_ids.key?(user.id) gitlab_user_id = find_by_external_uid(user.id) || find_by_email(user.email) cached_gitlab_users[user.id] = gitlab_user_id.present? cached_user_ids[user.id] = gitlab_user_id || fallback_id end def find_by_email(email) return nil unless email ::User.find_by_any_email(email)&.id end def find_by_external_uid(id) return nil unless id identities = ::Identity.arel_table ::User.select(:id) .joins(:identities) .where(identities[:provider].eq(:github).and(identities[:extern_uid].eq(id))) .first&.id end def format_description(body, author) return body if cached_gitlab_users[author.id] "*Created by: #{author.username}*\n\n#{body}" end def error(type, url, message) errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } end end end