2018-06-25 16:13:59 -04:00
|
|
|
module Gitlab
|
|
|
|
module BitbucketServerImport
|
|
|
|
class Importer
|
|
|
|
include Gitlab::ShellAdapter
|
2018-06-27 17:25:09 -04:00
|
|
|
attr_reader :project, :project_key, :repository_slug, :client, :errors, :users
|
2018-06-25 16:13:59 -04:00
|
|
|
|
2018-07-03 19:37:17 -04:00
|
|
|
REMOTE_NAME = 'bitbucket_server'.freeze
|
2018-07-06 00:51:12 -04:00
|
|
|
BATCH_SIZE = 100
|
2018-07-03 19:37:17 -04:00
|
|
|
|
|
|
|
def self.imports_repository?
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.refmap
|
|
|
|
[:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']
|
|
|
|
end
|
|
|
|
|
2018-06-25 16:13:59 -04:00
|
|
|
def initialize(project)
|
|
|
|
@project = project
|
2018-06-26 01:40:11 -04:00
|
|
|
@project_key = project.import_data.data['project_key']
|
|
|
|
@repository_slug = project.import_data.data['repo_slug']
|
2018-06-25 16:13:59 -04:00
|
|
|
@client = BitbucketServer::Client.new(project.import_data.credentials)
|
|
|
|
@formatter = Gitlab::ImportFormatter.new
|
|
|
|
@errors = []
|
|
|
|
@users = {}
|
2018-07-03 19:37:17 -04:00
|
|
|
@temp_branches = []
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def execute
|
2018-07-03 19:37:17 -04:00
|
|
|
import_repository
|
2018-06-25 16:13:59 -04:00
|
|
|
import_pull_requests
|
2018-07-13 18:43:15 -04:00
|
|
|
delete_temp_branches
|
2018-06-25 16:13:59 -04:00
|
|
|
handle_errors
|
|
|
|
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
def handle_errors
|
|
|
|
return unless errors.any?
|
|
|
|
|
|
|
|
project.update_column(:import_error, {
|
|
|
|
message: 'The remote data could not be fully imported.',
|
|
|
|
errors: errors
|
|
|
|
}.to_json)
|
|
|
|
end
|
|
|
|
|
2018-07-17 16:53:48 -04:00
|
|
|
def gitlab_user_id(email)
|
2018-06-27 17:25:09 -04:00
|
|
|
find_user_id(email) || project.creator_id
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
|
2018-06-27 17:25:09 -04:00
|
|
|
def find_user_id(email)
|
|
|
|
return nil unless email
|
2018-06-25 16:13:59 -04:00
|
|
|
|
2018-06-27 17:25:09 -04:00
|
|
|
return users[email] if users.key?(email)
|
2018-06-25 16:13:59 -04:00
|
|
|
|
2018-07-16 01:07:39 -04:00
|
|
|
user = User.find_by_any_email(email)
|
|
|
|
users[email] = user&.id if user
|
|
|
|
|
|
|
|
user&.id
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def repo
|
2018-06-27 17:25:09 -04:00
|
|
|
@repo ||= client.repo(project_key, repository_slug)
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
|
2018-07-03 19:37:17 -04:00
|
|
|
def sha_exists?(sha)
|
|
|
|
project.repository.commit(sha)
|
|
|
|
end
|
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
def temp_branch_name(pull_request, suffix)
|
|
|
|
"gitlab/import/pull-request/#{pull_request.iid}/#{suffix}"
|
|
|
|
end
|
|
|
|
|
|
|
|
def restore_branches(pull_requests)
|
|
|
|
shas_to_restore = []
|
|
|
|
pull_requests.each do |pull_request|
|
|
|
|
shas_to_restore << {
|
|
|
|
temp_branch_name(pull_request, :from) => pull_request.source_branch_sha,
|
|
|
|
temp_branch_name(pull_request, :to) => pull_request.target_branch_sha
|
|
|
|
}
|
|
|
|
end
|
2018-07-03 19:37:17 -04:00
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
created_branches = restore_branch_shas(shas_to_restore)
|
|
|
|
@temp_branches << created_branches
|
|
|
|
import_repository unless created_branches.empty?
|
2018-07-03 19:37:17 -04:00
|
|
|
end
|
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
def restore_branch_shas(shas_to_restore)
|
|
|
|
branches_created = []
|
2018-07-03 19:37:17 -04:00
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
shas_to_restore.each_with_index do |shas, index|
|
|
|
|
shas.each do |branch_name, sha|
|
|
|
|
next if sha_exists?(sha)
|
2018-07-03 19:37:17 -04:00
|
|
|
|
2018-07-13 02:17:05 -04:00
|
|
|
begin
|
|
|
|
client.create_branch(project_key, repository_slug, branch_name, sha)
|
2018-07-06 00:51:12 -04:00
|
|
|
branches_created << branch_name
|
2018-07-13 02:17:05 -04:00
|
|
|
rescue BitbucketServer::Connection::ConnectionError => e
|
|
|
|
Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}")
|
2018-07-06 00:51:12 -04:00
|
|
|
end
|
2018-07-03 19:37:17 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
branches_created
|
2018-07-03 19:37:17 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def import_repository
|
|
|
|
project.ensure_repository
|
|
|
|
project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME)
|
|
|
|
rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
|
|
|
|
# Expire cache to prevent scenarios such as:
|
|
|
|
# 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
|
|
|
|
# 2. Retried import, repo is broken or not imported but +exists?+ still returns true
|
|
|
|
project.repository.expire_content_cache if project.repository_exists?
|
|
|
|
|
2018-07-06 00:11:29 -04:00
|
|
|
raise e.message
|
2018-07-03 19:37:17 -04:00
|
|
|
end
|
|
|
|
|
2018-07-06 00:51:12 -04:00
|
|
|
# Bitbucket Server keeps tracks of references for open pull requests in
|
|
|
|
# refs/heads/pull-requests, but closed and merged requests get moved
|
|
|
|
# into hidden internal refs under stash-refs/pull-requests. Unless the
|
|
|
|
# SHAs involved are at the tip of a branch or tag, there is no way to
|
|
|
|
# retrieve the server for those commits.
|
|
|
|
#
|
|
|
|
# To avoid losing history, we use the Bitbucket API to re-create the branch
|
|
|
|
# on the remote server. Then we have to issue a `git fetch` to download these
|
|
|
|
# branches.
|
2018-06-25 16:13:59 -04:00
|
|
|
def import_pull_requests
|
2018-07-06 00:51:12 -04:00
|
|
|
pull_requests = client.pull_requests(project_key, repository_slug).to_a
|
|
|
|
|
|
|
|
# Creating branches on the server and fetching the newly-created branches
|
|
|
|
# may take a number of network round-trips. Do this in batches so that we can
|
|
|
|
# avoid doing a git fetch for every new branch.
|
|
|
|
pull_requests.each_slice(BATCH_SIZE) do |batch|
|
|
|
|
restore_branches(batch)
|
|
|
|
|
|
|
|
batch.each do |pull_request|
|
|
|
|
begin
|
|
|
|
import_bitbucket_pull_request(pull_request)
|
|
|
|
rescue StandardError => e
|
|
|
|
errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw }
|
|
|
|
end
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-07-13 18:43:15 -04:00
|
|
|
def delete_temp_branches
|
|
|
|
@temp_branches.each do |branch_name|
|
|
|
|
begin
|
|
|
|
client.delete_branch(project_key, repository_slug, branch_name)
|
|
|
|
project.repository.delete_branch(branch_name)
|
|
|
|
rescue BitbucketServer::Connection::ConnectionError => e
|
|
|
|
@errors << { type: :delete_temp_branches, branch_name: branch_name, errors: e.message }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-07-06 00:11:29 -04:00
|
|
|
def import_bitbucket_pull_request(pull_request)
|
|
|
|
description = ''
|
|
|
|
description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email)
|
|
|
|
description += pull_request.description
|
|
|
|
|
|
|
|
source_branch_sha = pull_request.source_branch_sha
|
|
|
|
target_branch_sha = pull_request.target_branch_sha
|
|
|
|
source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha
|
|
|
|
target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha
|
2018-07-17 16:53:48 -04:00
|
|
|
author_id = gitlab_user_id(pull_request.author_email)
|
2018-07-15 18:36:18 -04:00
|
|
|
|
2018-07-06 00:11:29 -04:00
|
|
|
project.merge_requests.find_by(iid: pull_request.iid)&.destroy
|
|
|
|
|
|
|
|
attributes = {
|
|
|
|
iid: pull_request.iid,
|
|
|
|
title: pull_request.title,
|
|
|
|
description: description,
|
|
|
|
source_project: project,
|
|
|
|
source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name),
|
|
|
|
source_branch_sha: source_branch_sha,
|
|
|
|
target_project: project,
|
|
|
|
target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name),
|
|
|
|
target_branch_sha: target_branch_sha,
|
|
|
|
state: pull_request.state,
|
2018-07-16 01:07:39 -04:00
|
|
|
author_id: author_id,
|
2018-07-06 00:11:29 -04:00
|
|
|
assignee_id: nil,
|
|
|
|
created_at: pull_request.created_at,
|
|
|
|
updated_at: pull_request.updated_at
|
|
|
|
}
|
|
|
|
|
|
|
|
attributes[:merge_commit_sha] = target_branch_sha if pull_request.merged?
|
|
|
|
merge_request = project.merge_requests.create!(attributes)
|
|
|
|
import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
|
|
|
|
end
|
|
|
|
|
2018-06-25 16:13:59 -04:00
|
|
|
def import_pull_request_comments(pull_request, merge_request)
|
2018-07-06 00:11:29 -04:00
|
|
|
comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?)
|
2018-06-27 17:25:09 -04:00
|
|
|
|
2018-07-06 00:11:29 -04:00
|
|
|
merge_event = other_activities.find(&:merge_event?)
|
2018-06-27 17:25:09 -04:00
|
|
|
import_merge_event(merge_request, merge_event) if merge_event
|
|
|
|
|
2018-06-26 18:59:34 -04:00
|
|
|
inline_comments, pr_comments = comments.partition(&:inline_comment?)
|
2018-06-25 16:13:59 -04:00
|
|
|
|
2018-06-28 03:50:10 -04:00
|
|
|
import_inline_comments(inline_comments.map(&:comment), pull_request, merge_request)
|
2018-06-28 03:27:04 -04:00
|
|
|
import_standalone_pr_comments(pr_comments.map(&:comment), merge_request)
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
|
2018-06-27 17:25:09 -04:00
|
|
|
def import_merge_event(merge_request, merge_event)
|
2018-06-28 03:27:04 -04:00
|
|
|
committer = merge_event.committer_email
|
2018-06-27 17:25:09 -04:00
|
|
|
|
2018-07-17 16:53:48 -04:00
|
|
|
user_id = gitlab_user_id(committer)
|
2018-06-27 17:25:09 -04:00
|
|
|
timestamp = merge_event.merge_timestamp
|
2018-07-06 00:11:29 -04:00
|
|
|
metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request)
|
2018-07-16 01:07:39 -04:00
|
|
|
metric.update(merged_by_id: user_id, merged_at: timestamp)
|
2018-06-27 17:25:09 -04:00
|
|
|
end
|
|
|
|
|
2018-06-25 16:13:59 -04:00
|
|
|
def import_inline_comments(inline_comments, pull_request, merge_request)
|
|
|
|
inline_comments.each do |comment|
|
2018-06-28 18:18:53 -04:00
|
|
|
parent = build_diff_note(merge_request, comment)
|
2018-06-28 04:49:35 -04:00
|
|
|
|
|
|
|
next unless parent&.persisted?
|
|
|
|
|
|
|
|
comment.comments.each do |reply|
|
2018-06-28 18:18:53 -04:00
|
|
|
begin
|
|
|
|
attributes = pull_request_comment_attributes(reply)
|
|
|
|
attributes.merge!(
|
|
|
|
position: build_position(merge_request, comment),
|
|
|
|
discussion_id: parent.discussion_id,
|
|
|
|
type: 'DiffNote')
|
|
|
|
merge_request.notes.create!(attributes)
|
|
|
|
rescue StandardError => e
|
|
|
|
errors << { type: :pull_request, id: comment.id, errors: e.message }
|
|
|
|
end
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-28 18:18:53 -04:00
|
|
|
def build_diff_note(merge_request, comment)
|
2018-06-28 04:49:35 -04:00
|
|
|
attributes = pull_request_comment_attributes(comment)
|
|
|
|
attributes.merge!(
|
|
|
|
position: build_position(merge_request, comment),
|
|
|
|
type: 'DiffNote')
|
|
|
|
|
|
|
|
merge_request.notes.create!(attributes)
|
|
|
|
rescue StandardError => e
|
|
|
|
errors << { type: :pull_request, id: comment.id, errors: e.message }
|
|
|
|
nil
|
|
|
|
end
|
|
|
|
|
2018-06-25 16:13:59 -04:00
|
|
|
def build_position(merge_request, pr_comment)
|
|
|
|
params = {
|
|
|
|
diff_refs: merge_request.diff_refs,
|
|
|
|
old_path: pr_comment.file_path,
|
|
|
|
new_path: pr_comment.file_path,
|
|
|
|
old_line: pr_comment.old_pos,
|
|
|
|
new_line: pr_comment.new_pos
|
|
|
|
}
|
|
|
|
|
|
|
|
Gitlab::Diff::Position.new(params)
|
|
|
|
end
|
|
|
|
|
|
|
|
def import_standalone_pr_comments(pr_comments, merge_request)
|
|
|
|
pr_comments.each do |comment|
|
|
|
|
begin
|
|
|
|
merge_request.notes.create!(pull_request_comment_attributes(comment))
|
2018-06-28 03:27:04 -04:00
|
|
|
|
|
|
|
comment.comments.each do |replies|
|
|
|
|
merge_request.notes.create!(pull_request_comment_attributes(replies))
|
|
|
|
end
|
2018-06-25 16:13:59 -04:00
|
|
|
rescue StandardError => e
|
2018-06-27 17:25:09 -04:00
|
|
|
errors << { type: :pull_request, iid: comment.id, errors: e.message }
|
2018-06-25 16:13:59 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def pull_request_comment_attributes(comment)
|
|
|
|
{
|
|
|
|
project: project,
|
|
|
|
note: comment.note,
|
2018-07-17 16:53:48 -04:00
|
|
|
author_id: gitlab_user_id(comment.author_email),
|
2018-06-25 16:13:59 -04:00
|
|
|
created_at: comment.created_at,
|
|
|
|
updated_at: comment.updated_at
|
|
|
|
}
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|