2018-07-17 16:50:37 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
module Projects
|
|
|
|
class ImportService < BaseService
|
|
|
|
include Gitlab::ShellAdapter
|
|
|
|
|
2017-03-01 11:00:37 +00:00
|
|
|
Error = Class.new(StandardError)
|
2016-01-21 18:09:32 +00:00
|
|
|
|
2017-10-13 16:50:36 +00:00
|
|
|
# Returns true if this importer is supposed to perform its work in the
|
|
|
|
# background.
|
|
|
|
#
|
|
|
|
# This method will only return `true` if async importing is explicitly
|
|
|
|
# supported by an importer class (`Gitlab::GithubImport::ParallelImporter`
|
|
|
|
# for example).
|
|
|
|
def async?
|
2017-11-15 12:27:37 +00:00
|
|
|
has_importer? && !!importer_class.try(:async?)
|
2017-10-13 16:50:36 +00:00
|
|
|
end
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
def execute
|
2017-11-15 12:27:37 +00:00
|
|
|
add_repository_to_project
|
2016-01-21 18:09:32 +00:00
|
|
|
|
2018-06-06 16:42:18 +00:00
|
|
|
download_lfs_objects
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
import_data
|
|
|
|
|
|
|
|
success
|
2018-12-05 13:31:43 +00:00
|
|
|
rescue Gitlab::UrlBlocker::BlockedUrlError => e
|
|
|
|
Gitlab::Sentry.track_acceptable_exception(e, extra: { project_path: project.full_path, importer: project.import_type })
|
|
|
|
|
2019-04-15 12:25:48 +00:00
|
|
|
error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: e.message })
|
2018-12-05 13:31:43 +00:00
|
|
|
rescue => e
|
|
|
|
message = Projects::ImportErrorFilter.filter_message(e.message)
|
|
|
|
|
|
|
|
Gitlab::Sentry.track_acceptable_exception(e, extra: { project_path: project.full_path, importer: project.import_type })
|
|
|
|
|
2019-04-15 12:25:48 +00:00
|
|
|
error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: message })
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2016-06-14 18:32:19 +00:00
|
|
|
def add_repository_to_project
|
2017-11-15 12:27:37 +00:00
|
|
|
if project.external_import? && !unknown_url?
|
2018-03-28 17:27:16 +00:00
|
|
|
begin
|
2018-06-01 11:43:53 +00:00
|
|
|
Gitlab::UrlBlocker.validate!(project.import_url, ports: Project::VALID_IMPORT_PORTS)
|
2018-03-28 17:27:16 +00:00
|
|
|
rescue Gitlab::UrlBlocker::BlockedUrlError => e
|
2019-04-15 12:25:48 +00:00
|
|
|
raise e, s_("ImportProjects|Blocked import URL: %{message}") % { message: e.message }
|
2018-03-28 17:27:16 +00:00
|
|
|
end
|
2017-11-15 12:27:37 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
# We should skip the repository for a GitHub import or GitLab project import,
|
|
|
|
# because these importers fetch the project repositories for us.
|
2018-06-06 16:42:18 +00:00
|
|
|
return if importer_imports_repository?
|
2017-11-15 12:27:37 +00:00
|
|
|
|
2016-06-14 18:32:19 +00:00
|
|
|
if unknown_url?
|
|
|
|
# In this case, we only want to import issues, not a repository.
|
|
|
|
create_repository
|
2016-10-19 12:21:27 +00:00
|
|
|
elsif !project.repository_exists?
|
2017-04-03 18:48:09 +00:00
|
|
|
import_repository
|
2016-06-14 18:32:19 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
def create_repository
|
|
|
|
unless project.create_repository
|
2019-04-15 12:25:48 +00:00
|
|
|
raise Error, s_('ImportProjects|The repository could not be created.')
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def import_repository
|
|
|
|
begin
|
2017-11-23 15:51:55 +00:00
|
|
|
refmap = importer_class.try(:refmap) if has_importer?
|
2017-11-15 15:46:08 +00:00
|
|
|
|
2017-11-23 15:51:55 +00:00
|
|
|
if refmap
|
2017-11-15 15:46:08 +00:00
|
|
|
project.ensure_repository
|
2017-11-23 15:51:55 +00:00
|
|
|
project.repository.fetch_as_mirror(project.import_url, refmap: refmap)
|
2017-04-03 18:48:09 +00:00
|
|
|
else
|
2019-01-17 08:35:40 +00:00
|
|
|
gitlab_shell.import_project_repository(project)
|
2017-04-03 18:48:09 +00:00
|
|
|
end
|
2018-07-19 18:40:36 +00:00
|
|
|
rescue Gitlab::Shell::Error => e
|
2016-09-23 07:42:07 +00:00
|
|
|
# Expire cache to prevent scenarios such as:
|
|
|
|
# 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
|
|
|
|
# 2. Retried import, repo is broken or not imported but +exists?+ still returns true
|
2017-04-03 18:48:09 +00:00
|
|
|
project.repository.expire_content_cache if project.repository_exists?
|
2016-09-23 07:42:07 +00:00
|
|
|
|
2017-04-03 18:48:09 +00:00
|
|
|
raise Error, e.message
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-06 16:42:18 +00:00
|
|
|
def download_lfs_objects
|
|
|
|
# In this case, we only want to import issues
|
|
|
|
return if unknown_url?
|
|
|
|
|
|
|
|
# If it has its own repository importer, it has to implements its own lfs import download
|
|
|
|
return if importer_imports_repository?
|
|
|
|
|
|
|
|
return unless project.lfs_enabled?
|
|
|
|
|
2019-04-30 08:21:21 +00:00
|
|
|
result = Projects::LfsPointers::LfsImportService.new(project).execute
|
2018-06-06 16:42:18 +00:00
|
|
|
|
2019-04-30 08:21:21 +00:00
|
|
|
if result[:status] == :error
|
|
|
|
# To avoid aborting the importing process, we silently fail
|
|
|
|
# if any exception raises.
|
|
|
|
Gitlab::AppLogger.error("The Lfs import process failed. #{result[:message]}")
|
2018-06-06 16:42:18 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
def import_data
|
|
|
|
return unless has_importer?
|
|
|
|
|
2017-04-03 18:48:09 +00:00
|
|
|
project.repository.expire_content_cache unless project.gitlab_project_import?
|
2016-04-04 22:35:39 +00:00
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
unless importer.execute
|
2019-04-15 12:25:48 +00:00
|
|
|
raise Error, s_('ImportProjects|The remote data could not be imported.')
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-10-18 19:46:05 +00:00
|
|
|
def importer_class
|
2017-11-15 12:27:37 +00:00
|
|
|
@importer_class ||= Gitlab::ImportSources.importer(project.import_type)
|
2017-10-18 19:46:05 +00:00
|
|
|
end
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
def has_importer?
|
2016-12-16 08:15:30 +00:00
|
|
|
Gitlab::ImportSources.importer_names.include?(project.import_type)
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def importer
|
2017-10-13 16:50:36 +00:00
|
|
|
importer_class.new(project)
|
|
|
|
end
|
|
|
|
|
2016-01-21 18:09:32 +00:00
|
|
|
def unknown_url?
|
|
|
|
project.import_url == Project::UNKNOWN_IMPORT_URL
|
|
|
|
end
|
2018-06-06 16:42:18 +00:00
|
|
|
|
|
|
|
def importer_imports_repository?
|
|
|
|
has_importer? && importer_class.try(:imports_repository?)
|
|
|
|
end
|
2016-01-21 18:09:32 +00:00
|
|
|
end
|
|
|
|
end
|