gitlab-org--gitlab-foss/spec/services/projects/import_service_spec.rb

312 lines
9.7 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require 'spec_helper'
describe Projects::ImportService do
let!(:project) { create(:project) }
let(:user) { project.creator }
subject { described_class.new(project, user) }
before do
allow(project).to receive(:lfs_enabled?).and_return(true)
end
Rewrite the GitHub importer from scratch Prior to this MR there were two GitHub related importers: * Github::Import: the main importer used for GitHub projects * Gitlab::GithubImport: importer that's somewhat confusingly used for importing Gitea projects (apparently they have a compatible API) This MR renames the Gitea importer to Gitlab::LegacyGithubImport and introduces a new GitHub importer in the Gitlab::GithubImport namespace. This new GitHub importer uses Sidekiq for importing multiple resources in parallel, though it also has the ability to import data sequentially should this be necessary. The new code is spread across the following directories: * lib/gitlab/github_import: this directory contains most of the importer code such as the classes used for importing resources. * app/workers/gitlab/github_import: this directory contains the Sidekiq workers, most of which simply use the code from the directory above. * app/workers/concerns/gitlab/github_import: this directory provides a few modules that are included in every GitHub importer worker. == Stages The import work is divided into separate stages, with each stage importing a specific set of data. Stages will schedule the work that needs to be performed, followed by scheduling a job for the "AdvanceStageWorker" worker. This worker will periodically check if all work is completed and schedule the next stage if this is the case. If work is not yet completed this worker will reschedule itself. Using this approach we don't have to block threads by calling `sleep()`, as doing so for large projects could block the thread from doing any work for many hours. == Retrying Work Workers will reschedule themselves whenever necessary. For example, hitting the GitHub API's rate limit will result in jobs rescheduling themselves. These jobs are not processed until the rate limit has been reset. == User Lookups Part of the importing process involves looking up user details in the GitHub API so we can map them to GitLab users. The old importer used an in-memory cache, but this obviously doesn't work when the work is spread across different threads. The new importer uses a Redis cache and makes sure we only perform API/database calls if absolutely necessary. Frequently used keys are refreshed, and lookup misses are also cached; removing the need for performing API/database calls if we know we don't have the data we're looking for. == Performance & Models The new importer in various places uses raw INSERT statements (as generated by `Gitlab::Database.bulk_insert`) instead of using Rails models. This allows us to bypass any validations and callbacks, drastically reducing the number of SQL queries and Gitaly RPC calls necessary to import projects. To ensure the code produces valid data the corresponding tests check if the produced rows are valid according to the model validation rules.
2017-10-13 16:50:36 +00:00
describe '#async?' do
it 'returns true for an asynchronous importer' do
importer_class = double(:importer, async?: true)
allow(subject).to receive(:has_importer?).and_return(true)
allow(subject).to receive(:importer_class).and_return(importer_class)
expect(subject).to be_async
end
it 'returns false for a regular importer' do
importer_class = double(:importer, async?: false)
allow(subject).to receive(:has_importer?).and_return(true)
allow(subject).to receive(:importer_class).and_return(importer_class)
expect(subject).not_to be_async
end
it 'returns false when the importer does not define #async?' do
importer_class = double(:importer)
allow(subject).to receive(:has_importer?).and_return(true)
allow(subject).to receive(:importer_class).and_return(importer_class)
expect(subject).not_to be_async
end
it 'returns false when the importer does not exist' do
allow(subject).to receive(:has_importer?).and_return(false)
expect(subject).not_to be_async
end
end
describe '#execute' do
context 'with unknown url' do
before do
project.import_url = Project::UNKNOWN_IMPORT_URL
end
it 'succeeds if repository is created successfully' do
expect(project).to receive(:create_repository).and_return(true)
result = subject.execute
expect(result[:status]).to eq :success
end
it 'fails if repository creation fails' do
expect(project).to receive(:create_repository).and_return(false)
result = subject.execute
expect(result[:status]).to eq :error
expect(result[:message]).to eq "Error importing repository #{project.safe_import_url} into #{project.full_path} - The repository could not be created."
end
context 'when repository creation succeeds' do
it 'does not download lfs files' do
expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute)
subject.execute
end
end
end
context 'with known url' do
before do
project.import_url = 'https://github.com/vim/vim.git'
2017-04-03 18:48:09 +00:00
project.import_type = 'github'
end
2017-04-03 18:48:09 +00:00
context 'with a Github repository' do
it 'succeeds if repository import was scheduled' do
expect_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
.and_return(true)
2017-04-03 18:48:09 +00:00
result = subject.execute
2017-04-03 18:48:09 +00:00
expect(result[:status]).to eq :success
end
it 'fails if repository import was not scheduled' do
expect_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
.and_return(false)
2017-04-03 18:48:09 +00:00
result = subject.execute
expect(result[:status]).to eq :error
end
context 'when repository import scheduled' do
it 'does not download lfs objects' do
expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute)
subject.execute
end
end
end
2017-04-03 18:48:09 +00:00
context 'with a non Github repository' do
before do
project.import_url = 'https://bitbucket.org/vim/vim.git'
project.import_type = 'bitbucket'
end
2018-10-30 10:53:01 +00:00
it 'succeeds if repository import is successful' do
expect(project.repository).to receive(:import_repository).and_return(true)
expect_next_instance_of(Gitlab::BitbucketImport::Importer) do |importer|
expect(importer).to receive(:execute).and_return(true)
end
expect_next_instance_of(Projects::LfsPointers::LfsImportService) do |service|
expect(service).to receive(:execute).and_return(status: :success)
end
2017-04-03 18:48:09 +00:00
result = subject.execute
expect(result[:status]).to eq :success
end
it 'fails if repository import fails' do
expect(project.repository)
.to receive(:import_repository)
.and_raise(Gitlab::Git::CommandError, 'Failed to import the repository /a/b/c')
2017-04-03 18:48:09 +00:00
result = subject.execute
expect(result[:status]).to eq :error
2018-12-05 13:31:43 +00:00
expect(result[:message]).to eq "Error importing repository #{project.safe_import_url} into #{project.full_path} - Failed to import the repository [FILTERED]"
2017-04-03 18:48:09 +00:00
end
context 'when lfs import fails' do
it 'logs the error' do
error_message = 'error message'
expect(project.repository).to receive(:import_repository).and_return(true)
expect_next_instance_of(Gitlab::BitbucketImport::Importer) do |importer|
expect(importer).to receive(:execute).and_return(true)
end
expect_next_instance_of(Projects::LfsPointers::LfsImportService) do |service|
expect(service).to receive(:execute).and_return(status: :error, message: error_message)
end
expect(Gitlab::AppLogger).to receive(:error).with("The Lfs import process failed. #{error_message}")
subject.execute
end
end
context 'when repository import scheduled' do
before do
expect(project.repository).to receive(:import_repository).and_return(true)
allow(subject).to receive(:import_data)
end
it 'downloads lfs objects if lfs_enabled is enabled for project' do
allow(project).to receive(:lfs_enabled?).and_return(true)
expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute)
subject.execute
end
it 'does not download lfs objects if lfs_enabled is not enabled for project' do
allow(project).to receive(:lfs_enabled?).and_return(false)
expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute)
subject.execute
end
end
end
end
context 'with valid importer' do
before do
stub_github_omniauth_provider
project.import_url = 'https://github.com/vim/vim.git'
project.import_type = 'github'
allow(project).to receive(:import_data).and_return(double.as_null_object)
end
it 'succeeds if importer succeeds' do
allow_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute).and_return(true)
result = subject.execute
expect(result[:status]).to eq :success
end
it 'fails if importer fails' do
allow_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
.and_return(false)
result = subject.execute
expect(result[:status]).to eq :error
end
context 'when importer' do
it 'has a custom repository importer it does not download lfs objects' do
allow(Gitlab::GithubImport::ParallelImporter).to receive(:imports_repository?).and_return(true)
expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute)
subject.execute
end
it 'does not have a custom repository importer downloads lfs objects' do
allow(Gitlab::GithubImport::ParallelImporter).to receive(:imports_repository?).and_return(false)
expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute)
subject.execute
end
context 'when lfs import fails' do
it 'logs the error' do
error_message = 'error message'
allow(Gitlab::GithubImport::ParallelImporter).to receive(:imports_repository?).and_return(false)
expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(status: :error, message: error_message)
expect(Gitlab::AppLogger).to receive(:error).with("The Lfs import process failed. #{error_message}")
subject.execute
end
end
end
end
context 'with blocked import_URL' do
it 'fails with localhost' do
project.import_url = 'https://localhost:9000/vim/vim.git'
result = described_class.new(project, user).execute
expect(result[:status]).to eq :error
expect(result[:message]).to include('Requests to localhost are not allowed')
end
it 'fails with port 25' do
project.import_url = "https://github.com:25/vim/vim.git"
result = subject.execute
expect(result[:status]).to eq :error
expect(result[:message]).to include('Only allowed ports are 80, 443')
end
end
it_behaves_like 'measurable service' do
let(:base_log_data) do
{
class: described_class.name,
current_user: user.name,
project_full_path: project.full_path,
import_type: project.import_type,
file_path: project.import_source
}
end
before do
project.import_type = 'github'
end
after do
subject.execute
end
end
def stub_github_omniauth_provider
provider = OpenStruct.new(
'name' => 'github',
'app_id' => 'asd123',
'app_secret' => 'asd123',
'args' => {
'client_options' => {
'site' => 'https://github.com/api/v3',
'authorize_url' => 'https://github.com/login/oauth/authorize',
'token_url' => 'https://github.com/login/oauth/access_token'
}
}
)
stub_omniauth_setting(providers: [provider])
end
end
end