291 lines
13 KiB
Ruby
291 lines
13 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'spec_helper'
|
|
|
|
RSpec.describe ContainerRegistry::Migration::GuardWorker, :aggregate_failures do
|
|
let(:worker) { described_class.new }
|
|
|
|
describe '#perform' do
|
|
let(:pre_importing_migrations) { ::ContainerRepository.with_migration_states(:pre_importing) }
|
|
let(:pre_import_done_migrations) { ::ContainerRepository.with_migration_states(:pre_import_done) }
|
|
let(:importing_migrations) { ::ContainerRepository.with_migration_states(:importing) }
|
|
let(:import_aborted_migrations) { ::ContainerRepository.with_migration_states(:import_aborted) }
|
|
let(:import_done_migrations) { ::ContainerRepository.with_migration_states(:import_done) }
|
|
let(:import_skipped_migrations) { ::ContainerRepository.with_migration_states(:import_skipped) }
|
|
|
|
subject { worker.perform }
|
|
|
|
before do
|
|
stub_container_registry_config(enabled: true, api_url: 'http://container-registry', key: 'spec/fixtures/x509_certificate_pk.key')
|
|
allow(::ContainerRegistry::Migration).to receive(:max_step_duration).and_return(5.minutes)
|
|
end
|
|
|
|
context 'on gitlab.com' do
|
|
before do
|
|
allow(::Gitlab).to receive(:com?).and_return(true)
|
|
end
|
|
|
|
shared_examples 'handling long running migrations' do |timeout:|
|
|
before do
|
|
allow_next_found_instance_of(ContainerRepository) do |repository|
|
|
allow(repository).to receive(:migration_cancel).and_return(migration_cancel_response)
|
|
end
|
|
end
|
|
|
|
shared_examples 'aborting the migration' do
|
|
it 'will abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_ids, [stale_migration.id])
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_paths, [stale_migration.path])
|
|
expect(ContainerRegistry::Migration).to receive(timeout).and_call_original
|
|
|
|
expect { subject }
|
|
.to change(import_aborted_migrations, :count).by(1)
|
|
.and change { stale_migration.reload.migration_state }.to('import_aborted')
|
|
.and not_change { ongoing_migration.migration_state }
|
|
end
|
|
end
|
|
|
|
context 'migration is canceled' do
|
|
let(:migration_cancel_response) { { status: :ok } }
|
|
|
|
before do
|
|
stub_application_setting(container_registry_import_max_retries: 3)
|
|
end
|
|
|
|
context 'when the retry limit has been reached' do
|
|
before do
|
|
stale_migration.update_column(:migration_retries_count, 2)
|
|
end
|
|
|
|
it 'will not abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_ids, [stale_migration.id])
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_long_running_migration_paths, [stale_migration.path])
|
|
expect(ContainerRegistry::Migration).to receive(timeout).and_call_original
|
|
|
|
expect { subject }
|
|
.to change(import_skipped_migrations, :count)
|
|
|
|
expect(stale_migration.reload.migration_state).to eq('import_skipped')
|
|
expect(stale_migration.reload.migration_skipped_reason).to eq('migration_canceled')
|
|
end
|
|
end
|
|
|
|
context 'when the retry limit has not been reached' do
|
|
it_behaves_like 'aborting the migration'
|
|
end
|
|
end
|
|
|
|
context 'migration cancelation fails with an error' do
|
|
let(:migration_cancel_response) { { status: :error } }
|
|
|
|
it_behaves_like 'aborting the migration'
|
|
end
|
|
|
|
context 'migration receives bad request with a new status' do
|
|
let(:migration_cancel_response) { { status: :bad_request, migration_state: :import_done } }
|
|
|
|
it_behaves_like 'aborting the migration'
|
|
end
|
|
end
|
|
|
|
context 'with no stale migrations' do
|
|
it_behaves_like 'an idempotent worker'
|
|
|
|
it 'will not update any migration state' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 0)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 0)
|
|
|
|
expect { subject }
|
|
.to not_change(pre_importing_migrations, :count)
|
|
.and not_change(pre_import_done_migrations, :count)
|
|
.and not_change(importing_migrations, :count)
|
|
.and not_change(import_aborted_migrations, :count)
|
|
end
|
|
end
|
|
|
|
context 'with pre_importing stale migrations' do
|
|
let_it_be(:ongoing_migration) { create(:container_repository, :pre_importing) }
|
|
let_it_be(:stale_migration) { create(:container_repository, :pre_importing, migration_pre_import_started_at: 11.minutes.ago) }
|
|
|
|
let(:import_status) { 'test' }
|
|
|
|
before do
|
|
allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client|
|
|
allow(client).to receive(:import_status).and_return(import_status)
|
|
end
|
|
end
|
|
|
|
it 'will abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1)
|
|
|
|
expect { subject }
|
|
.to change(pre_importing_migrations, :count).by(-1)
|
|
.and not_change(pre_import_done_migrations, :count)
|
|
.and not_change(importing_migrations, :count)
|
|
.and not_change(import_done_migrations, :count)
|
|
.and change(import_aborted_migrations, :count).by(1)
|
|
.and change { stale_migration.reload.migration_state }.from('pre_importing').to('import_aborted')
|
|
.and not_change { ongoing_migration.migration_state }
|
|
end
|
|
|
|
context 'the client returns pre_import_in_progress' do
|
|
let(:import_status) { 'pre_import_in_progress' }
|
|
|
|
shared_examples 'not aborting the stale migration' do
|
|
it 'will not abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 0)
|
|
|
|
expect { subject }
|
|
.to not_change(pre_importing_migrations, :count)
|
|
.and not_change(pre_import_done_migrations, :count)
|
|
.and not_change(importing_migrations, :count)
|
|
.and not_change(import_done_migrations, :count)
|
|
.and not_change(import_aborted_migrations, :count)
|
|
.and not_change { stale_migration.reload.migration_state }
|
|
.and not_change { ongoing_migration.migration_state }
|
|
end
|
|
end
|
|
|
|
context 'not long running' do
|
|
before do
|
|
stub_application_setting(container_registry_pre_import_timeout: 12.minutes.to_i)
|
|
end
|
|
|
|
it_behaves_like 'not aborting the stale migration'
|
|
end
|
|
|
|
context 'long running' do
|
|
before do
|
|
stub_application_setting(container_registry_pre_import_timeout: 9.minutes.to_i)
|
|
end
|
|
|
|
context 'with registry_migration_guard_dynamic_pre_import_timeout enabled' do
|
|
before do
|
|
stub_application_setting(container_registry_pre_import_tags_rate: 1)
|
|
end
|
|
|
|
context 'below the dynamic threshold' do
|
|
before do
|
|
allow_next_found_instance_of(ContainerRepository) do |repository|
|
|
allow(repository).to receive(:tags_count).and_return(11.minutes.to_i + 100)
|
|
end
|
|
end
|
|
|
|
it_behaves_like 'not aborting the stale migration'
|
|
end
|
|
|
|
context 'above the dynamic threshold' do
|
|
let(:tags) do
|
|
Array.new(11.minutes.to_i - 100) { |i| "tag#{i}" }
|
|
end
|
|
|
|
before do
|
|
# We can't allow_next_found_instance_of because the shared example
|
|
# 'handling long running migrations' is already using that.
|
|
# Instead, here we're going to stub the ContainerRegistry::Client instance.
|
|
allow_next_instance_of(ContainerRegistry::Client) do |client|
|
|
allow(client).to receive(:repository_tags).and_return({ 'tags' => tags })
|
|
end
|
|
end
|
|
|
|
it_behaves_like 'handling long running migrations', timeout: :pre_import_timeout
|
|
end
|
|
end
|
|
|
|
context 'with registry_migration_guard_dynamic_pre_import_timeout disabled' do
|
|
before do
|
|
stub_feature_flags(registry_migration_guard_dynamic_pre_import_timeout: false)
|
|
end
|
|
|
|
it_behaves_like 'handling long running migrations', timeout: :pre_import_timeout
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'with pre_import_done stale migrations' do
|
|
let(:ongoing_migration) { create(:container_repository, :pre_import_done) }
|
|
let(:stale_migration) { create(:container_repository, :pre_import_done, migration_pre_import_done_at: 11.minutes.ago) }
|
|
|
|
before do
|
|
allow(::ContainerRegistry::Migration).to receive(:max_step_duration).and_return(5.minutes)
|
|
end
|
|
|
|
it 'will abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1)
|
|
|
|
expect { subject }
|
|
.to not_change(pre_importing_migrations, :count)
|
|
.and change(pre_import_done_migrations, :count).by(-1)
|
|
.and not_change(importing_migrations, :count)
|
|
.and not_change(import_done_migrations, :count)
|
|
.and change(import_aborted_migrations, :count).by(1)
|
|
.and change { stale_migration.reload.migration_state }.from('pre_import_done').to('import_aborted')
|
|
.and not_change { ongoing_migration.migration_state }
|
|
end
|
|
end
|
|
|
|
context 'with importing stale migrations' do
|
|
let(:ongoing_migration) { create(:container_repository, :importing) }
|
|
let(:stale_migration) { create(:container_repository, :importing, migration_import_started_at: 11.minutes.ago) }
|
|
let(:import_status) { 'test' }
|
|
|
|
before do
|
|
allow_next_instance_of(ContainerRegistry::GitlabApiClient) do |client|
|
|
allow(client).to receive(:import_status).and_return(import_status)
|
|
end
|
|
|
|
stub_application_setting(container_registry_import_timeout: 10.minutes.to_i)
|
|
end
|
|
|
|
it 'will abort the migration' do
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:stale_migrations_count, 1)
|
|
expect(worker).to receive(:log_extra_metadata_on_done).with(:aborted_stale_migrations_count, 1)
|
|
|
|
expect { subject }
|
|
.to not_change(pre_importing_migrations, :count)
|
|
.and not_change(pre_import_done_migrations, :count)
|
|
.and change(importing_migrations, :count).by(-1)
|
|
.and not_change(import_done_migrations, :count)
|
|
.and change(import_aborted_migrations, :count).by(1)
|
|
.and change { stale_migration.reload.migration_state }.from('importing').to('import_aborted')
|
|
.and not_change { ongoing_migration.migration_state }
|
|
end
|
|
|
|
context 'the client returns import_in_progress' do
|
|
let(:import_status) { 'import_in_progress' }
|
|
|
|
it_behaves_like 'handling long running migrations', timeout: :import_timeout
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'not on gitlab.com' do
|
|
before do
|
|
allow(::Gitlab).to receive(:com?).and_return(false)
|
|
end
|
|
|
|
it 'is a no op' do
|
|
expect(::ContainerRepository).not_to receive(:with_stale_migration)
|
|
expect(worker).not_to receive(:log_extra_metadata_on_done)
|
|
|
|
subject
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'worker attributes' do
|
|
it 'has deduplication set' do
|
|
expect(described_class.get_deduplicate_strategy).to eq(:until_executed)
|
|
expect(described_class.get_deduplication_options).to include(ttl: 5.minutes)
|
|
end
|
|
end
|
|
end
|