2022-02-10 15:12:42 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
module ContainerRegistry
|
|
|
|
module Migration
|
|
|
|
class GuardWorker
|
|
|
|
include ApplicationWorker
|
|
|
|
# This is a general worker with no context.
|
|
|
|
# It is not scoped to a project, user or group.
|
|
|
|
# We don't have a context.
|
|
|
|
include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
|
|
|
|
|
|
|
|
data_consistency :always
|
|
|
|
feature_category :container_registry
|
|
|
|
urgency :low
|
|
|
|
worker_resource_boundary :unknown
|
2022-05-16 15:08:39 +00:00
|
|
|
deduplicate :until_executed, ttl: 5.minutes
|
2022-02-10 15:12:42 +00:00
|
|
|
idempotent!
|
|
|
|
|
|
|
|
def perform
|
|
|
|
return unless Gitlab.com?
|
|
|
|
|
|
|
|
repositories = ::ContainerRepository.with_stale_migration(step_before_timestamp)
|
|
|
|
.limit(max_capacity)
|
2022-02-15 15:15:04 +00:00
|
|
|
aborts_count = 0
|
2022-05-23 00:08:37 +00:00
|
|
|
long_running_migrations = []
|
2022-02-10 15:12:42 +00:00
|
|
|
|
|
|
|
# the #to_a is safe as the amount of entries is limited.
|
|
|
|
# In addition, we're calling #each in the next line and we don't want two different SQL queries for these two lines
|
|
|
|
log_extra_metadata_on_done(:stale_migrations_count, repositories.to_a.size)
|
|
|
|
|
|
|
|
repositories.each do |repository|
|
2022-03-18 18:07:26 +00:00
|
|
|
if actively_importing?(repository)
|
|
|
|
# if a repository is actively importing but not yet long_running, do nothing
|
|
|
|
if long_running_migration?(repository)
|
2022-05-23 00:08:37 +00:00
|
|
|
long_running_migrations << repository
|
2022-03-18 18:07:26 +00:00
|
|
|
cancel_long_running_migration(repository)
|
|
|
|
aborts_count += 1
|
|
|
|
end
|
|
|
|
else
|
2022-02-15 15:15:04 +00:00
|
|
|
repository.abort_import
|
|
|
|
aborts_count += 1
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
log_extra_metadata_on_done(:aborted_stale_migrations_count, aborts_count)
|
|
|
|
|
2022-05-23 00:08:37 +00:00
|
|
|
if long_running_migrations.any?
|
|
|
|
log_extra_metadata_on_done(:aborted_long_running_migration_ids, long_running_migrations.map(&:id))
|
|
|
|
log_extra_metadata_on_done(:aborted_long_running_migration_paths, long_running_migrations.map(&:path))
|
2022-02-10 15:12:42 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2022-03-18 18:07:26 +00:00
|
|
|
# A repository is actively_importing if it has an importing migration state
|
|
|
|
# and that state matches the state in the registry
|
|
|
|
# TODO We can have an API call n+1 situation here. It can be solved when the
|
|
|
|
# endpoint accepts multiple repository paths at once. This is issue
|
2022-02-15 15:15:04 +00:00
|
|
|
# https://gitlab.com/gitlab-org/container-registry/-/issues/582
|
2022-03-18 18:07:26 +00:00
|
|
|
def actively_importing?(repository)
|
|
|
|
return false unless repository.importing? || repository.pre_importing?
|
|
|
|
return false unless external_state_matches_migration_state?(repository)
|
2022-02-15 15:15:04 +00:00
|
|
|
|
2022-03-18 18:07:26 +00:00
|
|
|
true
|
2022-02-15 15:15:04 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def long_running_migration?(repository)
|
2022-05-30 18:08:57 +00:00
|
|
|
timeout = if repository.migration_state == 'pre_importing'
|
|
|
|
migration.pre_import_timeout.seconds
|
|
|
|
else
|
|
|
|
migration.import_timeout.seconds
|
|
|
|
end
|
2022-05-12 15:08:59 +00:00
|
|
|
|
2022-05-26 00:08:24 +00:00
|
|
|
if repository.migration_state == 'pre_importing' &&
|
|
|
|
Feature.enabled?(:registry_migration_guard_dynamic_pre_import_timeout) &&
|
|
|
|
migration_start_timestamp(repository).before?(timeout.ago)
|
2022-06-09 15:09:21 +00:00
|
|
|
timeout = migration.dynamic_pre_import_timeout_for(repository)
|
2022-05-26 00:08:24 +00:00
|
|
|
end
|
|
|
|
|
2022-05-12 15:08:59 +00:00
|
|
|
migration_start_timestamp(repository).before?(timeout.ago)
|
2022-02-15 15:15:04 +00:00
|
|
|
end
|
|
|
|
|
2022-03-18 18:07:26 +00:00
|
|
|
def external_state_matches_migration_state?(repository)
|
2022-02-15 15:15:04 +00:00
|
|
|
status = repository.external_import_status
|
|
|
|
|
|
|
|
(status == 'pre_import_in_progress' && repository.pre_importing?) ||
|
|
|
|
(status == 'import_in_progress' && repository.importing?)
|
|
|
|
end
|
|
|
|
|
|
|
|
def migration_start_timestamp(repository)
|
|
|
|
if repository.pre_importing?
|
|
|
|
repository.migration_pre_import_started_at
|
|
|
|
else
|
|
|
|
repository.migration_import_started_at
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-02-10 15:12:42 +00:00
|
|
|
def step_before_timestamp
|
2022-05-12 15:08:59 +00:00
|
|
|
migration.max_step_duration.seconds.ago
|
2022-02-10 15:12:42 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def max_capacity
|
|
|
|
# doubling the actual capacity to prevent issues in case the capacity
|
|
|
|
# is not properly applied
|
2022-05-12 15:08:59 +00:00
|
|
|
migration.capacity * 2
|
|
|
|
end
|
|
|
|
|
|
|
|
def migration
|
|
|
|
::ContainerRegistry::Migration
|
2022-02-10 15:12:42 +00:00
|
|
|
end
|
2022-02-15 15:15:04 +00:00
|
|
|
|
2022-03-18 18:07:26 +00:00
|
|
|
def cancel_long_running_migration(repository)
|
|
|
|
result = repository.migration_cancel
|
|
|
|
|
|
|
|
case result[:status]
|
|
|
|
when :ok
|
2022-04-20 15:10:23 +00:00
|
|
|
if repository.nearing_or_exceeded_retry_limit?
|
|
|
|
repository.skip_import(reason: :migration_canceled)
|
|
|
|
else
|
|
|
|
repository.abort_import
|
|
|
|
end
|
2022-03-18 18:07:26 +00:00
|
|
|
when :bad_request
|
|
|
|
repository.reconcile_import_status(result[:state]) do
|
|
|
|
repository.abort_import
|
|
|
|
end
|
|
|
|
else
|
|
|
|
repository.abort_import
|
|
|
|
end
|
|
|
|
end
|
2022-02-10 15:12:42 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|