2020-10-26 18:08:27 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
module ContainerExpirationPolicies
|
|
|
|
class CleanupContainerRepositoryWorker
|
|
|
|
include ApplicationWorker
|
2021-04-30 18:10:09 +00:00
|
|
|
|
|
|
|
sidekiq_options retry: 3
|
2020-10-26 18:08:27 +00:00
|
|
|
include LimitedCapacity::Worker
|
|
|
|
include Gitlab::Utils::StrongMemoize
|
|
|
|
|
|
|
|
queue_namespace :container_repository
|
|
|
|
feature_category :container_registry
|
2021-05-03 09:10:02 +00:00
|
|
|
tags :exclude_from_kubernetes
|
2020-10-26 18:08:27 +00:00
|
|
|
urgency :low
|
|
|
|
worker_resource_boundary :unknown
|
|
|
|
idempotent!
|
|
|
|
|
2021-01-11 15:10:32 +00:00
|
|
|
LOG_ON_DONE_FIELDS = %i[
|
|
|
|
cleanup_status
|
|
|
|
cleanup_tags_service_original_size
|
|
|
|
cleanup_tags_service_before_truncate_size
|
|
|
|
cleanup_tags_service_after_truncate_size
|
|
|
|
cleanup_tags_service_before_delete_size
|
2021-02-17 03:08:59 +00:00
|
|
|
cleanup_tags_service_deleted_size
|
2021-01-11 15:10:32 +00:00
|
|
|
].freeze
|
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
def perform_work
|
|
|
|
return unless throttling_enabled?
|
|
|
|
return unless container_repository
|
2021-04-21 15:09:35 +00:00
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
log_extra_metadata_on_done(:container_repository_id, container_repository.id)
|
|
|
|
log_extra_metadata_on_done(:project_id, project.id)
|
|
|
|
|
|
|
|
unless allowed_to_run?
|
|
|
|
container_repository.cleanup_unscheduled!
|
|
|
|
log_extra_metadata_on_done(:cleanup_status, :skipped)
|
|
|
|
return
|
2020-10-26 18:08:27 +00:00
|
|
|
end
|
2021-05-12 12:10:24 +00:00
|
|
|
|
|
|
|
result = ContainerExpirationPolicies::CleanupService.new(container_repository)
|
|
|
|
.execute
|
|
|
|
log_on_done(result)
|
2020-10-26 18:08:27 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def max_running_jobs
|
|
|
|
return 0 unless throttling_enabled?
|
|
|
|
|
2021-04-21 15:09:35 +00:00
|
|
|
::Gitlab::CurrentSettings.container_registry_expiration_policies_worker_capacity
|
2020-10-26 18:08:27 +00:00
|
|
|
end
|
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
def remaining_work_count
|
2021-07-09 12:08:17 +00:00
|
|
|
count = cleanup_scheduled_count
|
2021-05-12 12:10:24 +00:00
|
|
|
|
2021-07-09 12:08:17 +00:00
|
|
|
return count if count > max_running_jobs
|
2021-05-12 12:10:24 +00:00
|
|
|
|
2021-07-09 12:08:17 +00:00
|
|
|
count + cleanup_unfinished_count
|
2021-05-12 12:10:24 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
def container_repository
|
|
|
|
strong_memoize(:container_repository) do
|
|
|
|
ContainerRepository.transaction do
|
|
|
|
# We need a lock to prevent two workers from picking up the same row
|
2021-06-15 18:09:57 +00:00
|
|
|
container_repository = next_container_repository
|
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
container_repository&.tap(&:cleanup_ongoing!)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def next_container_repository
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
|
|
next_one_requiring = ContainerRepository.requiring_cleanup
|
|
|
|
.order(:expiration_policy_cleanup_status, :expiration_policy_started_at)
|
|
|
|
.limit(1)
|
|
|
|
.lock('FOR UPDATE SKIP LOCKED')
|
|
|
|
.first
|
|
|
|
return next_one_requiring if next_one_requiring
|
|
|
|
|
|
|
|
ContainerRepository.with_unfinished_cleanup
|
|
|
|
.order(:expiration_policy_started_at)
|
|
|
|
.limit(1)
|
|
|
|
.lock('FOR UPDATE SKIP LOCKED')
|
|
|
|
.first
|
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
end
|
|
|
|
|
|
|
|
def cleanup_scheduled_count
|
|
|
|
strong_memoize(:cleanup_scheduled_count) do
|
2021-06-15 18:09:57 +00:00
|
|
|
limit = max_running_jobs + 1
|
|
|
|
ContainerExpirationPolicy.with_container_repositories
|
|
|
|
.runnable_schedules
|
|
|
|
.limit(limit)
|
|
|
|
.count
|
2021-05-12 12:10:24 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def cleanup_unfinished_count
|
|
|
|
strong_memoize(:cleanup_unfinished_count) do
|
2021-06-15 18:09:57 +00:00
|
|
|
limit = max_running_jobs + 1
|
|
|
|
ContainerRepository.with_unfinished_cleanup
|
|
|
|
.limit(limit)
|
|
|
|
.count
|
2021-05-12 12:10:24 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def allowed_to_run?
|
|
|
|
return false unless policy&.enabled && policy&.next_run_at
|
|
|
|
|
|
|
|
now = Time.zone.now
|
|
|
|
|
2021-06-15 18:09:57 +00:00
|
|
|
policy.next_run_at < now || (now + max_cleanup_execution_time.seconds < policy.next_run_at)
|
2021-05-12 12:10:24 +00:00
|
|
|
end
|
|
|
|
|
2020-10-26 18:08:27 +00:00
|
|
|
def throttling_enabled?
|
|
|
|
Feature.enabled?(:container_registry_expiration_policies_throttling)
|
|
|
|
end
|
|
|
|
|
2021-04-21 15:09:35 +00:00
|
|
|
def max_cleanup_execution_time
|
|
|
|
::Gitlab::CurrentSettings.container_registry_delete_tags_service_timeout
|
2020-10-26 18:08:27 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def log_info(extra_structure)
|
|
|
|
logger.info(structured_payload(extra_structure))
|
|
|
|
end
|
2021-01-11 15:10:32 +00:00
|
|
|
|
|
|
|
def log_on_done(result)
|
2021-05-12 12:10:24 +00:00
|
|
|
if result.error?
|
|
|
|
log_extra_metadata_on_done(:cleanup_status, :error)
|
|
|
|
log_extra_metadata_on_done(:cleanup_error_message, result.message)
|
|
|
|
end
|
|
|
|
|
2021-01-11 15:10:32 +00:00
|
|
|
LOG_ON_DONE_FIELDS.each do |field|
|
|
|
|
value = result.payload[field]
|
|
|
|
|
|
|
|
next if value.nil?
|
|
|
|
|
|
|
|
log_extra_metadata_on_done(field, value)
|
|
|
|
end
|
2021-01-18 15:10:42 +00:00
|
|
|
|
|
|
|
before_truncate_size = result.payload[:cleanup_tags_service_before_truncate_size]
|
|
|
|
after_truncate_size = result.payload[:cleanup_tags_service_after_truncate_size]
|
|
|
|
truncated = before_truncate_size &&
|
|
|
|
after_truncate_size &&
|
|
|
|
before_truncate_size != after_truncate_size
|
|
|
|
log_extra_metadata_on_done(:cleanup_tags_service_truncated, !!truncated)
|
2021-02-17 03:08:59 +00:00
|
|
|
log_extra_metadata_on_done(:running_jobs_count, running_jobs_count)
|
2021-01-11 15:10:32 +00:00
|
|
|
end
|
2021-04-21 15:09:35 +00:00
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
def policy
|
|
|
|
project.container_expiration_policy
|
2021-04-21 15:09:35 +00:00
|
|
|
end
|
|
|
|
|
2021-05-12 12:10:24 +00:00
|
|
|
def project
|
|
|
|
container_repository.project
|
2021-04-21 15:09:35 +00:00
|
|
|
end
|
2020-10-26 18:08:27 +00:00
|
|
|
end
|
|
|
|
end
|