gitlab-org--gitlab-foss/app/workers/background_migration_worker.rb

# frozen_string_literal: true

class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker
  include ApplicationWorker

  feature_category :database
  urgency :throttled
  loggable_arguments 0, 1

  # The minimum amount of time between processing two jobs of the same migration
  # class.
  #
  # This interval is set to 2 or 5 minutes so autovacuuming and other
  # maintenance related tasks have plenty of time to clean up after a migration
  # has been performed.
  def self.minimum_interval
    2.minutes.to_i
  end

  # Performs the background migration.
  #
  # See Gitlab::BackgroundMigration.perform for more information.
  #
  # class_name - The class name of the background migration to run.
  # arguments - The arguments to pass to the migration class.
  # lease_attempts - The number of times we will try to obtain an exclusive
  #   lease on the class before running anyway.  Pass 0 to always run.
  def perform(class_name, arguments = [], lease_attempts = 5)
    with_context(caller_id: class_name.to_s) do
      should_perform, ttl = perform_and_ttl(class_name)

      if should_perform
        Gitlab::BackgroundMigration.perform(class_name, arguments)
      else
        # If the lease could not be obtained this means either another process is
        # running a migration of this class or we ran one recently. In this case
        # we'll reschedule the job in such a way that it is picked up again around
        # the time the lease expires.
        self.class
          .perform_in(ttl || self.class.minimum_interval, class_name, arguments)
      end
    end
  end

  def perform_and_ttl(class_name)
    if always_perform?
      # In test environments `perform_in` will run right away. This can then
      # lead to stack level errors in the above `#perform`. To work around this
      # we'll just perform the migration right away in the test environment.
      [true, nil]
    else
      lease = lease_for(class_name)
      perform = !!lease.try_obtain

      # If we managed to acquire the lease but the DB is not healthy, then we
      # want to simply reschedule our job and try again _after_ the lease
      # expires.
      if perform && !healthy_database?
        database_unhealthy_counter.increment

        perform = false
      end

      [perform, lease.ttl]
    end
  end

  def lease_for(class_name)
    Gitlab::ExclusiveLease
      .new(lease_key_for(class_name), timeout: self.class.minimum_interval)
  end

  def lease_key_for(class_name)
    "#{self.class.name}:#{class_name}"
  end

  def always_perform?
    Rails.env.test?
  end

  # Returns true if the database is healthy enough to allow the migration to be
  # performed.
  #
  # class_name - The name of the background migration that we might want to
  #              run.
  def healthy_database?
    !Postgresql::ReplicationSlot.lag_too_great?
  end

  def database_unhealthy_counter
    Gitlab::Metrics.counter(
      :background_migration_database_health_reschedules,
      'The number of times a background migration is rescheduled because the database is unhealthy.'
    )
  end
end