gitlab-org--gitlab-foss/app/workers/repository_check/batch_worker.rb

module RepositoryCheck
  class BatchWorker
    include ApplicationWorker
    include CronjobQueue

    RUN_TIME = 3600
    BATCH_SIZE = 10_000

    def perform
      return unless Gitlab::CurrentSettings.repository_checks_enabled

      start = Time.now

      # This loop will break after a little more than one hour ('a little
      # more' because `git fsck` may take a few minutes), or if it runs out of
      # projects to check. By default sidekiq-cron will start a new
      # RepositoryCheckWorker each hour so that as long as there are repositories to
      # check, only one (or two) will be checked at a time.
      project_ids.each do |project_id|
        break if Time.now - start >= RUN_TIME

        next unless try_obtain_lease(project_id)

        SingleRepositoryWorker.new.perform(project_id)
      end
    end

    private

    # Project.find_each does not support WHERE clauses and
    # Project.find_in_batches does not support ordering. So we just build an
    # array of ID's. This is OK because we do it only once an hour, because
    # getting ID's from Postgres is not terribly slow, and because no user
    # has to sit and wait for this query to finish.
    def project_ids(batch_size = BATCH_SIZE)
      project_ids = never_checked_project_ids(batch_size)

      remaining_capacity = batch_size - project_ids.count

      if remaining_capacity > 0
        project_ids + old_checked_project_ids(remaining_capacity)
      else
        project_ids
      end
    end

    def never_checked_project_ids(batch_size)
      Project.where(last_repository_check_at: nil)
        .where('created_at < ?', 24.hours.ago)
        .limit(batch_size).pluck(:id)
    end

    def old_checked_project_ids(batch_size)
      Project.where.not(last_repository_check_at: nil)
        .where('last_repository_check_at < ?', 1.month.ago)
        .reorder(last_repository_check_at: :asc)
        .limit(batch_size).pluck(:id)
    end

    def try_obtain_lease(id)
      # Use a 24-hour timeout because on servers/projects where 'git fsck' is
      # super slow we definitely do not want to run it twice in parallel.
      Gitlab::ExclusiveLease.new(
        "project_repository_check:#{id}",
        timeout: 24.hours
      ).try_obtain
    end
  end
end