gitlab-org--gitlab-foss/config/object_store_settings.rb

# frozen_string_literal: true

# Set default values for object_store settings
class ObjectStoreSettings
  SUPPORTED_TYPES = %w(artifacts external_diffs lfs uploads packages dependency_proxy terraform_state pages secure_files).freeze
  ALLOWED_OBJECT_STORE_OVERRIDES = %w(bucket enabled proxy_download).freeze

  # To ensure the one Workhorse credential matches the Rails config, we
  # enforce consolidated settings on those accelerated
  # endpoints. Technically dependency_proxy and terraform_state fall
  # into this category, but they will likely be handled by Workhorse in
  # the future.
  WORKHORSE_ACCELERATED_TYPES = SUPPORTED_TYPES - %w(pages)

  # pages may be enabled but use legacy disk storage
  # we don't need to raise an error in that case
  ALLOWED_INCOMPLETE_TYPES = %w(pages).freeze

  # A fallback switch in case anyone gets a trouble with background upload removal
  # Epic: https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/734
  LEGACY_BACKGROUND_UPLOADS_ENV = "GITLAB_LEGACY_BACKGROUND_UPLOADS"

  attr_accessor :settings

  # Legacy parser
  def self.legacy_parse(object_store, object_store_type)
    object_store ||= Settingslogic.new({})
    object_store['enabled'] = false if object_store['enabled'].nil?
    object_store['remote_directory'], object_store['bucket_prefix'] = split_bucket_prefix(
      object_store['remote_directory']
    )

    if support_legacy_background_upload?(object_store_type)
      object_store['direct_upload'] = false
      object_store['background_upload'] = true
    else
      object_store['direct_upload'] = true
      object_store['background_upload'] = false
    end

    object_store['proxy_download'] = false if object_store['proxy_download'].nil?
    object_store['storage_options'] ||= {}

    # Convert upload connection settings to use string keys, to make Fog happy
    object_store['connection']&.deep_stringify_keys!
    object_store
  end

  def self.support_legacy_background_upload?(object_store_type)
    ENV[LEGACY_BACKGROUND_UPLOADS_ENV].to_s.split(',').map(&:strip).include?(object_store_type)
  end

  def self.split_bucket_prefix(bucket)
    return [nil, nil] unless bucket.present?

    # Strictly speaking, object storage keys are not Unix paths and
    # characters like '/' and '.' have no special meaning. But in practice,
    # we do treat them like paths, and somewhere along the line something or
    # somebody may turn '//' into '/' or try to resolve '/..'. To guard
    # against this we reject "bad" combinations of '/' and '.'.
    [%r{\A\.*/}, %r{/\.*/}, %r{/\.*\z}].each do |re|
      raise 'invalid bucket' if re.match(bucket)
    end

    bucket, prefix = bucket.split('/', 2)
    [bucket, prefix]
  end

  def initialize(settings)
    @settings = settings
  end

  # This method converts the common object storage settings to
  # the legacy, internal representation.
  #
  # For example, with the folowing YAML:
  #
  # object_store:
  #   enabled: true
  #   connection:
  #     provider: AWS
  #     aws_access_key_id: minio
  #     aws_secret_access_key: gdk-minio
  #     region: gdk
  #     endpoint: 'http://127.0.0.1:9000'
  #     path_style: true
  #   storage_options:
  #     server_side_encryption: AES256
  #   proxy_download: true
  #   objects:
  #     artifacts:
  #       bucket: artifacts
  #       proxy_download: false
  #     lfs:
  #       bucket: lfs-objects
  #
  # This method then will essentially call:
  #
  # Settings.artifacts['object_store'] = {
  #   "enabled" => true,
  #   "connection" => {
  #     "provider" => "AWS",
  #     "aws_access_key_id" => "minio",
  #     "aws_secret_access_key" => "gdk-minio",
  #     "region" => "gdk",
  #     "endpoint" => "http://127.0.0.1:9000",
  #     "path_style" => true
  #   },
  #   "storage_options" => {
  #     "server_side_encryption" => "AES256"
  #   },
  #   "direct_upload" => true,
  #   "background_upload" => false,
  #   "proxy_download" => false,
  #   "remote_directory" => "artifacts"
  # }
  #
  # Settings.lfs['object_store'] = {
  #   "enabled" => true,
  #   "connection" => {
  #     "provider" => "AWS",
  #     "aws_access_key_id" => "minio",
  #     "aws_secret_access_key" => "gdk-minio",
  #     "region" => "gdk",
  #     "endpoint" => "http://127.0.0.1:9000",
  #     "path_style" => true
  #   },
  #   "storage_options" => {
  #     "server_side_encryption" => "AES256"
  #   },
  #   "direct_upload" => true,
  #   "background_upload" => false,
  #   "proxy_download" => true,
  #   "remote_directory" => "lfs-objects"
  # }
  #
  # Note that with the common config:
  # 1. Only one object store credentials can now be used. This is
  #    necessary to limit configuration overhead when an object storage
  #    client (e.g. AWS S3) is used inside GitLab Workhorse.
  # 2. However, a bucket has to be specified for each object
  #    type. Reusing buckets is not really supported, but we don't
  #    enforce that yet.
  # 3. direct_upload and background_upload cannot be configured anymore.
  def parse!
    return unless use_consolidated_settings?

    main_config = settings['object_store']
    common_config = main_config.slice('enabled', 'connection', 'proxy_download', 'storage_options')
    # Convert connection settings to use string keys, to make Fog happy
    common_config['connection']&.deep_stringify_keys!
    # These are no longer configurable if common config is used
    common_config['direct_upload'] = true
    common_config['background_upload'] = false
    common_config['storage_options'] ||= {}

    SUPPORTED_TYPES.each do |store_type|
      overrides = main_config.dig('objects', store_type) || {}
      target_config = common_config.merge(overrides.slice(*ALLOWED_OBJECT_STORE_OVERRIDES))
      section = settings.try(store_type)

      # Admins can selectively disable object storage for a specific
      # type as an override in the consolidated settings.
      next unless overrides.fetch('enabled', true)
      next unless section

      if section['enabled'] && target_config['bucket'].blank?
        missing_bucket_for(store_type)
        next
      end

      # If a storage type such as Pages defines its own connection and does not
      # use Workhorse acceleration, we allow it to override the consolidated form.
      next if allowed_storage_specific_settings?(store_type, section.to_h)

      # Map bucket (external name) -> remote_directory (internal representation)
      target_config['remote_directory'], target_config['bucket_prefix'] = self.class.split_bucket_prefix(
        target_config.delete('bucket')
      )
      target_config['consolidated_settings'] = true
      section['object_store'] = target_config
      # Settingslogic internally stores data as a Hash, but it also
      # creates a Settings object for every key. To avoid confusion, we should
      # update both so that Settings.artifacts and Settings['artifacts'] return
      # the same result.
      settings[store_type]['object_store'] = target_config
    end

    settings
  end

  private

  # We only can use the common object storage settings if:
  # 1. The common settings are defined
  # 2. The legacy settings are not defined
  def use_consolidated_settings?
    return false unless settings.dig('object_store', 'enabled')
    return false unless settings.dig('object_store', 'connection').present?

    WORKHORSE_ACCELERATED_TYPES.each do |store|
      # to_h is needed because we define `default` as a Gitaly storage name
      # in stub_storage_settings. This causes Settingslogic to redefine Hash#default,
      # which causes Hash#dig to fail when the key doesn't exist: https://gitlab.com/gitlab-org/gitlab/-/issues/286873
      #
      # (byebug) section.dig
      # *** ArgumentError Exception: wrong number of arguments (given 0, expected 1+)
      # (byebug) section.dig('object_store')
      # *** ArgumentError Exception: wrong number of arguments (given 1, expected 0)
      section = settings.try(store)&.to_h

      next unless section

      return false if section.dig('object_store', 'enabled')
      # Omnibus defaults to an empty hash
      return false if section.dig('object_store', 'connection').present?
    end

    true
  end

  def missing_bucket_for(store_type)
    message = "Object storage for #{store_type} must have a bucket specified"

    if ALLOWED_INCOMPLETE_TYPES.include?(store_type)
      warn "[WARNING] #{message}"
    else
      raise message
    end
  end

  def allowed_storage_specific_settings?(store_type, section)
    return false if WORKHORSE_ACCELERATED_TYPES.include?(store_type)

    has_object_store_configured?(section)
  end

  def has_object_store_configured?(section)
    # Omnibus defaults to an empty hash for connection
    section.dig('object_store', 'enabled') && section.dig('object_store', 'connection').present?
  end
end