2021-02-03 16:09:17 -05:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2018-07-23 10:28:29 -04:00
|
|
|
# Set default values for object_store settings
|
|
|
|
class ObjectStoreSettings
|
2022-01-13 16:14:07 -05:00
|
|
|
SUPPORTED_TYPES = %w(artifacts external_diffs lfs uploads packages dependency_proxy terraform_state pages secure_files).freeze
|
2020-06-17 11:08:36 -04:00
|
|
|
ALLOWED_OBJECT_STORE_OVERRIDES = %w(bucket enabled proxy_download).freeze
|
|
|
|
|
2020-11-19 19:09:06 -05:00
|
|
|
# To ensure the one Workhorse credential matches the Rails config, we
|
|
|
|
# enforce consolidated settings on those accelerated
|
|
|
|
# endpoints. Technically dependency_proxy and terraform_state fall
|
|
|
|
# into this category, but they will likely be handled by Workhorse in
|
|
|
|
# the future.
|
|
|
|
WORKHORSE_ACCELERATED_TYPES = SUPPORTED_TYPES - %w(pages)
|
|
|
|
|
2020-10-06 14:08:49 -04:00
|
|
|
# pages may be enabled but use legacy disk storage
|
|
|
|
# we don't need to raise an error in that case
|
|
|
|
ALLOWED_INCOMPLETE_TYPES = %w(pages).freeze
|
|
|
|
|
2022-05-09 17:07:53 -04:00
|
|
|
# A fallback switch in case anyone gets a trouble with background upload removal
|
|
|
|
# Epic: https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/734
|
|
|
|
LEGACY_BACKGROUND_UPLOADS_ENV = "GITLAB_LEGACY_BACKGROUND_UPLOADS"
|
|
|
|
|
2020-06-17 11:08:36 -04:00
|
|
|
attr_accessor :settings
|
|
|
|
|
|
|
|
# Legacy parser
|
2022-05-09 17:07:53 -04:00
|
|
|
def self.legacy_parse(object_store, object_store_type)
|
2018-07-23 10:28:29 -04:00
|
|
|
object_store ||= Settingslogic.new({})
|
|
|
|
object_store['enabled'] = false if object_store['enabled'].nil?
|
2022-07-01 08:08:08 -04:00
|
|
|
object_store['remote_directory'], object_store['bucket_prefix'] = split_bucket_prefix(
|
|
|
|
object_store['remote_directory']
|
|
|
|
)
|
2022-05-09 17:07:53 -04:00
|
|
|
|
|
|
|
if support_legacy_background_upload?(object_store_type)
|
|
|
|
object_store['direct_upload'] = false
|
|
|
|
object_store['background_upload'] = true
|
|
|
|
else
|
|
|
|
object_store['direct_upload'] = true
|
|
|
|
object_store['background_upload'] = false
|
|
|
|
end
|
|
|
|
|
2018-07-23 10:28:29 -04:00
|
|
|
object_store['proxy_download'] = false if object_store['proxy_download'].nil?
|
2020-08-05 05:10:01 -04:00
|
|
|
object_store['storage_options'] ||= {}
|
2018-07-23 10:33:06 -04:00
|
|
|
|
2018-07-23 10:28:29 -04:00
|
|
|
# Convert upload connection settings to use string keys, to make Fog happy
|
|
|
|
object_store['connection']&.deep_stringify_keys!
|
2018-07-23 10:33:06 -04:00
|
|
|
object_store
|
2018-07-23 10:28:29 -04:00
|
|
|
end
|
2020-06-17 11:08:36 -04:00
|
|
|
|
2022-05-09 17:07:53 -04:00
|
|
|
def self.support_legacy_background_upload?(object_store_type)
|
|
|
|
ENV[LEGACY_BACKGROUND_UPLOADS_ENV].to_s.split(',').map(&:strip).include?(object_store_type)
|
|
|
|
end
|
|
|
|
|
2022-07-01 08:08:08 -04:00
|
|
|
def self.split_bucket_prefix(bucket)
|
|
|
|
return [nil, nil] unless bucket.present?
|
|
|
|
|
|
|
|
# Strictly speaking, object storage keys are not Unix paths and
|
|
|
|
# characters like '/' and '.' have no special meaning. But in practice,
|
|
|
|
# we do treat them like paths, and somewhere along the line something or
|
|
|
|
# somebody may turn '//' into '/' or try to resolve '/..'. To guard
|
|
|
|
# against this we reject "bad" combinations of '/' and '.'.
|
|
|
|
[%r{\A\.*/}, %r{/\.*/}, %r{/\.*\z}].each do |re|
|
|
|
|
raise 'invalid bucket' if re.match(bucket)
|
|
|
|
end
|
|
|
|
|
|
|
|
bucket, prefix = bucket.split('/', 2)
|
|
|
|
[bucket, prefix]
|
|
|
|
end
|
|
|
|
|
2020-06-17 11:08:36 -04:00
|
|
|
def initialize(settings)
|
|
|
|
@settings = settings
|
|
|
|
end
|
|
|
|
|
|
|
|
# This method converts the common object storage settings to
|
|
|
|
# the legacy, internal representation.
|
|
|
|
#
|
|
|
|
# For example, with the folowing YAML:
|
|
|
|
#
|
|
|
|
# object_store:
|
|
|
|
# enabled: true
|
|
|
|
# connection:
|
|
|
|
# provider: AWS
|
|
|
|
# aws_access_key_id: minio
|
|
|
|
# aws_secret_access_key: gdk-minio
|
|
|
|
# region: gdk
|
|
|
|
# endpoint: 'http://127.0.0.1:9000'
|
|
|
|
# path_style: true
|
2020-08-05 05:10:01 -04:00
|
|
|
# storage_options:
|
|
|
|
# server_side_encryption: AES256
|
2020-06-17 11:08:36 -04:00
|
|
|
# proxy_download: true
|
|
|
|
# objects:
|
|
|
|
# artifacts:
|
|
|
|
# bucket: artifacts
|
|
|
|
# proxy_download: false
|
|
|
|
# lfs:
|
|
|
|
# bucket: lfs-objects
|
|
|
|
#
|
|
|
|
# This method then will essentially call:
|
|
|
|
#
|
|
|
|
# Settings.artifacts['object_store'] = {
|
|
|
|
# "enabled" => true,
|
2020-08-05 05:10:01 -04:00
|
|
|
# "connection" => {
|
2020-06-17 11:08:36 -04:00
|
|
|
# "provider" => "AWS",
|
|
|
|
# "aws_access_key_id" => "minio",
|
|
|
|
# "aws_secret_access_key" => "gdk-minio",
|
|
|
|
# "region" => "gdk",
|
|
|
|
# "endpoint" => "http://127.0.0.1:9000",
|
|
|
|
# "path_style" => true
|
|
|
|
# },
|
2020-08-05 05:10:01 -04:00
|
|
|
# "storage_options" => {
|
|
|
|
# "server_side_encryption" => "AES256"
|
|
|
|
# },
|
2020-06-17 11:08:36 -04:00
|
|
|
# "direct_upload" => true,
|
|
|
|
# "background_upload" => false,
|
|
|
|
# "proxy_download" => false,
|
|
|
|
# "remote_directory" => "artifacts"
|
2021-01-29 13:09:17 -05:00
|
|
|
# }
|
2020-06-17 11:08:36 -04:00
|
|
|
#
|
|
|
|
# Settings.lfs['object_store'] = {
|
|
|
|
# "enabled" => true,
|
|
|
|
# "connection" => {
|
|
|
|
# "provider" => "AWS",
|
|
|
|
# "aws_access_key_id" => "minio",
|
|
|
|
# "aws_secret_access_key" => "gdk-minio",
|
|
|
|
# "region" => "gdk",
|
|
|
|
# "endpoint" => "http://127.0.0.1:9000",
|
|
|
|
# "path_style" => true
|
|
|
|
# },
|
2020-08-05 05:10:01 -04:00
|
|
|
# "storage_options" => {
|
|
|
|
# "server_side_encryption" => "AES256"
|
|
|
|
# },
|
2020-06-17 11:08:36 -04:00
|
|
|
# "direct_upload" => true,
|
|
|
|
# "background_upload" => false,
|
|
|
|
# "proxy_download" => true,
|
|
|
|
# "remote_directory" => "lfs-objects"
|
2021-01-29 13:09:17 -05:00
|
|
|
# }
|
2020-06-17 11:08:36 -04:00
|
|
|
#
|
|
|
|
# Note that with the common config:
|
|
|
|
# 1. Only one object store credentials can now be used. This is
|
|
|
|
# necessary to limit configuration overhead when an object storage
|
|
|
|
# client (e.g. AWS S3) is used inside GitLab Workhorse.
|
|
|
|
# 2. However, a bucket has to be specified for each object
|
|
|
|
# type. Reusing buckets is not really supported, but we don't
|
|
|
|
# enforce that yet.
|
|
|
|
# 3. direct_upload and background_upload cannot be configured anymore.
|
|
|
|
def parse!
|
|
|
|
return unless use_consolidated_settings?
|
|
|
|
|
|
|
|
main_config = settings['object_store']
|
2020-08-05 05:10:01 -04:00
|
|
|
common_config = main_config.slice('enabled', 'connection', 'proxy_download', 'storage_options')
|
2020-06-17 11:08:36 -04:00
|
|
|
# Convert connection settings to use string keys, to make Fog happy
|
|
|
|
common_config['connection']&.deep_stringify_keys!
|
|
|
|
# These are no longer configurable if common config is used
|
|
|
|
common_config['direct_upload'] = true
|
|
|
|
common_config['background_upload'] = false
|
2020-08-05 05:10:01 -04:00
|
|
|
common_config['storage_options'] ||= {}
|
2020-06-17 11:08:36 -04:00
|
|
|
|
|
|
|
SUPPORTED_TYPES.each do |store_type|
|
|
|
|
overrides = main_config.dig('objects', store_type) || {}
|
|
|
|
target_config = common_config.merge(overrides.slice(*ALLOWED_OBJECT_STORE_OVERRIDES))
|
|
|
|
section = settings.try(store_type)
|
|
|
|
|
2021-02-03 13:09:25 -05:00
|
|
|
# Admins can selectively disable object storage for a specific
|
|
|
|
# type as an override in the consolidated settings.
|
|
|
|
next unless overrides.fetch('enabled', true)
|
|
|
|
next unless section
|
2020-06-17 11:08:36 -04:00
|
|
|
|
2021-02-03 13:09:25 -05:00
|
|
|
if section['enabled'] && target_config['bucket'].blank?
|
2020-10-06 14:08:49 -04:00
|
|
|
missing_bucket_for(store_type)
|
2020-12-04 16:09:29 -05:00
|
|
|
next
|
2020-10-06 14:08:49 -04:00
|
|
|
end
|
2020-06-17 11:08:36 -04:00
|
|
|
|
2020-11-19 19:09:06 -05:00
|
|
|
# If a storage type such as Pages defines its own connection and does not
|
|
|
|
# use Workhorse acceleration, we allow it to override the consolidated form.
|
2020-11-20 04:09:06 -05:00
|
|
|
next if allowed_storage_specific_settings?(store_type, section.to_h)
|
2020-11-19 19:09:06 -05:00
|
|
|
|
2020-06-17 11:08:36 -04:00
|
|
|
# Map bucket (external name) -> remote_directory (internal representation)
|
2022-07-01 08:08:08 -04:00
|
|
|
target_config['remote_directory'], target_config['bucket_prefix'] = self.class.split_bucket_prefix(
|
|
|
|
target_config.delete('bucket')
|
|
|
|
)
|
2020-07-03 08:08:53 -04:00
|
|
|
target_config['consolidated_settings'] = true
|
2020-06-17 11:08:36 -04:00
|
|
|
section['object_store'] = target_config
|
2021-04-29 11:10:07 -04:00
|
|
|
# Settingslogic internally stores data as a Hash, but it also
|
|
|
|
# creates a Settings object for every key. To avoid confusion, we should
|
|
|
|
# update both so that Settings.artifacts and Settings['artifacts'] return
|
|
|
|
# the same result.
|
|
|
|
settings[store_type]['object_store'] = target_config
|
2020-06-17 11:08:36 -04:00
|
|
|
end
|
2021-01-29 13:09:17 -05:00
|
|
|
|
|
|
|
settings
|
2020-06-17 11:08:36 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
# We only can use the common object storage settings if:
|
|
|
|
# 1. The common settings are defined
|
|
|
|
# 2. The legacy settings are not defined
|
|
|
|
def use_consolidated_settings?
|
|
|
|
return false unless settings.dig('object_store', 'enabled')
|
2020-07-03 08:08:53 -04:00
|
|
|
return false unless settings.dig('object_store', 'connection').present?
|
2020-06-17 11:08:36 -04:00
|
|
|
|
2020-11-19 19:09:06 -05:00
|
|
|
WORKHORSE_ACCELERATED_TYPES.each do |store|
|
2021-01-29 13:09:17 -05:00
|
|
|
# to_h is needed because we define `default` as a Gitaly storage name
|
|
|
|
# in stub_storage_settings. This causes Settingslogic to redefine Hash#default,
|
|
|
|
# which causes Hash#dig to fail when the key doesn't exist: https://gitlab.com/gitlab-org/gitlab/-/issues/286873
|
2020-06-17 11:08:36 -04:00
|
|
|
#
|
|
|
|
# (byebug) section.dig
|
|
|
|
# *** ArgumentError Exception: wrong number of arguments (given 0, expected 1+)
|
|
|
|
# (byebug) section.dig('object_store')
|
|
|
|
# *** ArgumentError Exception: wrong number of arguments (given 1, expected 0)
|
|
|
|
section = settings.try(store)&.to_h
|
|
|
|
|
|
|
|
next unless section
|
|
|
|
|
|
|
|
return false if section.dig('object_store', 'enabled')
|
2020-07-03 08:08:53 -04:00
|
|
|
# Omnibus defaults to an empty hash
|
|
|
|
return false if section.dig('object_store', 'connection').present?
|
2020-06-17 11:08:36 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
true
|
|
|
|
end
|
2020-10-06 14:08:49 -04:00
|
|
|
|
|
|
|
def missing_bucket_for(store_type)
|
|
|
|
message = "Object storage for #{store_type} must have a bucket specified"
|
|
|
|
|
|
|
|
if ALLOWED_INCOMPLETE_TYPES.include?(store_type)
|
|
|
|
warn "[WARNING] #{message}"
|
|
|
|
else
|
|
|
|
raise message
|
|
|
|
end
|
|
|
|
end
|
2020-11-19 19:09:06 -05:00
|
|
|
|
|
|
|
def allowed_storage_specific_settings?(store_type, section)
|
|
|
|
return false if WORKHORSE_ACCELERATED_TYPES.include?(store_type)
|
|
|
|
|
|
|
|
has_object_store_configured?(section)
|
|
|
|
end
|
|
|
|
|
|
|
|
def has_object_store_configured?(section)
|
|
|
|
# Omnibus defaults to an empty hash for connection
|
|
|
|
section.dig('object_store', 'enabled') && section.dig('object_store', 'connection').present?
|
|
|
|
end
|
2018-07-23 10:28:29 -04:00
|
|
|
end
|