gitlab-org--gitlab-foss/app/uploaders/file_uploader.rb

223 lines
5.4 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
# This class breaks the actual CarrierWave concept.
# Every uploader should use a base_dir that is model agnostic so we can build
# back URLs from base_dir-relative paths saved in the `Upload` model.
#
# As the `.base_dir` is model dependent and **not** saved in the upload model (see #upload_path)
# there is no way to build back the correct file path without the model, which defies
# CarrierWave way of storing files.
#
2016-08-18 10:31:44 -04:00
class FileUploader < GitlabUploader
include UploaderHelper
include RecordsUploads::Concern
include ObjectStorage::Concern
prepend ObjectStorage::Extension::RecordsUploads
2019-05-05 06:19:14 -04:00
MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}.freeze
DYNAMIC_PATH_PATTERN = %r{.*(?<secret>\b(\h{10}|\h{32}))\/(?<identifier>.*)}.freeze
VALID_SECRET_PATTERN = %r{\A\h{10,32}\z}.freeze
InvalidSecret = Class.new(StandardError)
after :remove, :prune_store_dir
2018-02-21 11:43:21 -05:00
# FileUploader do not run in a model transaction, so we can simply
# enqueue a job after the :store hook.
after :store, :schedule_background_upload
def self.root
File.join(options.storage_path, 'uploads')
end
def self.absolute_path(upload)
File.join(
root,
relative_path(upload)
)
end
def self.relative_path(upload)
File.join(
base_dir(upload.model),
upload.path # already contain the dynamic_segment, see #upload_path
)
end
def self.base_dir(model, store = Store::LOCAL)
decorated_model = model
decorated_model = Storage::Hashed.new(model) if store == Store::REMOTE
model_path_segment(decorated_model)
end
# used in migrations and import/exports
def self.absolute_base_dir(model)
File.join(root, base_dir(model))
end
# Returns the part of `store_dir` that can change based on the model's current
# path
#
# This is used to build Upload paths dynamically based on the model's current
# namespace and path, allowing us to ignore renames or transfers.
#
# model - Object that responds to `full_path` and `disk_path`
#
# Returns a String without a trailing slash
def self.model_path_segment(model)
case model
when Storage::Hashed then model.disk_path
else
model.hashed_storage?(:attachments) ? model.disk_path : model.full_path
end
end
def self.generate_secret
SecureRandom.hex
end
def self.extract_dynamic_path(path)
DYNAMIC_PATH_PATTERN.match(path)
end
2018-06-06 16:45:42 -04:00
def upload_paths(identifier)
[
2018-06-06 16:45:42 -04:00
File.join(secret, identifier),
File.join(base_dir(Store::REMOTE), secret, identifier)
]
end
attr_accessor :model
def initialize(model, mounted_as = nil, **uploader_context)
super(model, nil, **uploader_context)
@model = model
apply_context!(uploader_context)
end
def initialize_copy(from)
super
@secret = self.class.generate_secret
@upload = nil # calling record_upload would delete the old upload if set
end
# enforce the usage of Hashed storage when storing to
# remote store as the FileMover doesn't support OS
def base_dir(store = nil)
self.class.base_dir(@model, store || object_store)
end
# we don't need to know the actual path, an uploader instance should be
# able to yield the file content on demand, so we should build the digest
def absolute_path
self.class.absolute_path(@upload)
end
def upload_path
if file_storage?
# Legacy path relative to project.full_path
Fix incorrect prefix used in new uploads for personal snippets https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24550 fixed the case where the wrong path on disk was being searched, but it inadvertently ommitted the `/uploads/-/system` prefix when rendering the Markdown for personal snippet uploads when they were stored directly in object storage. A personal snippet path is stored using FileUploader#upload_path. The format for the path: Local storage: :random_hex/:filename. Object storage: personal_snippet/:id/:random_hex/:filename. upload_paths represent the possible paths for a given identifier, which will vary depending on whether the file is stored in local or object storage. upload_path should match an element in upload_paths. base_dir represents the path seen by the user in Markdown, and it should always be prefixed with uploads/-/system. store_dirs represent the paths that are actually used on disk. For object storage, this should omit the prefix /uploads/-/system. For example, consider the requested path /uploads/-/system/personal_snippet/172/ff4ad5c2/file.png. For local storage: base_dir: uploads/-/system/personal_snippet/172 upload_path: ff4ad5c2/file.png upload_paths: ["ff4ad5c2/file.png", "personal_snippet/172/ff4ad5c2/file.png"]. store_dirs: {1=>"uploads/-/system/personal_snippet/172/ff4ad5c2", 2=>"personal_snippet/172/ff4ad5c2"} For object storage: upload_path: personal_snippet/172/ff4ad5c2/file.png Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/61671
2019-05-15 12:26:18 -04:00
local_storage_path(identifier)
else
Fix incorrect prefix used in new uploads for personal snippets https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24550 fixed the case where the wrong path on disk was being searched, but it inadvertently ommitted the `/uploads/-/system` prefix when rendering the Markdown for personal snippet uploads when they were stored directly in object storage. A personal snippet path is stored using FileUploader#upload_path. The format for the path: Local storage: :random_hex/:filename. Object storage: personal_snippet/:id/:random_hex/:filename. upload_paths represent the possible paths for a given identifier, which will vary depending on whether the file is stored in local or object storage. upload_path should match an element in upload_paths. base_dir represents the path seen by the user in Markdown, and it should always be prefixed with uploads/-/system. store_dirs represent the paths that are actually used on disk. For object storage, this should omit the prefix /uploads/-/system. For example, consider the requested path /uploads/-/system/personal_snippet/172/ff4ad5c2/file.png. For local storage: base_dir: uploads/-/system/personal_snippet/172 upload_path: ff4ad5c2/file.png upload_paths: ["ff4ad5c2/file.png", "personal_snippet/172/ff4ad5c2/file.png"]. store_dirs: {1=>"uploads/-/system/personal_snippet/172/ff4ad5c2", 2=>"personal_snippet/172/ff4ad5c2"} For object storage: upload_path: personal_snippet/172/ff4ad5c2/file.png Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/61671
2019-05-15 12:26:18 -04:00
remote_storage_path(identifier)
end
end
2016-01-08 11:38:53 -05:00
Fix incorrect prefix used in new uploads for personal snippets https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24550 fixed the case where the wrong path on disk was being searched, but it inadvertently ommitted the `/uploads/-/system` prefix when rendering the Markdown for personal snippet uploads when they were stored directly in object storage. A personal snippet path is stored using FileUploader#upload_path. The format for the path: Local storage: :random_hex/:filename. Object storage: personal_snippet/:id/:random_hex/:filename. upload_paths represent the possible paths for a given identifier, which will vary depending on whether the file is stored in local or object storage. upload_path should match an element in upload_paths. base_dir represents the path seen by the user in Markdown, and it should always be prefixed with uploads/-/system. store_dirs represent the paths that are actually used on disk. For object storage, this should omit the prefix /uploads/-/system. For example, consider the requested path /uploads/-/system/personal_snippet/172/ff4ad5c2/file.png. For local storage: base_dir: uploads/-/system/personal_snippet/172 upload_path: ff4ad5c2/file.png upload_paths: ["ff4ad5c2/file.png", "personal_snippet/172/ff4ad5c2/file.png"]. store_dirs: {1=>"uploads/-/system/personal_snippet/172/ff4ad5c2", 2=>"personal_snippet/172/ff4ad5c2"} For object storage: upload_path: personal_snippet/172/ff4ad5c2/file.png Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/61671
2019-05-15 12:26:18 -04:00
def local_storage_path(file_identifier)
File.join(dynamic_segment, file_identifier)
end
def remote_storage_path(file_identifier)
File.join(store_dir, file_identifier)
end
def store_dirs
{
Store::LOCAL => File.join(base_dir, dynamic_segment),
Store::REMOTE => File.join(base_dir(ObjectStorage::Store::REMOTE), dynamic_segment)
}
end
def to_h
2016-01-08 11:38:53 -05:00
{
alt: markdown_name,
url: secure_url,
markdown: markdown_link
2016-01-08 11:38:53 -05:00
}
end
def upload=(value)
2018-01-29 12:57:34 -05:00
super
2018-01-30 14:38:10 -05:00
return unless value
return if apply_context!(value.uploader_context)
# fallback to the regex based extraction
if matches = self.class.extract_dynamic_path(value.path)
@secret = matches[:secret]
@identifier = matches[:identifier]
end
end
def secret
@secret ||= self.class.generate_secret
raise InvalidSecret unless @secret =~ VALID_SECRET_PATTERN
@secret
end
# return a new uploader with a file copy on another project
def self.copy_to(uploader, to_project)
moved = self.new(to_project)
moved.object_store = uploader.object_store
moved.filename = uploader.filename
moved.copy_file(uploader.file)
moved
end
def copy_file(file)
2018-06-28 11:25:40 -04:00
to_path = if file_storage?
File.join(self.class.root, store_path)
else
store_path
end
self.file = file.copy_to(to_path)
record_upload # after_store is not triggered
end
private
def apply_context!(uploader_context)
@secret, @identifier = uploader_context.values_at(:secret, :identifier)
!!(@secret && @identifier)
end
def build_upload
super.tap do |upload|
upload.secret = secret
end
end
def prune_store_dir
storage.delete_dir!(store_dir) # only remove when empty
end
def identifier
@identifier ||= filename
end
def dynamic_segment
secret
end
def secure_url
File.join('/uploads', @secret, filename)
end
end