2017-11-07 20:53:24 +00:00
|
|
|
module Gitlab
|
|
|
|
module BackgroundMigration
|
|
|
|
class PopulateUntrackedUploads
|
2017-11-10 01:17:56 +00:00
|
|
|
class UntrackedFile < ActiveRecord::Base
|
|
|
|
self.table_name = 'untracked_files_for_uploads'
|
2017-11-07 20:53:24 +00:00
|
|
|
|
2017-11-08 03:08:02 +00:00
|
|
|
# Ends with /:random_hex/:filename
|
2017-11-08 20:44:49 +00:00
|
|
|
FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z}
|
|
|
|
FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}}
|
2017-11-08 03:08:02 +00:00
|
|
|
|
|
|
|
# These regex patterns are tested against a relative path, relative to
|
|
|
|
# the upload directory.
|
|
|
|
# For convenience, if there exists a capture group in the pattern, then
|
|
|
|
# it indicates the model_id.
|
|
|
|
PATH_PATTERNS = [
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/appearance/logo/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AttachmentUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'Appearance'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AttachmentUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'Appearance'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/note/attachment/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AttachmentUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'Note'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/user/avatar/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AvatarUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'User'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/group/avatar/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AvatarUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'Namespace'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
2017-11-08 20:44:49 +00:00
|
|
|
pattern: %r{\A-/system/project/avatar/(\d+)/},
|
2017-11-08 03:08:02 +00:00
|
|
|
uploader: 'AvatarUploader',
|
2017-11-08 04:54:54 +00:00
|
|
|
model_type: 'Project'
|
2017-11-08 03:08:02 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
pattern: FILE_UPLOADER_PATH_PATTERN,
|
|
|
|
uploader: 'FileUploader',
|
|
|
|
model_type: 'Project'
|
2017-11-08 04:54:54 +00:00
|
|
|
}
|
|
|
|
].freeze
|
2017-11-08 03:08:02 +00:00
|
|
|
|
2017-11-24 07:12:24 +00:00
|
|
|
def to_h
|
|
|
|
{
|
|
|
|
path: upload_path,
|
|
|
|
uploader: uploader,
|
|
|
|
model_type: model_type,
|
|
|
|
model_id: model_id,
|
2017-11-24 07:49:16 +00:00
|
|
|
size: file_size,
|
|
|
|
checksum: checksum
|
2017-11-24 07:12:24 +00:00
|
|
|
}
|
2017-11-08 03:08:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def upload_path
|
2017-11-10 01:17:56 +00:00
|
|
|
# UntrackedFile#path is absolute, but Upload#path depends on uploader
|
2017-11-24 07:12:24 +00:00
|
|
|
@upload_path ||= if uploader == 'FileUploader'
|
|
|
|
# Path relative to project directory in uploads
|
|
|
|
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN)
|
|
|
|
matchd[0].sub(%r{\A/}, '') # remove leading slash
|
|
|
|
else
|
|
|
|
path
|
|
|
|
end
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def uploader
|
2017-11-08 04:15:28 +00:00
|
|
|
matching_pattern_map[:uploader]
|
2017-11-08 03:08:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def model_type
|
2017-11-08 04:15:28 +00:00
|
|
|
matching_pattern_map[:model_type]
|
2017-11-08 03:08:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def model_id
|
2017-11-24 08:49:04 +00:00
|
|
|
return @model_id if defined?(@model_id)
|
|
|
|
|
2017-11-08 04:15:28 +00:00
|
|
|
matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern])
|
2017-11-08 03:08:02 +00:00
|
|
|
|
2017-11-08 04:15:28 +00:00
|
|
|
# If something is captured (matchd[1] is not nil), it is a model_id
|
2017-11-08 03:08:02 +00:00
|
|
|
# Only the FileUploader pattern will not match an ID
|
2017-11-24 08:49:04 +00:00
|
|
|
@model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
|
2017-11-08 03:08:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def file_size
|
2017-11-15 00:11:53 +00:00
|
|
|
File.size(absolute_path)
|
2017-11-08 03:08:02 +00:00
|
|
|
end
|
|
|
|
|
2017-11-24 07:49:16 +00:00
|
|
|
def checksum
|
|
|
|
Digest::SHA256.file(absolute_path).hexdigest
|
|
|
|
end
|
|
|
|
|
2017-11-08 03:08:02 +00:00
|
|
|
private
|
|
|
|
|
2017-11-08 04:15:28 +00:00
|
|
|
def matching_pattern_map
|
|
|
|
@matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
|
|
|
|
path_relative_to_upload_dir.match(path_pattern_map[:pattern])
|
|
|
|
end
|
|
|
|
|
|
|
|
raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map
|
|
|
|
|
|
|
|
@matching_pattern_map
|
|
|
|
end
|
|
|
|
|
2017-11-08 03:08:02 +00:00
|
|
|
def file_uploader_model_id
|
2017-11-08 20:44:49 +00:00
|
|
|
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN)
|
2017-11-08 03:08:02 +00:00
|
|
|
raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd
|
2017-11-22 00:05:33 +00:00
|
|
|
|
2017-11-08 03:08:02 +00:00
|
|
|
full_path = matchd[1]
|
|
|
|
project = Project.find_by_full_path(full_path)
|
2017-11-24 08:49:04 +00:00
|
|
|
return nil unless project
|
|
|
|
|
|
|
|
project.id
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
2017-11-08 03:54:28 +00:00
|
|
|
|
2017-11-24 08:52:16 +00:00
|
|
|
# Not including a leading slash
|
|
|
|
def path_relative_to_upload_dir
|
|
|
|
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/}
|
|
|
|
@path_relative_to_upload_dir ||= path.sub(base, '')
|
|
|
|
end
|
|
|
|
|
2017-11-08 03:54:28 +00:00
|
|
|
def absolute_path
|
2017-11-24 07:49:16 +00:00
|
|
|
File.join(CarrierWave.root, path)
|
2017-11-08 03:54:28 +00:00
|
|
|
end
|
2017-11-24 07:49:16 +00:00
|
|
|
end
|
2017-11-08 03:54:28 +00:00
|
|
|
|
2017-11-24 07:49:16 +00:00
|
|
|
class Upload < ActiveRecord::Base
|
|
|
|
self.table_name = 'uploads'
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def perform(start_id, end_id)
|
|
|
|
return unless migrate?
|
|
|
|
|
2017-11-22 18:44:33 +00:00
|
|
|
files = UntrackedFile.where(id: start_id..end_id)
|
2017-11-24 07:12:24 +00:00
|
|
|
insert_uploads_if_needed(files)
|
|
|
|
files.delete_all
|
2017-11-15 10:36:25 +00:00
|
|
|
|
|
|
|
drop_temp_table_if_finished
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
def migrate?
|
2017-11-10 01:17:56 +00:00
|
|
|
UntrackedFile.table_exists? && Upload.table_exists?
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
2017-11-15 10:36:25 +00:00
|
|
|
|
2017-11-24 07:12:24 +00:00
|
|
|
def insert_uploads_if_needed(files)
|
|
|
|
filtered_files = filter_existing_uploads(files)
|
|
|
|
filtered_files = filter_deleted_models(filtered_files)
|
|
|
|
insert(filtered_files)
|
|
|
|
end
|
|
|
|
|
|
|
|
def filter_existing_uploads(files)
|
|
|
|
paths = files.map(&:upload_path)
|
|
|
|
existing_paths = Upload.where(path: paths).pluck(:path).to_set
|
|
|
|
|
|
|
|
files.reject do |file|
|
|
|
|
existing_paths.include?(file.upload_path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-11-24 08:49:04 +00:00
|
|
|
# There are files on disk that are not in the uploads table because their
|
|
|
|
# model was deleted, and we don't delete the files on disk.
|
2017-11-24 07:12:24 +00:00
|
|
|
def filter_deleted_models(files)
|
2017-11-24 08:49:04 +00:00
|
|
|
ids = deleted_model_ids(files)
|
|
|
|
|
|
|
|
files.reject do |file|
|
|
|
|
ids[file.model_type].include?(file.model_id)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def deleted_model_ids(files)
|
|
|
|
ids = {
|
|
|
|
'Appearance' => [],
|
|
|
|
'Namespace' => [],
|
|
|
|
'Note' => [],
|
|
|
|
'Project' => [],
|
|
|
|
'User' => []
|
|
|
|
}
|
|
|
|
|
|
|
|
# group model IDs by model type
|
|
|
|
files.each do |file|
|
|
|
|
ids[file.model_type] << file.model_id
|
|
|
|
end
|
|
|
|
|
|
|
|
ids.each do |model_type, model_ids|
|
|
|
|
found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id)
|
|
|
|
ids[model_type] = ids[model_type] - found_ids # replace with deleted ids
|
|
|
|
end
|
|
|
|
|
|
|
|
ids
|
2017-11-24 07:12:24 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def insert(files)
|
2017-11-24 07:49:16 +00:00
|
|
|
rows = files.map do |file|
|
|
|
|
file.to_h.merge(created_at: 'NOW()')
|
2017-11-24 07:12:24 +00:00
|
|
|
end
|
2017-11-24 07:49:16 +00:00
|
|
|
|
2017-11-24 18:04:44 +00:00
|
|
|
Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at)
|
2017-11-24 07:12:24 +00:00
|
|
|
end
|
|
|
|
|
2017-11-15 10:36:25 +00:00
|
|
|
def drop_temp_table_if_finished
|
2017-11-22 18:44:33 +00:00
|
|
|
UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty?
|
2017-11-15 10:36:25 +00:00
|
|
|
end
|
2017-11-07 20:53:24 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|