2018-11-17 00:37:17 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
module Gitlab
|
|
|
|
module ImportExport
|
|
|
|
class FileImporter
|
|
|
|
include Gitlab::ImportExport::CommandLineUtil
|
|
|
|
|
2019-12-10 07:53:40 +00:00
|
|
|
ImporterError = Class.new(StandardError)
|
|
|
|
|
2016-07-27 15:56:25 +00:00
|
|
|
MAX_RETRIES = 8
|
2018-07-12 12:56:15 +00:00
|
|
|
IGNORED_FILENAMES = %w(. ..).freeze
|
2016-07-27 15:56:25 +00:00
|
|
|
|
2020-10-15 18:08:43 +00:00
|
|
|
def self.import(*args, **kwargs)
|
|
|
|
new(*args, **kwargs).import
|
2016-06-14 10:47:07 +00:00
|
|
|
end
|
|
|
|
|
2019-12-10 07:53:40 +00:00
|
|
|
def initialize(importable:, archive_file:, shared:)
|
|
|
|
@importable = importable
|
2016-06-14 10:47:07 +00:00
|
|
|
@archive_file = archive_file
|
|
|
|
@shared = shared
|
|
|
|
end
|
|
|
|
|
|
|
|
def import
|
2016-09-29 15:17:22 +00:00
|
|
|
mkdir_p(@shared.export_path)
|
2018-07-26 10:55:21 +00:00
|
|
|
mkdir_p(@shared.archive_path)
|
2016-07-27 15:56:25 +00:00
|
|
|
|
2018-07-26 10:55:21 +00:00
|
|
|
remove_symlinks
|
|
|
|
copy_archive
|
2018-01-08 15:42:41 +00:00
|
|
|
|
2016-07-27 15:56:25 +00:00
|
|
|
wait_for_archived_file do
|
2022-05-06 15:09:03 +00:00
|
|
|
validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size)
|
2016-07-27 15:56:25 +00:00
|
|
|
decompress_archive
|
|
|
|
end
|
2021-04-26 12:09:44 +00:00
|
|
|
rescue StandardError => e
|
2016-06-14 10:47:07 +00:00
|
|
|
@shared.error(e)
|
|
|
|
false
|
2018-01-08 15:42:41 +00:00
|
|
|
ensure
|
2018-07-26 10:55:21 +00:00
|
|
|
remove_import_file
|
|
|
|
remove_symlinks
|
2016-06-14 10:47:07 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2016-07-27 15:56:25 +00:00
|
|
|
# Exponentially sleep until I/O finishes copying the file
|
|
|
|
def wait_for_archived_file
|
|
|
|
MAX_RETRIES.times do |retry_number|
|
2016-08-01 09:07:06 +00:00
|
|
|
break if File.exist?(@archive_file)
|
2016-07-27 15:56:25 +00:00
|
|
|
|
2016-08-01 09:07:06 +00:00
|
|
|
sleep(2**retry_number)
|
2016-07-27 15:56:25 +00:00
|
|
|
end
|
2016-08-01 09:07:06 +00:00
|
|
|
|
|
|
|
yield
|
2016-07-27 15:56:25 +00:00
|
|
|
end
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
def decompress_archive
|
2016-06-24 08:50:23 +00:00
|
|
|
result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
|
|
|
|
|
2021-05-04 15:10:36 +00:00
|
|
|
raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result
|
2016-06-24 08:50:23 +00:00
|
|
|
|
2018-01-08 15:42:41 +00:00
|
|
|
result
|
2016-10-27 15:10:19 +00:00
|
|
|
end
|
|
|
|
|
2018-07-26 10:55:21 +00:00
|
|
|
def copy_archive
|
|
|
|
return if @archive_file
|
|
|
|
|
2019-12-10 07:53:40 +00:00
|
|
|
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable))
|
2018-07-26 10:55:21 +00:00
|
|
|
|
2021-05-05 21:09:59 +00:00
|
|
|
remote_download_or_download_or_copy_upload
|
|
|
|
end
|
|
|
|
|
|
|
|
def remote_download_or_download_or_copy_upload
|
|
|
|
import_export_upload = @importable.import_export_upload
|
|
|
|
|
|
|
|
if import_export_upload.remote_import_url.present?
|
2022-03-07 06:15:33 +00:00
|
|
|
download(
|
|
|
|
import_export_upload.remote_import_url,
|
|
|
|
@archive_file,
|
|
|
|
size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT
|
|
|
|
)
|
2021-05-05 21:09:59 +00:00
|
|
|
else
|
2022-03-07 06:15:33 +00:00
|
|
|
download_or_copy_upload(
|
|
|
|
import_export_upload.import_file,
|
|
|
|
@archive_file,
|
|
|
|
size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT
|
|
|
|
)
|
2021-05-05 21:09:59 +00:00
|
|
|
end
|
2018-07-26 10:55:21 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def remove_symlinks
|
2017-08-08 21:18:02 +00:00
|
|
|
extracted_files.each do |path|
|
2016-10-27 15:10:19 +00:00
|
|
|
FileUtils.rm(path) if File.lstat(path).symlink?
|
|
|
|
end
|
|
|
|
|
2016-06-24 08:50:23 +00:00
|
|
|
true
|
2016-06-14 10:47:07 +00:00
|
|
|
end
|
2017-08-08 21:18:02 +00:00
|
|
|
|
2018-07-26 10:55:21 +00:00
|
|
|
def remove_import_file
|
|
|
|
FileUtils.rm_rf(@archive_file)
|
|
|
|
end
|
|
|
|
|
2017-08-08 21:18:02 +00:00
|
|
|
def extracted_files
|
2018-07-12 12:56:15 +00:00
|
|
|
Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
|
2017-08-08 21:18:02 +00:00
|
|
|
end
|
2020-08-05 21:09:40 +00:00
|
|
|
|
|
|
|
def validate_decompressed_archive_size
|
2021-05-04 15:10:36 +00:00
|
|
|
raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid?
|
2020-08-05 21:09:40 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def size_validator
|
|
|
|
@size_validator ||= DecompressedArchiveSizeValidator.new(archive_path: @archive_file)
|
|
|
|
end
|
2016-06-14 10:47:07 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|