2013-08-05 09:26:33 +00:00
|
|
|
module Backup
|
|
|
|
class Manager
|
2017-02-22 18:18:40 +00:00
|
|
|
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
|
|
|
|
FOLDERS_TO_BACKUP = %w[repositories db].freeze
|
|
|
|
FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze
|
2016-05-23 15:17:07 +00:00
|
|
|
|
2013-08-05 09:26:33 +00:00
|
|
|
def pack
|
2016-02-08 16:31:24 +00:00
|
|
|
# Make sure there is a connection
|
|
|
|
ActiveRecord::Base.connection.reconnect!
|
|
|
|
|
2017-04-26 10:54:11 +00:00
|
|
|
Dir.chdir(backup_path) do
|
|
|
|
File.open("#{backup_path}/backup_information.yml", "w+") do |file|
|
2017-07-20 08:50:07 +00:00
|
|
|
file << backup_information.to_yaml.gsub(/^---\n/, '')
|
2015-03-20 01:24:57 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
# create archive
|
|
|
|
$progress.print "Creating backup archive: #{tar_file} ... "
|
2015-07-06 16:43:17 +00:00
|
|
|
# Set file permissions on open to prevent chmod races.
|
2017-02-22 18:18:40 +00:00
|
|
|
tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
|
2015-07-06 16:43:17 +00:00
|
|
|
if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "done".color(:green)
|
2015-03-20 01:24:57 +00:00
|
|
|
else
|
2016-06-01 22:37:15 +00:00
|
|
|
puts "creating archive #{tar_file} failed".color(:red)
|
2015-03-20 01:24:57 +00:00
|
|
|
abort 'Backup failed'
|
|
|
|
end
|
|
|
|
|
2017-07-20 08:50:07 +00:00
|
|
|
upload
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
2014-09-29 13:02:39 +00:00
|
|
|
end
|
|
|
|
|
2017-07-20 08:50:07 +00:00
|
|
|
def upload
|
2014-11-20 14:46:04 +00:00
|
|
|
$progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
|
2014-09-29 13:02:39 +00:00
|
|
|
|
|
|
|
connection_settings = Gitlab.config.backup.upload.connection
|
|
|
|
if connection_settings.blank?
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "skipped".color(:yellow)
|
2014-09-29 13:02:39 +00:00
|
|
|
return
|
|
|
|
end
|
|
|
|
|
2016-06-09 01:04:31 +00:00
|
|
|
directory = connect_to_remote_directory(connection_settings)
|
2015-03-27 13:18:59 +00:00
|
|
|
|
2017-07-20 08:50:07 +00:00
|
|
|
if directory.files.create(key: remote_target, body: File.open(tar_file), public: false,
|
2017-02-22 18:18:40 +00:00
|
|
|
multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
|
2017-02-28 21:30:58 +00:00
|
|
|
encryption: Gitlab.config.backup.upload.encryption,
|
|
|
|
storage_class: Gitlab.config.backup.upload.storage_class)
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "done".color(:green)
|
2013-08-05 09:26:33 +00:00
|
|
|
else
|
2016-06-01 22:37:15 +00:00
|
|
|
puts "uploading backup to #{remote_directory} failed".color(:red)
|
2014-10-01 13:43:27 +00:00
|
|
|
abort 'Backup failed'
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def cleanup
|
2014-11-20 14:46:04 +00:00
|
|
|
$progress.print "Deleting tmp directories ... "
|
2015-09-10 15:57:43 +00:00
|
|
|
|
2015-03-27 13:18:59 +00:00
|
|
|
backup_contents.each do |dir|
|
2017-04-26 10:54:11 +00:00
|
|
|
next unless File.exist?(File.join(backup_path, dir))
|
2015-03-27 13:18:59 +00:00
|
|
|
|
2017-04-26 10:54:11 +00:00
|
|
|
if FileUtils.rm_rf(File.join(backup_path, dir))
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "done".color(:green)
|
2015-03-15 18:54:36 +00:00
|
|
|
else
|
2016-06-01 22:37:15 +00:00
|
|
|
puts "deleting tmp directory '#{dir}' failed".color(:red)
|
2015-03-15 18:54:36 +00:00
|
|
|
abort 'Backup failed'
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def remove_old
|
|
|
|
# delete backups
|
2014-11-20 14:46:04 +00:00
|
|
|
$progress.print "Deleting old backups ... "
|
2013-08-05 09:26:33 +00:00
|
|
|
keep_time = Gitlab.config.backup.keep_time.to_i
|
|
|
|
|
|
|
|
if keep_time > 0
|
|
|
|
removed = 0
|
2015-09-10 15:57:43 +00:00
|
|
|
|
2017-04-26 10:54:11 +00:00
|
|
|
Dir.chdir(backup_path) do
|
|
|
|
backup_file_list.each do |file|
|
2017-05-23 08:20:54 +00:00
|
|
|
# For backward compatibility, there are 3 names the backups can have:
|
|
|
|
# - 1495527122_gitlab_backup.tar
|
|
|
|
# - 1495527068_2017_05_23_gitlab_backup.tar
|
|
|
|
# - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
|
2017-09-21 12:00:58 +00:00
|
|
|
next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
|
2016-12-07 12:39:19 +00:00
|
|
|
|
|
|
|
timestamp = $1.to_i
|
|
|
|
|
|
|
|
if Time.at(timestamp) < (Time.now - keep_time)
|
|
|
|
begin
|
|
|
|
FileUtils.rm(file)
|
2015-03-25 02:37:19 +00:00
|
|
|
removed += 1
|
2016-12-07 12:39:19 +00:00
|
|
|
rescue => e
|
|
|
|
$progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
|
2015-03-25 02:37:19 +00:00
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2015-03-27 13:18:59 +00:00
|
|
|
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "done. (#{removed} removed)".color(:green)
|
2013-08-05 09:26:33 +00:00
|
|
|
else
|
2016-06-01 22:37:15 +00:00
|
|
|
$progress.puts "skipping".color(:yellow)
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def unpack
|
2017-10-10 15:24:31 +00:00
|
|
|
Dir.chdir(backup_path) do
|
|
|
|
# check for existing backups in the backup dir
|
|
|
|
if backup_file_list.empty?
|
|
|
|
$progress.puts "No backups found in #{backup_path}"
|
|
|
|
$progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
|
|
|
|
exit 1
|
|
|
|
elsif backup_file_list.many? && ENV["BACKUP"].nil?
|
|
|
|
$progress.puts 'Found more than one backup, please specify which one you want to restore:'
|
|
|
|
$progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
|
|
|
|
exit 1
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
|
2017-10-10 15:24:31 +00:00
|
|
|
tar_file = if ENV['BACKUP'].present?
|
|
|
|
"#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}"
|
|
|
|
else
|
|
|
|
backup_file_list.first
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
|
2017-10-10 15:24:31 +00:00
|
|
|
unless File.exist?(tar_file)
|
|
|
|
$progress.puts "The backup file #{tar_file} does not exist!"
|
|
|
|
exit 1
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
|
2017-10-10 15:24:31 +00:00
|
|
|
$progress.print 'Unpacking backup ... '
|
2015-03-27 13:18:59 +00:00
|
|
|
|
2017-10-10 15:24:31 +00:00
|
|
|
unless Kernel.system(*%W(tar -xf #{tar_file}))
|
|
|
|
$progress.puts 'unpacking backup failed'.color(:red)
|
|
|
|
exit 1
|
|
|
|
else
|
|
|
|
$progress.puts 'done'.color(:green)
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
|
2017-10-10 15:24:31 +00:00
|
|
|
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
|
|
|
|
|
|
|
|
# restoring mismatching backups can lead to unexpected problems
|
|
|
|
if settings[:gitlab_version] != Gitlab::VERSION
|
2017-10-22 17:12:35 +00:00
|
|
|
$progress.puts(<<~HEREDOC.color(:red))
|
|
|
|
GitLab version mismatch:
|
|
|
|
Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
|
|
|
|
Please switch to the following version and try again:
|
|
|
|
version: #{settings[:gitlab_version]}
|
|
|
|
HEREDOC
|
2017-10-10 15:24:31 +00:00
|
|
|
$progress.puts
|
|
|
|
$progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
|
|
|
|
exit 1
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|
2014-02-25 11:12:09 +00:00
|
|
|
|
|
|
|
def tar_version
|
2017-02-22 18:18:40 +00:00
|
|
|
tar_version, _ = Gitlab::Popen.popen(%w(tar --version))
|
2014-05-08 09:02:04 +00:00
|
|
|
tar_version.force_encoding('locale').split("\n").first
|
2014-02-25 11:12:09 +00:00
|
|
|
end
|
2015-03-27 13:18:59 +00:00
|
|
|
|
|
|
|
def skipped?(item)
|
2016-05-25 03:52:31 +00:00
|
|
|
settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item)
|
2015-03-27 13:18:59 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2017-04-26 10:54:11 +00:00
|
|
|
def backup_path
|
|
|
|
Gitlab.config.backup.path
|
|
|
|
end
|
|
|
|
|
|
|
|
def backup_file_list
|
|
|
|
@backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}")
|
|
|
|
end
|
|
|
|
|
2016-06-09 01:04:31 +00:00
|
|
|
def connect_to_remote_directory(connection_settings)
|
2017-07-20 08:50:07 +00:00
|
|
|
# our settings use string keys, but Fog expects symbols
|
|
|
|
connection = ::Fog::Storage.new(connection_settings.symbolize_keys)
|
2016-06-09 01:04:31 +00:00
|
|
|
|
|
|
|
# We only attempt to create the directory for local backups. For AWS
|
|
|
|
# and other cloud providers, we cannot guarantee the user will have
|
|
|
|
# permission to create the bucket.
|
|
|
|
if connection.service == ::Fog::Storage::Local
|
|
|
|
connection.directories.create(key: remote_directory)
|
|
|
|
else
|
|
|
|
connection.directories.get(remote_directory)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def remote_directory
|
|
|
|
Gitlab.config.backup.upload.remote_directory
|
|
|
|
end
|
|
|
|
|
2017-07-20 08:50:07 +00:00
|
|
|
def remote_target
|
|
|
|
if ENV['DIRECTORY']
|
|
|
|
File.join(ENV['DIRECTORY'], tar_file)
|
|
|
|
else
|
|
|
|
tar_file
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2015-03-27 13:18:59 +00:00
|
|
|
def backup_contents
|
2015-11-18 12:01:35 +00:00
|
|
|
folders_to_backup + archives_to_backup + ["backup_information.yml"]
|
2015-03-27 13:18:59 +00:00
|
|
|
end
|
|
|
|
|
2015-11-18 12:01:35 +00:00
|
|
|
def archives_to_backup
|
2017-08-09 09:52:22 +00:00
|
|
|
ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact
|
2015-11-18 12:01:35 +00:00
|
|
|
end
|
2015-03-27 13:18:59 +00:00
|
|
|
|
2015-11-18 12:01:35 +00:00
|
|
|
def folders_to_backup
|
2017-08-09 09:52:22 +00:00
|
|
|
FOLDERS_TO_BACKUP.reject { |name| skipped?(name) }
|
2016-05-23 15:17:07 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def disabled_features
|
|
|
|
features = []
|
|
|
|
features << 'registry' unless Gitlab.config.registry.enabled
|
|
|
|
features
|
2015-03-27 13:18:59 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def settings
|
|
|
|
@settings ||= YAML.load_file("backup_information.yml")
|
|
|
|
end
|
2017-07-20 08:50:07 +00:00
|
|
|
|
|
|
|
def tar_file
|
|
|
|
@tar_file ||= "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
|
|
|
|
end
|
|
|
|
|
|
|
|
def backup_information
|
|
|
|
@backup_information ||= {
|
|
|
|
db_version: ActiveRecord::Migrator.current_version.to_s,
|
|
|
|
backup_created_at: Time.now,
|
|
|
|
gitlab_version: Gitlab::VERSION,
|
|
|
|
tar_version: tar_version,
|
|
|
|
skipped: ENV["SKIP"]
|
|
|
|
}
|
|
|
|
end
|
2013-08-05 09:26:33 +00:00
|
|
|
end
|
|
|
|
end
|