gitlab-org--gitlab-foss/lib/backup/repository.rb
2018-06-25 16:21:54 +02:00

290 lines
8.8 KiB
Ruby

require 'yaml'
require_relative 'helper'
module Backup
class Repository
include Backup::Helper
attr_reader :progress
def initialize(progress)
@progress = progress
end
def dump
prepare
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... "
if project.hashed_storage?(:repository)
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
else
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
end
if !empty_repo?(project)
backup_project(project)
progress.puts "[DONE]".color(:green)
else
progress.puts "[SKIPPED]".color(:cyan)
end
wiki = ProjectWiki.new(project)
if !empty_repo?(wiki)
backup_project(wiki)
progress.puts "[DONE] Wiki".color(:green)
else
progress.puts "[SKIPPED] Wiki".color(:cyan)
end
end
end
def prepare_directories
Gitlab.config.repositories.storages.each do |name, repository_storage|
delete_all_repositories(name, repository_storage)
end
end
def backup_project(project)
gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
backup_project_gitaly(project)
else
backup_project_local(project)
end
end
backup_custom_hooks(project)
rescue => e
progress_warn(project, e, 'Failed to backup repo')
end
def backup_project_gitaly(project)
path_to_project_bundle = path_to_bundle(project)
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.create_bundle(path_to_project_bundle)
end
def backup_project_local(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
path_to_project_bundle = path_to_bundle(project)
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
progress_warn(project, cmd.join(' '), output) unless status.zero?
end
def delete_all_repositories(name, repository_storage)
gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
else
local_delete_all_repositories(name, repository_storage)
end
end
end
def local_delete_all_repositories(name, repository_storage)
path = repository_storage.legacy_disk_path
return unless File.exist?(path)
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
end
end
def local_restore_custom_hooks(project, dir)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
def gitaly_restore_custom_hooks(project, dir)
custom_hooks_path = path_to_tars(project, dir)
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.restore_custom_hooks(custom_hooks_path)
end
def local_backup_custom_hooks(project)
in_path(path_to_tars(project)) do |dir|
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
break unless File.exist?(File.join(path_to_project_repo, dir))
FileUtils.mkdir_p(path_to_tars(project))
cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
end
def gitaly_backup_custom_hooks(project)
FileUtils.mkdir_p(path_to_tars(project))
custom_hooks_path = path_to_tars(project, 'custom_hooks')
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.backup_custom_hooks(custom_hooks_path)
end
def backup_custom_hooks(project)
gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
gitaly_backup_custom_hooks(project)
else
local_backup_custom_hooks(project)
end
end
end
def restore_custom_hooks(project)
in_path(path_to_tars(project)) do |dir|
gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
gitaly_restore_custom_hooks(project, dir)
else
local_restore_custom_hooks(project, dir)
end
end
end
end
def restore
prepare_directories
gitlab_shell = Gitlab::Shell.new
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{project.full_path} ... "
path_to_project_bundle = path_to_bundle(project)
project.ensure_storage_path_exists
restore_repo_success = nil
if File.exist?(path_to_project_bundle)
begin
project.repository.create_from_bundle path_to_project_bundle
restore_repo_success = true
rescue => e
restore_repo_success = false
progress.puts "Error: #{e}".color(:red)
end
else
restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
end
if restore_repo_success
progress.puts "[DONE]".color(:green)
else
progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
end
restore_custom_hooks(project)
wiki = ProjectWiki.new(project)
path_to_wiki_bundle = path_to_bundle(wiki)
if File.exist?(path_to_wiki_bundle)
progress.print " * #{wiki.full_path} ... "
begin
wiki.repository.create_from_bundle(path_to_wiki_bundle)
restore_custom_hooks(wiki)
progress.puts "[DONE]".color(:green)
rescue => e
progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
progress.puts "Error #{e}".color(:red)
end
end
end
end
protected
def path_to_repo(project)
project.repository.path_to_repo
end
def path_to_bundle(project)
File.join(backup_repos_path, project.disk_path + '.bundle')
end
def path_to_tars(project, dir = nil)
path = File.join(backup_repos_path, project.disk_path)
if dir
File.join(path, "#{dir}.tar")
else
path
end
end
def backup_repos_path
File.join(Gitlab.config.backup.path, 'repositories')
end
def in_path(path)
return unless Dir.exist?(path)
dir_entries = Dir.entries(path)
if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
yield('custom_hooks')
end
end
def prepare
FileUtils.rm_rf(backup_repos_path)
FileUtils.mkdir_p(Gitlab.config.backup.path)
FileUtils.mkdir(backup_repos_path, mode: 0700)
end
def silent
{ err: '/dev/null', out: '/dev/null' }
end
private
def progress_warn(project, cmd, output)
progress.puts "[WARNING] Executing #{cmd}".color(:orange)
progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
end
def empty_repo?(project_or_wiki)
project_or_wiki.repository.expire_emptiness_caches
project_or_wiki.repository.empty?
end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
end
def display_repo_path(project)
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
end
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
Gitlab::GitalyClient.migrate(method, status: status, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
raise Error, e
end
end
end