2018-09-29 18:34:47 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2013-04-05 12:01:19 -04:00
|
|
|
require 'yaml'
|
|
|
|
|
|
|
|
module Backup
|
|
|
|
class Repository
|
2018-12-10 03:31:28 -05:00
|
|
|
include Gitlab::ShellAdapter
|
2018-05-24 10:58:25 -04:00
|
|
|
attr_reader :progress
|
|
|
|
|
|
|
|
def initialize(progress)
|
|
|
|
@progress = progress
|
|
|
|
end
|
|
|
|
|
2013-04-05 12:01:19 -04:00
|
|
|
def dump
|
|
|
|
prepare
|
|
|
|
|
|
|
|
Project.find_each(batch_size: 1000) do |project|
|
2017-11-15 09:20:36 -05:00
|
|
|
progress.print " * #{display_repo_path(project)} ... "
|
2018-06-05 11:51:14 -04:00
|
|
|
|
2017-11-15 09:20:36 -05:00
|
|
|
if project.hashed_storage?(:repository)
|
|
|
|
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
|
|
|
|
else
|
|
|
|
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
|
|
|
|
end
|
2013-04-05 12:01:19 -04:00
|
|
|
|
2018-06-13 05:19:17 -04:00
|
|
|
if !empty_repo?(project)
|
|
|
|
backup_project(project)
|
2018-06-18 10:42:39 -04:00
|
|
|
progress.puts "[DONE]".color(:green)
|
2013-04-05 12:01:19 -04:00
|
|
|
else
|
2018-06-13 05:19:17 -04:00
|
|
|
progress.puts "[SKIPPED]".color(:cyan)
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
2013-04-05 14:20:11 -04:00
|
|
|
|
2014-04-09 07:35:58 -04:00
|
|
|
wiki = ProjectWiki.new(project)
|
2013-04-05 14:20:11 -04:00
|
|
|
|
2018-06-18 10:42:39 -04:00
|
|
|
if !empty_repo?(wiki)
|
2018-06-13 05:19:17 -04:00
|
|
|
backup_project(wiki)
|
2018-06-18 10:42:39 -04:00
|
|
|
progress.puts "[DONE] Wiki".color(:green)
|
2018-06-13 05:19:17 -04:00
|
|
|
else
|
|
|
|
progress.puts "[SKIPPED] Wiki".color(:cyan)
|
2013-04-05 14:20:11 -04:00
|
|
|
end
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-03-05 03:25:02 -05:00
|
|
|
def prepare_directories
|
2018-07-26 04:05:32 -04:00
|
|
|
Gitlab.config.repositories.storages.each do |name, _repository_storage|
|
|
|
|
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
|
2018-05-28 05:58:14 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-13 05:19:17 -04:00
|
|
|
def backup_project(project)
|
|
|
|
path_to_project_bundle = path_to_bundle(project)
|
|
|
|
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
|
|
|
.create_bundle(path_to_project_bundle)
|
2018-05-28 05:58:14 -04:00
|
|
|
|
2018-07-26 04:05:32 -04:00
|
|
|
backup_custom_hooks(project)
|
|
|
|
rescue => e
|
|
|
|
progress_warn(project, e, 'Failed to backup repo')
|
2018-06-04 09:15:54 -04:00
|
|
|
end
|
|
|
|
|
2018-07-26 04:05:32 -04:00
|
|
|
def backup_custom_hooks(project)
|
|
|
|
FileUtils.mkdir_p(project_backup_path(project))
|
2018-06-13 05:19:17 -04:00
|
|
|
|
2018-07-26 04:05:32 -04:00
|
|
|
custom_hooks_path = custom_hooks_tar(project)
|
2018-06-13 05:19:17 -04:00
|
|
|
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
|
|
|
.backup_custom_hooks(custom_hooks_path)
|
|
|
|
end
|
|
|
|
|
2018-05-23 06:31:53 -04:00
|
|
|
def restore_custom_hooks(project)
|
2018-07-26 04:05:32 -04:00
|
|
|
return unless Dir.exist?(project_backup_path(project))
|
|
|
|
return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
|
|
|
|
|
|
|
|
custom_hooks_path = custom_hooks_tar(project)
|
|
|
|
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
|
|
|
.restore_custom_hooks(custom_hooks_path)
|
2018-05-17 08:39:59 -04:00
|
|
|
end
|
|
|
|
|
2018-03-05 03:25:02 -05:00
|
|
|
def restore
|
|
|
|
prepare_directories
|
2018-05-28 05:58:14 -04:00
|
|
|
|
2013-04-05 12:01:19 -04:00
|
|
|
Project.find_each(batch_size: 1000) do |project|
|
2018-05-11 18:09:26 -04:00
|
|
|
progress.print " * #{project.full_path} ... "
|
2016-08-11 13:37:39 -04:00
|
|
|
path_to_project_bundle = path_to_bundle(project)
|
2017-08-15 22:49:54 -04:00
|
|
|
project.ensure_storage_path_exists
|
2013-04-05 12:01:19 -04:00
|
|
|
|
2018-05-14 16:49:04 -04:00
|
|
|
restore_repo_success = nil
|
2018-05-06 15:42:39 -04:00
|
|
|
if File.exist?(path_to_project_bundle)
|
|
|
|
begin
|
2018-07-26 04:05:32 -04:00
|
|
|
project.repository.create_from_bundle(path_to_project_bundle)
|
|
|
|
restore_custom_hooks(project)
|
2018-05-14 16:49:04 -04:00
|
|
|
restore_repo_success = true
|
2018-05-11 14:33:29 -04:00
|
|
|
rescue => e
|
2018-05-14 16:49:04 -04:00
|
|
|
restore_repo_success = false
|
2018-05-06 15:42:39 -04:00
|
|
|
progress.puts "Error: #{e}".color(:red)
|
|
|
|
end
|
|
|
|
else
|
2018-05-14 16:49:04 -04:00
|
|
|
restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
|
2018-05-11 14:33:29 -04:00
|
|
|
end
|
2014-11-13 07:09:47 -05:00
|
|
|
|
2018-05-14 16:49:04 -04:00
|
|
|
if restore_repo_success
|
2017-06-07 10:50:21 -04:00
|
|
|
progress.puts "[DONE]".color(:green)
|
2013-04-05 12:01:19 -04:00
|
|
|
else
|
2018-05-11 18:09:26 -04:00
|
|
|
progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
2013-04-05 14:20:11 -04:00
|
|
|
|
2014-04-09 07:35:58 -04:00
|
|
|
wiki = ProjectWiki.new(project)
|
2016-08-11 13:37:39 -04:00
|
|
|
path_to_wiki_bundle = path_to_bundle(wiki)
|
2013-04-05 14:20:11 -04:00
|
|
|
|
2016-08-11 13:37:39 -04:00
|
|
|
if File.exist?(path_to_wiki_bundle)
|
2018-05-11 18:09:26 -04:00
|
|
|
progress.print " * #{wiki.full_path} ... "
|
2018-05-06 15:42:39 -04:00
|
|
|
begin
|
2018-05-11 18:09:26 -04:00
|
|
|
wiki.repository.create_from_bundle(path_to_wiki_bundle)
|
2018-06-21 07:56:00 -04:00
|
|
|
restore_custom_hooks(wiki)
|
|
|
|
|
2018-05-11 18:09:26 -04:00
|
|
|
progress.puts "[DONE]".color(:green)
|
2018-05-14 16:49:04 -04:00
|
|
|
rescue => e
|
2018-05-11 18:09:26 -04:00
|
|
|
progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
|
2018-05-06 15:42:39 -04:00
|
|
|
progress.puts "Error #{e}".color(:red)
|
2014-11-30 11:24:05 -05:00
|
|
|
end
|
2013-04-05 14:20:11 -04:00
|
|
|
end
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
2018-12-10 03:31:28 -05:00
|
|
|
|
|
|
|
restore_object_pools
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
protected
|
|
|
|
|
|
|
|
def path_to_bundle(project)
|
2017-07-21 00:13:26 -04:00
|
|
|
File.join(backup_repos_path, project.disk_path + '.bundle')
|
2016-08-11 13:37:39 -04:00
|
|
|
end
|
|
|
|
|
2018-07-26 04:05:32 -04:00
|
|
|
def project_backup_path(project)
|
|
|
|
File.join(backup_repos_path, project.disk_path)
|
|
|
|
end
|
2016-08-11 13:37:39 -04:00
|
|
|
|
2018-07-26 04:05:32 -04:00
|
|
|
def custom_hooks_tar(project)
|
|
|
|
File.join(project_backup_path(project), "custom_hooks.tar")
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def backup_repos_path
|
2016-08-11 13:37:39 -04:00
|
|
|
File.join(Gitlab.config.backup.path, 'repositories')
|
|
|
|
end
|
|
|
|
|
2013-04-05 12:01:19 -04:00
|
|
|
def prepare
|
|
|
|
FileUtils.rm_rf(backup_repos_path)
|
2015-07-30 04:17:34 -04:00
|
|
|
FileUtils.mkdir_p(Gitlab.config.backup.path)
|
|
|
|
FileUtils.mkdir(backup_repos_path, mode: 0700)
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
2013-11-05 09:00:48 -05:00
|
|
|
|
2016-06-22 17:04:51 -04:00
|
|
|
private
|
|
|
|
|
2017-06-07 09:38:12 -04:00
|
|
|
def progress_warn(project, cmd, output)
|
2017-06-07 10:50:21 -04:00
|
|
|
progress.puts "[WARNING] Executing #{cmd}".color(:orange)
|
2017-11-15 09:20:36 -05:00
|
|
|
progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
|
2017-06-07 09:38:12 -04:00
|
|
|
end
|
|
|
|
|
2017-06-07 10:50:21 -04:00
|
|
|
def empty_repo?(project_or_wiki)
|
2017-12-07 10:33:30 -05:00
|
|
|
project_or_wiki.repository.expire_emptiness_caches
|
|
|
|
project_or_wiki.repository.empty?
|
2017-06-07 09:38:12 -04:00
|
|
|
end
|
|
|
|
|
2017-11-15 09:20:36 -05:00
|
|
|
def display_repo_path(project)
|
|
|
|
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
|
|
|
|
end
|
2018-12-10 03:31:28 -05:00
|
|
|
|
|
|
|
def restore_object_pools
|
|
|
|
PoolRepository.includes(:source_project).find_each do |pool|
|
|
|
|
progress.puts " - Object pool #{pool.disk_path}..."
|
|
|
|
|
|
|
|
pool.source_project ||= pool.member_projects.first.root_of_fork_network
|
|
|
|
pool.state = 'none'
|
|
|
|
pool.save
|
|
|
|
|
|
|
|
pool.schedule
|
|
|
|
end
|
|
|
|
end
|
2013-04-05 12:01:19 -04:00
|
|
|
end
|
|
|
|
end
|