2013-04-05 16:01:19 +00:00
|
|
|
require 'yaml'
|
2018-03-05 08:25:02 +00:00
|
|
|
require_relative 'helper'
|
2013-04-05 16:01:19 +00:00
|
|
|
|
|
|
|
module Backup
|
|
|
|
class Repository
|
2018-03-05 08:25:02 +00:00
|
|
|
include Backup::Helper
|
2017-02-22 18:18:40 +00:00
|
|
|
# rubocop:disable Metrics/AbcSize
|
2018-03-05 08:25:02 +00:00
|
|
|
|
2018-05-24 14:58:25 +00:00
|
|
|
attr_reader :progress
|
|
|
|
|
|
|
|
def initialize(progress)
|
|
|
|
@progress = progress
|
|
|
|
end
|
|
|
|
|
2013-04-05 16:01:19 +00:00
|
|
|
def dump
|
|
|
|
prepare
|
|
|
|
|
|
|
|
Project.find_each(batch_size: 1000) do |project|
|
2017-11-15 14:20:36 +00:00
|
|
|
progress.print " * #{display_repo_path(project)} ... "
|
2018-06-05 15:51:14 +00:00
|
|
|
|
|
|
|
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
|
|
|
path_to_repo(project)
|
|
|
|
end
|
2016-08-11 17:37:39 +00:00
|
|
|
path_to_project_bundle = path_to_bundle(project)
|
2013-04-05 16:01:19 +00:00
|
|
|
|
2017-11-15 14:20:36 +00:00
|
|
|
# Create namespace dir or hashed path if missing
|
|
|
|
if project.hashed_storage?(:repository)
|
|
|
|
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
|
|
|
|
else
|
|
|
|
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
|
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
|
2017-06-07 13:38:12 +00:00
|
|
|
if empty_repo?(project)
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts "[SKIPPED]".color(:cyan)
|
2013-04-05 16:01:19 +00:00
|
|
|
else
|
2016-08-11 17:37:39 +00:00
|
|
|
in_path(path_to_project_repo) do |dir|
|
|
|
|
FileUtils.mkdir_p(path_to_tars(project))
|
|
|
|
cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
|
|
|
|
output, status = Gitlab::Popen.popen(cmd)
|
|
|
|
|
|
|
|
unless status.zero?
|
2017-06-07 13:38:12 +00:00
|
|
|
progress_warn(project, cmd.join(' '), output)
|
2016-08-11 17:37:39 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
|
2014-11-20 14:46:04 +00:00
|
|
|
output, status = Gitlab::Popen.popen(cmd)
|
2016-08-11 17:37:39 +00:00
|
|
|
|
2014-10-01 13:43:27 +00:00
|
|
|
if status.zero?
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts "[DONE]".color(:green)
|
2014-10-01 13:43:27 +00:00
|
|
|
else
|
2017-06-07 13:38:12 +00:00
|
|
|
progress_warn(project, cmd.join(' '), output)
|
2014-10-01 13:43:27 +00:00
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
2013-04-05 18:20:11 +00:00
|
|
|
|
2014-04-09 11:35:58 +00:00
|
|
|
wiki = ProjectWiki.new(project)
|
2018-06-05 15:51:14 +00:00
|
|
|
path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
|
|
|
path_to_repo(wiki)
|
|
|
|
end
|
2016-08-11 17:37:39 +00:00
|
|
|
path_to_wiki_bundle = path_to_bundle(wiki)
|
2013-04-05 18:20:11 +00:00
|
|
|
|
2016-08-11 17:37:39 +00:00
|
|
|
if File.exist?(path_to_wiki_repo)
|
2017-11-15 14:20:36 +00:00
|
|
|
progress.print " * #{display_repo_path(wiki)} ... "
|
2018-01-11 16:34:01 +00:00
|
|
|
|
2017-06-07 14:54:12 +00:00
|
|
|
if empty_repo?(wiki)
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts " [SKIPPED]".color(:cyan)
|
2013-04-05 18:20:11 +00:00
|
|
|
else
|
2016-08-11 17:37:39 +00:00
|
|
|
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
|
2014-11-20 14:46:04 +00:00
|
|
|
output, status = Gitlab::Popen.popen(cmd)
|
2014-10-01 13:43:27 +00:00
|
|
|
if status.zero?
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts " [DONE]".color(:green)
|
2014-10-01 13:43:27 +00:00
|
|
|
else
|
2017-06-07 13:38:12 +00:00
|
|
|
progress_warn(wiki, cmd.join(' '), output)
|
2014-10-01 13:43:27 +00:00
|
|
|
end
|
2013-04-05 18:20:11 +00:00
|
|
|
end
|
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-03-05 08:25:02 +00:00
|
|
|
def prepare_directories
|
2017-02-28 21:08:40 +00:00
|
|
|
Gitlab.config.repositories.storages.each do |name, repository_storage|
|
2018-05-28 09:58:14 +00:00
|
|
|
delete_all_repositories(name, repository_storage)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def delete_all_repositories(name, repository_storage)
|
|
|
|
gitaly_migrate(:delete_all_repositories) do |is_enabled|
|
|
|
|
if is_enabled
|
|
|
|
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
|
|
|
|
else
|
|
|
|
local_delete_all_repositories(name, repository_storage)
|
2018-03-05 08:25:02 +00:00
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
2018-03-05 08:25:02 +00:00
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
|
2018-05-28 09:58:14 +00:00
|
|
|
def local_delete_all_repositories(name, repository_storage)
|
|
|
|
path = repository_storage.legacy_disk_path
|
|
|
|
return unless File.exist?(path)
|
|
|
|
|
|
|
|
# Move all files in the existing repos directory except . and .. to
|
|
|
|
# repositories.old.<timestamp> directory
|
|
|
|
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
|
|
|
|
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
|
|
|
|
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
|
|
|
|
|
|
|
|
begin
|
|
|
|
FileUtils.mv(files, bk_repos_path)
|
|
|
|
rescue Errno::EACCES
|
|
|
|
access_denied_error(path)
|
|
|
|
rescue Errno::EBUSY
|
|
|
|
resource_busy_error(path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-04 13:15:54 +00:00
|
|
|
def local_restore_custom_hooks(project, dir)
|
|
|
|
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
|
|
|
path_to_repo(project)
|
|
|
|
end
|
|
|
|
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
|
|
|
|
output, status = Gitlab::Popen.popen(cmd)
|
|
|
|
unless status.zero?
|
|
|
|
progress_warn(project, cmd.join(' '), output)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def gitaly_restore_custom_hooks(project, dir)
|
|
|
|
custom_hooks_path = path_to_tars(project, dir)
|
|
|
|
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
|
|
|
.restore_custom_hooks(custom_hooks_path)
|
|
|
|
end
|
|
|
|
|
2018-05-23 10:31:53 +00:00
|
|
|
def restore_custom_hooks(project)
|
2018-05-21 09:45:41 +00:00
|
|
|
in_path(path_to_tars(project)) do |dir|
|
2018-06-04 13:15:54 +00:00
|
|
|
gitaly_migrate(:restore_custom_hooks) do |is_enabled|
|
|
|
|
if is_enabled
|
|
|
|
local_restore_custom_hooks(project, dir)
|
|
|
|
else
|
|
|
|
gitaly_restore_custom_hooks(project, dir)
|
|
|
|
end
|
2018-05-17 12:39:59 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-03-05 08:25:02 +00:00
|
|
|
def restore
|
|
|
|
prepare_directories
|
2018-05-11 18:33:29 +00:00
|
|
|
gitlab_shell = Gitlab::Shell.new
|
2018-05-28 09:58:14 +00:00
|
|
|
|
2013-04-05 16:01:19 +00:00
|
|
|
Project.find_each(batch_size: 1000) do |project|
|
2018-05-11 22:09:26 +00:00
|
|
|
progress.print " * #{project.full_path} ... "
|
2016-08-11 17:37:39 +00:00
|
|
|
path_to_project_bundle = path_to_bundle(project)
|
2017-08-16 02:49:54 +00:00
|
|
|
project.ensure_storage_path_exists
|
2013-04-05 16:01:19 +00:00
|
|
|
|
2018-05-14 20:49:04 +00:00
|
|
|
restore_repo_success = nil
|
2018-05-06 19:42:39 +00:00
|
|
|
if File.exist?(path_to_project_bundle)
|
|
|
|
begin
|
2018-05-11 18:33:29 +00:00
|
|
|
project.repository.create_from_bundle path_to_project_bundle
|
2018-05-14 20:49:04 +00:00
|
|
|
restore_repo_success = true
|
2018-05-11 18:33:29 +00:00
|
|
|
rescue => e
|
2018-05-14 20:49:04 +00:00
|
|
|
restore_repo_success = false
|
2018-05-06 19:42:39 +00:00
|
|
|
progress.puts "Error: #{e}".color(:red)
|
|
|
|
end
|
|
|
|
else
|
2018-05-14 20:49:04 +00:00
|
|
|
restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
|
2018-05-11 18:33:29 +00:00
|
|
|
end
|
2014-11-13 12:09:47 +00:00
|
|
|
|
2018-05-14 20:49:04 +00:00
|
|
|
if restore_repo_success
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts "[DONE]".color(:green)
|
2013-04-05 16:01:19 +00:00
|
|
|
else
|
2018-05-11 22:09:26 +00:00
|
|
|
progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
2013-04-05 18:20:11 +00:00
|
|
|
|
2018-05-23 10:31:53 +00:00
|
|
|
restore_custom_hooks(project)
|
2016-08-11 17:37:39 +00:00
|
|
|
|
2014-04-09 11:35:58 +00:00
|
|
|
wiki = ProjectWiki.new(project)
|
2016-08-11 17:37:39 +00:00
|
|
|
path_to_wiki_bundle = path_to_bundle(wiki)
|
2013-04-05 18:20:11 +00:00
|
|
|
|
2016-08-11 17:37:39 +00:00
|
|
|
if File.exist?(path_to_wiki_bundle)
|
2018-05-11 22:09:26 +00:00
|
|
|
progress.print " * #{wiki.full_path} ... "
|
2018-05-06 19:42:39 +00:00
|
|
|
begin
|
2018-05-11 22:09:26 +00:00
|
|
|
wiki.repository.create_from_bundle(path_to_wiki_bundle)
|
|
|
|
progress.puts "[DONE]".color(:green)
|
2018-05-14 20:49:04 +00:00
|
|
|
rescue => e
|
2018-05-11 22:09:26 +00:00
|
|
|
progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
|
2018-05-06 19:42:39 +00:00
|
|
|
progress.puts "Error #{e}".color(:red)
|
2014-11-30 16:24:05 +00:00
|
|
|
end
|
2013-04-05 18:20:11 +00:00
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
end
|
2017-02-22 18:18:40 +00:00
|
|
|
# rubocop:enable Metrics/AbcSize
|
2013-04-05 16:01:19 +00:00
|
|
|
|
|
|
|
protected
|
|
|
|
|
|
|
|
def path_to_repo(project)
|
2014-11-05 16:51:08 +00:00
|
|
|
project.repository.path_to_repo
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def path_to_bundle(project)
|
2017-07-21 04:13:26 +00:00
|
|
|
File.join(backup_repos_path, project.disk_path + '.bundle')
|
2016-08-11 17:37:39 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def path_to_tars(project, dir = nil)
|
2017-07-21 04:13:26 +00:00
|
|
|
path = File.join(backup_repos_path, project.disk_path)
|
2016-08-11 17:37:39 +00:00
|
|
|
|
|
|
|
if dir
|
|
|
|
File.join(path, "#{dir}.tar")
|
|
|
|
else
|
|
|
|
path
|
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def backup_repos_path
|
2016-08-11 17:37:39 +00:00
|
|
|
File.join(Gitlab.config.backup.path, 'repositories')
|
|
|
|
end
|
|
|
|
|
|
|
|
def in_path(path)
|
|
|
|
return unless Dir.exist?(path)
|
|
|
|
|
|
|
|
dir_entries = Dir.entries(path)
|
2017-02-24 12:18:07 +00:00
|
|
|
|
2017-03-28 14:24:40 +00:00
|
|
|
if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
|
|
|
|
yield('custom_hooks')
|
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def prepare
|
|
|
|
FileUtils.rm_rf(backup_repos_path)
|
2015-07-30 08:17:34 +00:00
|
|
|
# Ensure the parent dir of backup_repos_path exists
|
|
|
|
FileUtils.mkdir_p(Gitlab.config.backup.path)
|
|
|
|
# Fail if somebody raced to create backup_repos_path before us
|
|
|
|
FileUtils.mkdir(backup_repos_path, mode: 0700)
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
2013-11-05 14:00:48 +00:00
|
|
|
|
|
|
|
def silent
|
2017-02-22 18:18:40 +00:00
|
|
|
{ err: '/dev/null', out: '/dev/null' }
|
2013-11-05 14:00:48 +00:00
|
|
|
end
|
2016-06-22 21:04:51 +00:00
|
|
|
|
|
|
|
private
|
|
|
|
|
2017-06-07 13:38:12 +00:00
|
|
|
def progress_warn(project, cmd, output)
|
2017-06-07 14:50:21 +00:00
|
|
|
progress.puts "[WARNING] Executing #{cmd}".color(:orange)
|
2017-11-15 14:20:36 +00:00
|
|
|
progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
|
2017-06-07 13:38:12 +00:00
|
|
|
end
|
|
|
|
|
2017-06-07 14:50:21 +00:00
|
|
|
def empty_repo?(project_or_wiki)
|
2017-12-07 15:33:30 +00:00
|
|
|
# Protect against stale caches
|
|
|
|
project_or_wiki.repository.expire_emptiness_caches
|
|
|
|
project_or_wiki.repository.empty?
|
2017-06-07 13:38:12 +00:00
|
|
|
end
|
|
|
|
|
2016-06-22 21:04:51 +00:00
|
|
|
def repository_storage_paths_args
|
2018-03-14 13:42:49 +00:00
|
|
|
Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
|
2016-06-22 21:04:51 +00:00
|
|
|
end
|
2017-06-07 14:50:21 +00:00
|
|
|
|
2017-11-15 14:20:36 +00:00
|
|
|
def display_repo_path(project)
|
|
|
|
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
|
|
|
|
end
|
2018-05-28 09:58:14 +00:00
|
|
|
|
|
|
|
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
|
|
|
|
Gitlab::GitalyClient.migrate(method, status: status, &block)
|
|
|
|
rescue GRPC::NotFound, GRPC::BadStatus => e
|
|
|
|
raise Error, e
|
|
|
|
end
|
2013-04-05 16:01:19 +00:00
|
|
|
end
|
|
|
|
end
|