gitlab-org--gitlab-foss/lib/backup/repository.rb

227 lines
7.1 KiB
Ruby
Raw Normal View History

2013-04-05 12:01:19 -04:00
require 'yaml'
require_relative 'helper'
2013-04-05 12:01:19 -04:00
module Backup
class Repository
include Backup::Helper
# rubocop:disable Metrics/AbcSize
2013-04-05 12:01:19 -04:00
def dump
prepare
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... "
path_to_project_repo = path_to_repo(project)
path_to_project_bundle = path_to_bundle(project)
2013-04-05 12:01:19 -04:00
# Create namespace dir or hashed path if missing
if project.hashed_storage?(:repository)
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
else
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
end
2013-04-05 12:01:19 -04:00
if empty_repo?(project)
2017-06-07 10:50:21 -04:00
progress.puts "[SKIPPED]".color(:cyan)
2013-04-05 12:01:19 -04:00
else
in_path(path_to_project_repo) do |dir|
FileUtils.mkdir_p(path_to_tars(project))
cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
2017-06-07 10:50:21 -04:00
progress.puts "[DONE]".color(:green)
else
progress_warn(project, cmd.join(' '), output)
end
2013-04-05 12:01:19 -04:00
end
2013-04-05 14:20:11 -04:00
wiki = ProjectWiki.new(project)
path_to_wiki_repo = path_to_repo(wiki)
path_to_wiki_bundle = path_to_bundle(wiki)
2013-04-05 14:20:11 -04:00
if File.exist?(path_to_wiki_repo)
progress.print " * #{display_repo_path(wiki)} ... "
2017-06-07 10:54:12 -04:00
if empty_repo?(wiki)
2017-06-07 10:50:21 -04:00
progress.puts " [SKIPPED]".color(:cyan)
2013-04-05 14:20:11 -04:00
else
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
2017-06-07 10:50:21 -04:00
progress.puts " [DONE]".color(:green)
else
progress_warn(wiki, cmd.join(' '), output)
end
2013-04-05 14:20:11 -04:00
end
end
2013-04-05 12:01:19 -04:00
end
end
def prepare_directories
Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage.legacy_disk_path
next unless File.exist?(path)
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
end
2013-04-05 12:01:19 -04:00
end
end
2013-04-05 12:01:19 -04:00
def restore
prepare_directories
2013-04-05 12:01:19 -04:00
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... "
path_to_project_repo = path_to_repo(project)
path_to_project_bundle = path_to_bundle(project)
2013-04-05 12:01:19 -04:00
2017-08-15 22:49:54 -04:00
project.ensure_storage_path_exists
2013-04-05 12:01:19 -04:00
cmd = if File.exist?(path_to_project_bundle)
%W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo})
else
%W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo})
end
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
2017-06-07 10:50:21 -04:00
progress.puts "[DONE]".color(:green)
2013-04-05 12:01:19 -04:00
else
progress_warn(project, cmd.join(' '), output)
2013-04-05 12:01:19 -04:00
end
2013-04-05 14:20:11 -04:00
in_path(path_to_tars(project)) do |dir|
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
wiki = ProjectWiki.new(project)
path_to_wiki_repo = path_to_repo(wiki)
path_to_wiki_bundle = path_to_bundle(wiki)
2013-04-05 14:20:11 -04:00
if File.exist?(path_to_wiki_bundle)
progress.print " * #{display_repo_path(wiki)} ... "
# If a wiki bundle exists, first remove the empty repo
# that was initialized with ProjectWiki.new() and then
# try to restore with 'git clone --bare'.
FileUtils.rm_rf(path_to_wiki_repo)
cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo})
2014-11-29 14:59:28 -05:00
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
2017-06-07 10:50:21 -04:00
progress.puts " [DONE]".color(:green)
else
progress_warn(project, cmd.join(' '), output)
end
2013-04-05 14:20:11 -04:00
end
2013-04-05 12:01:19 -04:00
end
2013-04-10 08:11:45 -04:00
2017-06-07 10:50:21 -04:00
progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
2017-06-07 10:50:21 -04:00
progress.puts " [DONE]".color(:green)
2013-04-10 08:11:45 -04:00
else
puts " [FAILED]".color(:red)
puts "failed: #{cmd}"
puts output
2013-04-10 08:11:45 -04:00
end
2013-04-05 12:01:19 -04:00
end
# rubocop:enable Metrics/AbcSize
2013-04-05 12:01:19 -04:00
protected
def path_to_repo(project)
2014-11-05 11:51:08 -05:00
project.repository.path_to_repo
2013-04-05 12:01:19 -04:00
end
def path_to_bundle(project)
File.join(backup_repos_path, project.disk_path + '.bundle')
end
def path_to_tars(project, dir = nil)
path = File.join(backup_repos_path, project.disk_path)
if dir
File.join(path, "#{dir}.tar")
else
path
end
2013-04-05 12:01:19 -04:00
end
def backup_repos_path
File.join(Gitlab.config.backup.path, 'repositories')
end
def in_path(path)
return unless Dir.exist?(path)
dir_entries = Dir.entries(path)
2017-02-24 07:18:07 -05:00
2017-03-28 10:24:40 -04:00
if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
yield('custom_hooks')
end
2013-04-05 12:01:19 -04:00
end
def prepare
FileUtils.rm_rf(backup_repos_path)
# Ensure the parent dir of backup_repos_path exists
FileUtils.mkdir_p(Gitlab.config.backup.path)
# Fail if somebody raced to create backup_repos_path before us
FileUtils.mkdir(backup_repos_path, mode: 0700)
2013-04-05 12:01:19 -04:00
end
2013-11-05 09:00:48 -05:00
def silent
{ err: '/dev/null', out: '/dev/null' }
2013-11-05 09:00:48 -05:00
end
private
def progress_warn(project, cmd, output)
2017-06-07 10:50:21 -04:00
progress.puts "[WARNING] Executing #{cmd}".color(:orange)
progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
end
2017-06-07 10:50:21 -04:00
def empty_repo?(project_or_wiki)
2017-12-07 10:33:30 -05:00
# Protect against stale caches
project_or_wiki.repository.expire_emptiness_caches
project_or_wiki.repository.empty?
end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
end
2017-06-07 10:50:21 -04:00
def progress
2017-06-07 11:00:32 -04:00
$progress
2017-06-07 10:50:21 -04:00
end
def display_repo_path(project)
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
end
2013-04-05 12:01:19 -04:00
end
end