Remove feature flags from lib/backup
Moved to OPT_OUT in 7d14b725a0, Now, by
removing the feature gates, this is an mandatory feature.
Related issues:
- https://gitlab.com/gitlab-org/gitaly/issues/526
- https://gitlab.com/gitlab-org/gitaly/issues/1194
Closes https://gitlab.com/gitlab-org/gitaly/issues/749
Closes https://gitlab.com/gitlab-org/gitaly/issues/1212
Closes https://gitlab.com/gitlab-org/gitaly/issues/1195
This commit is contained in:
Zeger-Jan van de Weg
parent
5035250337
commit
d7afed34c4
4 changed files with 72 additions and 177 deletions
|
|
@ -1,10 +1,7 @@
|
|||
require 'yaml'
|
||||
require_relative 'helper'
|
||||
|
||||
module Backup
|
||||
class Repository
|
||||
include Backup::Helper
|
||||
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
|
|
@ -42,131 +39,36 @@ module Backup
|
|||
end
|
||||
|
||||
def prepare_directories
|
||||
Gitlab.config.repositories.storages.each do |name, repository_storage|
|
||||
delete_all_repositories(name, repository_storage)
|
||||
Gitlab.config.repositories.storages.each do |name, _repository_storage|
|
||||
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
|
||||
end
|
||||
end
|
||||
|
||||
def backup_project(project)
|
||||
gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
|
||||
if is_enabled
|
||||
backup_project_gitaly(project)
|
||||
else
|
||||
backup_project_local(project)
|
||||
end
|
||||
end
|
||||
path_to_project_bundle = path_to_bundle(project)
|
||||
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
||||
.create_bundle(path_to_project_bundle)
|
||||
|
||||
backup_custom_hooks(project)
|
||||
rescue => e
|
||||
progress_warn(project, e, 'Failed to backup repo')
|
||||
end
|
||||
|
||||
def backup_project_gitaly(project)
|
||||
path_to_project_bundle = path_to_bundle(project)
|
||||
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
||||
.create_bundle(path_to_project_bundle)
|
||||
end
|
||||
def backup_custom_hooks(project)
|
||||
FileUtils.mkdir_p(project_backup_path(project))
|
||||
|
||||
def backup_project_local(project)
|
||||
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
||||
path_to_repo(project)
|
||||
end
|
||||
|
||||
path_to_project_bundle = path_to_bundle(project)
|
||||
|
||||
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
|
||||
output, status = Gitlab::Popen.popen(cmd)
|
||||
progress_warn(project, cmd.join(' '), output) unless status.zero?
|
||||
end
|
||||
|
||||
def delete_all_repositories(name, repository_storage)
|
||||
gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
|
||||
if is_enabled
|
||||
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
|
||||
else
|
||||
local_delete_all_repositories(name, repository_storage)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def local_delete_all_repositories(name, repository_storage)
|
||||
path = repository_storage.legacy_disk_path
|
||||
return unless File.exist?(path)
|
||||
|
||||
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
|
||||
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
|
||||
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
|
||||
|
||||
begin
|
||||
FileUtils.mv(files, bk_repos_path)
|
||||
rescue Errno::EACCES
|
||||
access_denied_error(path)
|
||||
rescue Errno::EBUSY
|
||||
resource_busy_error(path)
|
||||
end
|
||||
end
|
||||
|
||||
def local_restore_custom_hooks(project, dir)
|
||||
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
||||
path_to_repo(project)
|
||||
end
|
||||
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
|
||||
output, status = Gitlab::Popen.popen(cmd)
|
||||
unless status.zero?
|
||||
progress_warn(project, cmd.join(' '), output)
|
||||
end
|
||||
end
|
||||
|
||||
def gitaly_restore_custom_hooks(project, dir)
|
||||
custom_hooks_path = path_to_tars(project, dir)
|
||||
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
||||
.restore_custom_hooks(custom_hooks_path)
|
||||
end
|
||||
|
||||
def local_backup_custom_hooks(project)
|
||||
in_path(path_to_tars(project)) do |dir|
|
||||
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
||||
path_to_repo(project)
|
||||
end
|
||||
break unless File.exist?(File.join(path_to_project_repo, dir))
|
||||
|
||||
FileUtils.mkdir_p(path_to_tars(project))
|
||||
cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
|
||||
output, status = Gitlab::Popen.popen(cmd)
|
||||
|
||||
unless status.zero?
|
||||
progress_warn(project, cmd.join(' '), output)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def gitaly_backup_custom_hooks(project)
|
||||
FileUtils.mkdir_p(path_to_tars(project))
|
||||
custom_hooks_path = path_to_tars(project, 'custom_hooks')
|
||||
custom_hooks_path = custom_hooks_tar(project)
|
||||
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
||||
.backup_custom_hooks(custom_hooks_path)
|
||||
end
|
||||
|
||||
def backup_custom_hooks(project)
|
||||
gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
|
||||
if is_enabled
|
||||
gitaly_backup_custom_hooks(project)
|
||||
else
|
||||
local_backup_custom_hooks(project)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def restore_custom_hooks(project)
|
||||
in_path(path_to_tars(project)) do |dir|
|
||||
gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
|
||||
if is_enabled
|
||||
gitaly_restore_custom_hooks(project, dir)
|
||||
else
|
||||
local_restore_custom_hooks(project, dir)
|
||||
end
|
||||
end
|
||||
end
|
||||
return unless Dir.exist?(project_backup_path(project))
|
||||
return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
|
||||
|
||||
custom_hooks_path = custom_hooks_tar(project)
|
||||
Gitlab::GitalyClient::RepositoryService.new(project.repository)
|
||||
.restore_custom_hooks(custom_hooks_path)
|
||||
end
|
||||
|
||||
def restore
|
||||
|
|
@ -181,7 +83,8 @@ module Backup
|
|||
restore_repo_success = nil
|
||||
if File.exist?(path_to_project_bundle)
|
||||
begin
|
||||
project.repository.create_from_bundle path_to_project_bundle
|
||||
project.repository.create_from_bundle(path_to_project_bundle)
|
||||
restore_custom_hooks(project)
|
||||
restore_repo_success = true
|
||||
rescue => e
|
||||
restore_repo_success = false
|
||||
|
|
@ -197,8 +100,6 @@ module Backup
|
|||
progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
|
||||
end
|
||||
|
||||
restore_custom_hooks(project)
|
||||
|
||||
wiki = ProjectWiki.new(project)
|
||||
path_to_wiki_bundle = path_to_bundle(wiki)
|
||||
|
||||
|
|
@ -219,48 +120,28 @@ module Backup
|
|||
|
||||
protected
|
||||
|
||||
def path_to_repo(project)
|
||||
project.repository.path_to_repo
|
||||
end
|
||||
|
||||
def path_to_bundle(project)
|
||||
File.join(backup_repos_path, project.disk_path + '.bundle')
|
||||
end
|
||||
|
||||
def path_to_tars(project, dir = nil)
|
||||
path = File.join(backup_repos_path, project.disk_path)
|
||||
def project_backup_path(project)
|
||||
File.join(backup_repos_path, project.disk_path)
|
||||
end
|
||||
|
||||
if dir
|
||||
File.join(path, "#{dir}.tar")
|
||||
else
|
||||
path
|
||||
end
|
||||
def custom_hooks_tar(project)
|
||||
File.join(project_backup_path(project), "custom_hooks.tar")
|
||||
end
|
||||
|
||||
def backup_repos_path
|
||||
File.join(Gitlab.config.backup.path, 'repositories')
|
||||
end
|
||||
|
||||
def in_path(path)
|
||||
return unless Dir.exist?(path)
|
||||
|
||||
dir_entries = Dir.entries(path)
|
||||
|
||||
if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
|
||||
yield('custom_hooks')
|
||||
end
|
||||
end
|
||||
|
||||
def prepare
|
||||
FileUtils.rm_rf(backup_repos_path)
|
||||
FileUtils.mkdir_p(Gitlab.config.backup.path)
|
||||
FileUtils.mkdir(backup_repos_path, mode: 0700)
|
||||
end
|
||||
|
||||
def silent
|
||||
{ err: '/dev/null', out: '/dev/null' }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def progress_warn(project, cmd, output)
|
||||
|
|
@ -273,18 +154,8 @@ module Backup
|
|||
project_or_wiki.repository.empty?
|
||||
end
|
||||
|
||||
def repository_storage_paths_args
|
||||
Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
|
||||
end
|
||||
|
||||
def display_repo_path(project)
|
||||
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
|
||||
end
|
||||
|
||||
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
|
||||
Gitlab::GitalyClient.migrate(method, status: status, &block)
|
||||
rescue GRPC::NotFound, GRPC::BadStatus => e
|
||||
raise Error, e
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -73,37 +73,27 @@ describe Backup::Repository do
|
|||
end
|
||||
end
|
||||
|
||||
describe '#delete_all_repositories', :seed_helper do
|
||||
shared_examples('delete_all_repositories') do
|
||||
before do
|
||||
allow(FileUtils).to receive(:mkdir_p).and_call_original
|
||||
allow(FileUtils).to receive(:mv).and_call_original
|
||||
end
|
||||
|
||||
after(:all) do
|
||||
ensure_seeds
|
||||
end
|
||||
|
||||
it 'removes all repositories' do
|
||||
# Sanity check: there should be something for us to delete
|
||||
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
|
||||
|
||||
subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default'])
|
||||
|
||||
expect(list_repositories).to be_empty
|
||||
end
|
||||
|
||||
def list_repositories
|
||||
Dir[File.join(SEED_STORAGE_PATH, '*.git')]
|
||||
end
|
||||
describe '#prepare_directories', :seed_helper do
|
||||
before do
|
||||
allow(FileUtils).to receive(:mkdir_p).and_call_original
|
||||
allow(FileUtils).to receive(:mv).and_call_original
|
||||
end
|
||||
|
||||
context 'with gitaly' do
|
||||
it_behaves_like 'delete_all_repositories'
|
||||
after(:all) do
|
||||
ensure_seeds
|
||||
end
|
||||
|
||||
context 'without gitaly', :skip_gitaly_mock do
|
||||
it_behaves_like 'delete_all_repositories'
|
||||
it' removes all repositories' do
|
||||
# Sanity check: there should be something for us to delete
|
||||
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
|
||||
|
||||
subject.prepare_directories
|
||||
|
||||
expect(list_repositories).to be_empty
|
||||
end
|
||||
|
||||
def list_repositories
|
||||
Dir[File.join(SEED_STORAGE_PATH, '*.git')]
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
13
spec/lib/gitlab/gitaly_client/storage_service_spec.rb
Normal file
13
spec/lib/gitlab/gitaly_client/storage_service_spec.rb
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Gitlab::GitalyClient::StorageService do
|
||||
describe '#delete_all_repositories' do
|
||||
let!(:project) { create(:project, :repository) }
|
||||
|
||||
it 'removes all repositories' do
|
||||
described_class.new(project.repository_storage).delete_all_repositories
|
||||
|
||||
expect(project.repository.exists?).to be(false)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -87,6 +87,27 @@ describe 'gitlab:app namespace rake task' do
|
|||
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the restore directory is not empty' do
|
||||
before do
|
||||
# We only need a backup of the repositories for this test
|
||||
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
|
||||
end
|
||||
|
||||
it 'removes stale data' do
|
||||
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
|
||||
|
||||
excluded_project = create(:project, :repository, name: 'mepmep')
|
||||
|
||||
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
|
||||
|
||||
raw_repo = excluded_project.repository.raw
|
||||
|
||||
# The restore will not find the repository in the backup, but will create
|
||||
# an empty one in its place
|
||||
expect(raw_repo.empty?).to be(true)
|
||||
end
|
||||
end
|
||||
end # backup_restore task
|
||||
|
||||
describe 'backup' do
|
||||
|
|
|
|||
Loading…
Reference in a new issue