Merge branch 'restore-backup-when-env-variable-is-passed' into 'master'
Restore backup correctly when "BACKUP" environment variable is passed Closes #26090 See merge request !8477
This commit is contained in:
commit
e93cd61793
8 changed files with 150 additions and 41 deletions
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
title: Restore backup correctly when "BACKUP" environment variable is passed
|
||||||
|
merge_request: 8477
|
||||||
|
author:
|
|
@ -9,6 +9,9 @@ This archive will be saved in `backup_path`, which is specified in the
|
||||||
The filename will be `[TIMESTAMP]_gitlab_backup.tar`, where `TIMESTAMP`
|
The filename will be `[TIMESTAMP]_gitlab_backup.tar`, where `TIMESTAMP`
|
||||||
identifies the time at which each backup was created.
|
identifies the time at which each backup was created.
|
||||||
|
|
||||||
|
> In GitLab 8.15 we changed the timestamp format from `EPOCH` (`1393513186`)
|
||||||
|
> to `EPOCH_YYYY_MM_DD` (`1393513186_2014_02_27`)
|
||||||
|
|
||||||
You can only restore a backup to exactly the same version of GitLab on which it
|
You can only restore a backup to exactly the same version of GitLab on which it
|
||||||
was created. The best way to migrate your repositories from one server to
|
was created. The best way to migrate your repositories from one server to
|
||||||
another is through backup restore.
|
another is through backup restore.
|
||||||
|
@ -223,7 +226,8 @@ For installations from source:
|
||||||
|
|
||||||
## Backup archive permissions
|
## Backup archive permissions
|
||||||
|
|
||||||
The backup archives created by GitLab (123456_gitlab_backup.tar) will have owner/group git:git and 0600 permissions by default.
|
The backup archives created by GitLab (`1393513186_2014_02_27_gitlab_backup.tar`)
|
||||||
|
will have owner/group git:git and 0600 permissions by default.
|
||||||
This is meant to avoid other system users reading GitLab's data.
|
This is meant to avoid other system users reading GitLab's data.
|
||||||
If you need the backup archives to have different permissions you can use the 'archive_permissions' setting.
|
If you need the backup archives to have different permissions you can use the 'archive_permissions' setting.
|
||||||
|
|
||||||
|
@ -335,7 +339,7 @@ First make sure your backup tar file is in the backup directory described in the
|
||||||
`/var/opt/gitlab/backups`.
|
`/var/opt/gitlab/backups`.
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo cp 1393513186_gitlab_backup.tar /var/opt/gitlab/backups/
|
sudo cp 1393513186_2014_02_27_gitlab_backup.tar /var/opt/gitlab/backups/
|
||||||
```
|
```
|
||||||
|
|
||||||
Stop the processes that are connected to the database. Leave the rest of GitLab
|
Stop the processes that are connected to the database. Leave the rest of GitLab
|
||||||
|
|
|
@ -2,6 +2,7 @@ module Backup
|
||||||
class Manager
|
class Manager
|
||||||
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry]
|
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry]
|
||||||
FOLDERS_TO_BACKUP = %w[repositories db]
|
FOLDERS_TO_BACKUP = %w[repositories db]
|
||||||
|
FILE_NAME_SUFFIX = '_gitlab_backup.tar'
|
||||||
|
|
||||||
def pack
|
def pack
|
||||||
# Make sure there is a connection
|
# Make sure there is a connection
|
||||||
|
@ -14,7 +15,7 @@ module Backup
|
||||||
s[:gitlab_version] = Gitlab::VERSION
|
s[:gitlab_version] = Gitlab::VERSION
|
||||||
s[:tar_version] = tar_version
|
s[:tar_version] = tar_version
|
||||||
s[:skipped] = ENV["SKIP"]
|
s[:skipped] = ENV["SKIP"]
|
||||||
tar_file = s[:backup_created_at].strftime('%s_%Y_%m_%d') + '_gitlab_backup.tar'
|
tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}"
|
||||||
|
|
||||||
Dir.chdir(Gitlab.config.backup.path) do
|
Dir.chdir(Gitlab.config.backup.path) do
|
||||||
File.open("#{Gitlab.config.backup.path}/backup_information.yml",
|
File.open("#{Gitlab.config.backup.path}/backup_information.yml",
|
||||||
|
@ -82,7 +83,7 @@ module Backup
|
||||||
removed = 0
|
removed = 0
|
||||||
|
|
||||||
Dir.chdir(Gitlab.config.backup.path) do
|
Dir.chdir(Gitlab.config.backup.path) do
|
||||||
Dir.glob('*_gitlab_backup.tar').each do |file|
|
Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file|
|
||||||
next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/
|
next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/
|
||||||
|
|
||||||
timestamp = $1.to_i
|
timestamp = $1.to_i
|
||||||
|
@ -108,41 +109,50 @@ module Backup
|
||||||
Dir.chdir(Gitlab.config.backup.path)
|
Dir.chdir(Gitlab.config.backup.path)
|
||||||
|
|
||||||
# check for existing backups in the backup dir
|
# check for existing backups in the backup dir
|
||||||
file_list = Dir.glob("*_gitlab_backup.tar")
|
file_list = Dir.glob("*#{FILE_NAME_SUFFIX}")
|
||||||
puts "no backups found" if file_list.count == 0
|
|
||||||
|
if file_list.count == 0
|
||||||
|
$progress.puts "No backups found in #{Gitlab.config.backup.path}"
|
||||||
|
$progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
|
||||||
|
exit 1
|
||||||
|
end
|
||||||
|
|
||||||
if file_list.count > 1 && ENV["BACKUP"].nil?
|
if file_list.count > 1 && ENV["BACKUP"].nil?
|
||||||
puts "Found more than one backup, please specify which one you want to restore:"
|
$progress.puts 'Found more than one backup, please specify which one you want to restore:'
|
||||||
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
|
$progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
|
||||||
exit 1
|
exit 1
|
||||||
end
|
end
|
||||||
|
|
||||||
tar_file = ENV["BACKUP"].nil? ? file_list.first : file_list.grep(ENV['BACKUP']).first
|
if ENV['BACKUP'].present?
|
||||||
|
tar_file = "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}"
|
||||||
|
else
|
||||||
|
tar_file = file_list.first
|
||||||
|
end
|
||||||
|
|
||||||
unless File.exist?(tar_file)
|
unless File.exist?(tar_file)
|
||||||
puts "The specified backup doesn't exist!"
|
$progress.puts "The backup file #{tar_file} does not exist!"
|
||||||
exit 1
|
exit 1
|
||||||
end
|
end
|
||||||
|
|
||||||
$progress.print "Unpacking backup ... "
|
$progress.print 'Unpacking backup ... '
|
||||||
|
|
||||||
unless Kernel.system(*%W(tar -xf #{tar_file}))
|
unless Kernel.system(*%W(tar -xf #{tar_file}))
|
||||||
puts "unpacking backup failed".color(:red)
|
$progress.puts 'unpacking backup failed'.color(:red)
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
$progress.puts "done".color(:green)
|
$progress.puts 'done'.color(:green)
|
||||||
end
|
end
|
||||||
|
|
||||||
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
|
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
|
||||||
|
|
||||||
# restoring mismatching backups can lead to unexpected problems
|
# restoring mismatching backups can lead to unexpected problems
|
||||||
if settings[:gitlab_version] != Gitlab::VERSION
|
if settings[:gitlab_version] != Gitlab::VERSION
|
||||||
puts "GitLab version mismatch:".color(:red)
|
$progress.puts 'GitLab version mismatch:'.color(:red)
|
||||||
puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red)
|
$progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red)
|
||||||
puts " Please switch to the following version and try again:".color(:red)
|
$progress.puts ' Please switch to the following version and try again:'.color(:red)
|
||||||
puts " version: #{settings[:gitlab_version]}".color(:red)
|
$progress.puts " version: #{settings[:gitlab_version]}".color(:red)
|
||||||
puts
|
$progress.puts
|
||||||
puts "Hint: git checkout v#{settings[:gitlab_version]}"
|
$progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
|
||||||
exit 1
|
exit 1
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -2,10 +2,11 @@ require 'spec_helper'
|
||||||
require 'rainbow/ext/string'
|
require 'rainbow/ext/string'
|
||||||
|
|
||||||
describe 'seed production settings', lib: true do
|
describe 'seed production settings', lib: true do
|
||||||
|
include StubENV
|
||||||
|
|
||||||
context 'GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN is set in the environment' do
|
context 'GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN is set in the environment' do
|
||||||
before do
|
before do
|
||||||
allow(ENV).to receive(:[]).and_call_original
|
stub_env('GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN', '013456789')
|
||||||
allow(ENV).to receive(:[]).with('GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN').and_return('013456789')
|
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'writes the token to the database' do
|
it 'writes the token to the database' do
|
||||||
|
|
|
@ -2,10 +2,11 @@ require 'spec_helper'
|
||||||
require_relative '../../config/initializers/secret_token'
|
require_relative '../../config/initializers/secret_token'
|
||||||
|
|
||||||
describe 'create_tokens', lib: true do
|
describe 'create_tokens', lib: true do
|
||||||
|
include StubENV
|
||||||
|
|
||||||
let(:secrets) { ActiveSupport::OrderedOptions.new }
|
let(:secrets) { ActiveSupport::OrderedOptions.new }
|
||||||
|
|
||||||
before do
|
before do
|
||||||
allow(ENV).to receive(:[]).and_call_original
|
|
||||||
allow(File).to receive(:write)
|
allow(File).to receive(:write)
|
||||||
allow(File).to receive(:delete)
|
allow(File).to receive(:delete)
|
||||||
allow(Rails).to receive_message_chain(:application, :secrets).and_return(secrets)
|
allow(Rails).to receive_message_chain(:application, :secrets).and_return(secrets)
|
||||||
|
@ -17,7 +18,7 @@ describe 'create_tokens', lib: true do
|
||||||
context 'setting secret_key_base and otp_key_base' do
|
context 'setting secret_key_base and otp_key_base' do
|
||||||
context 'when none of the secrets exist' do
|
context 'when none of the secrets exist' do
|
||||||
before do
|
before do
|
||||||
allow(ENV).to receive(:[]).with('SECRET_KEY_BASE').and_return(nil)
|
stub_env('SECRET_KEY_BASE', nil)
|
||||||
allow(File).to receive(:exist?).with('.secret').and_return(false)
|
allow(File).to receive(:exist?).with('.secret').and_return(false)
|
||||||
allow(File).to receive(:exist?).with('config/secrets.yml').and_return(false)
|
allow(File).to receive(:exist?).with('config/secrets.yml').and_return(false)
|
||||||
allow(self).to receive(:warn_missing_secret)
|
allow(self).to receive(:warn_missing_secret)
|
||||||
|
@ -69,7 +70,7 @@ describe 'create_tokens', lib: true do
|
||||||
|
|
||||||
context 'when secret_key_base exists in the environment and secrets.yml' do
|
context 'when secret_key_base exists in the environment and secrets.yml' do
|
||||||
before do
|
before do
|
||||||
allow(ENV).to receive(:[]).with('SECRET_KEY_BASE').and_return('env_key')
|
stub_env('SECRET_KEY_BASE', 'env_key')
|
||||||
secrets.secret_key_base = 'secret_key_base'
|
secrets.secret_key_base = 'secret_key_base'
|
||||||
secrets.otp_key_base = 'otp_key_base'
|
secrets.otp_key_base = 'otp_key_base'
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,9 +1,27 @@
|
||||||
require 'spec_helper'
|
require 'spec_helper'
|
||||||
|
|
||||||
describe Backup::Manager, lib: true do
|
describe Backup::Manager, lib: true do
|
||||||
describe '#remove_old' do
|
include StubENV
|
||||||
let(:progress) { StringIO.new }
|
|
||||||
|
|
||||||
|
let(:progress) { StringIO.new }
|
||||||
|
|
||||||
|
before do
|
||||||
|
allow(progress).to receive(:puts)
|
||||||
|
allow(progress).to receive(:print)
|
||||||
|
|
||||||
|
allow_any_instance_of(String).to receive(:color) do |string, _color|
|
||||||
|
string
|
||||||
|
end
|
||||||
|
|
||||||
|
@old_progress = $progress # rubocop:disable Style/GlobalVars
|
||||||
|
$progress = progress # rubocop:disable Style/GlobalVars
|
||||||
|
end
|
||||||
|
|
||||||
|
after do
|
||||||
|
$progress = @old_progress # rubocop:disable Style/GlobalVars
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#remove_old' do
|
||||||
let(:files) do
|
let(:files) do
|
||||||
[
|
[
|
||||||
'1451606400_2016_01_01_gitlab_backup.tar',
|
'1451606400_2016_01_01_gitlab_backup.tar',
|
||||||
|
@ -20,20 +38,6 @@ describe Backup::Manager, lib: true do
|
||||||
allow(Dir).to receive(:glob).and_return(files)
|
allow(Dir).to receive(:glob).and_return(files)
|
||||||
allow(FileUtils).to receive(:rm)
|
allow(FileUtils).to receive(:rm)
|
||||||
allow(Time).to receive(:now).and_return(Time.utc(2016))
|
allow(Time).to receive(:now).and_return(Time.utc(2016))
|
||||||
|
|
||||||
allow(progress).to receive(:puts)
|
|
||||||
allow(progress).to receive(:print)
|
|
||||||
|
|
||||||
allow_any_instance_of(String).to receive(:color) do |string, _color|
|
|
||||||
string
|
|
||||||
end
|
|
||||||
|
|
||||||
@old_progress = $progress # rubocop:disable Style/GlobalVars
|
|
||||||
$progress = progress # rubocop:disable Style/GlobalVars
|
|
||||||
end
|
|
||||||
|
|
||||||
after do
|
|
||||||
$progress = @old_progress # rubocop:disable Style/GlobalVars
|
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'when keep_time is zero' do
|
context 'when keep_time is zero' do
|
||||||
|
@ -124,4 +128,82 @@ describe Backup::Manager, lib: true do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe '#unpack' do
|
||||||
|
before do
|
||||||
|
allow(Dir).to receive(:chdir)
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when there are no backup files in the directory' do
|
||||||
|
before do
|
||||||
|
allow(Dir).to receive(:glob).and_return([])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails the operation and prints an error' do
|
||||||
|
expect { subject.unpack }.to raise_error SystemExit
|
||||||
|
expect(progress).to have_received(:puts)
|
||||||
|
.with(a_string_matching('No backups found'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when there are two backup files in the directory and BACKUP variable is not set' do
|
||||||
|
before do
|
||||||
|
allow(Dir).to receive(:glob).and_return(
|
||||||
|
[
|
||||||
|
'1451606400_2016_01_01_gitlab_backup.tar',
|
||||||
|
'1451520000_2015_12_31_gitlab_backup.tar',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails the operation and prints an error' do
|
||||||
|
expect { subject.unpack }.to raise_error SystemExit
|
||||||
|
expect(progress).to have_received(:puts)
|
||||||
|
.with(a_string_matching('Found more than one backup'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when BACKUP variable is set to a non-existing file' do
|
||||||
|
before do
|
||||||
|
allow(Dir).to receive(:glob).and_return(
|
||||||
|
[
|
||||||
|
'1451606400_2016_01_01_gitlab_backup.tar'
|
||||||
|
]
|
||||||
|
)
|
||||||
|
allow(File).to receive(:exist?).and_return(false)
|
||||||
|
|
||||||
|
stub_env('BACKUP', 'wrong')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails the operation and prints an error' do
|
||||||
|
expect { subject.unpack }.to raise_error SystemExit
|
||||||
|
expect(File).to have_received(:exist?).with('wrong_gitlab_backup.tar')
|
||||||
|
expect(progress).to have_received(:puts)
|
||||||
|
.with(a_string_matching('The backup file wrong_gitlab_backup.tar does not exist'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when BACKUP variable is set to a correct file' do
|
||||||
|
before do
|
||||||
|
allow(Dir).to receive(:glob).and_return(
|
||||||
|
[
|
||||||
|
'1451606400_2016_01_01_gitlab_backup.tar'
|
||||||
|
]
|
||||||
|
)
|
||||||
|
allow(File).to receive(:exist?).and_return(true)
|
||||||
|
allow(Kernel).to receive(:system).and_return(true)
|
||||||
|
allow(YAML).to receive(:load_file).and_return(gitlab_version: Gitlab::VERSION)
|
||||||
|
|
||||||
|
stub_env('BACKUP', '1451606400_2016_01_01')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'unpacks the file' do
|
||||||
|
subject.unpack
|
||||||
|
|
||||||
|
expect(Kernel).to have_received(:system)
|
||||||
|
.with("tar", "-xf", "1451606400_2016_01_01_gitlab_backup.tar")
|
||||||
|
expect(progress).to have_received(:puts).with(a_string_matching('done'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
7
spec/support/stub_env.rb
Normal file
7
spec/support/stub_env.rb
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
module StubENV
|
||||||
|
def stub_env(key, value)
|
||||||
|
allow(ENV).to receive(:[]).and_call_original unless @env_already_stubbed
|
||||||
|
@env_already_stubbed ||= true
|
||||||
|
allow(ENV).to receive(:[]).with(key).and_return(value)
|
||||||
|
end
|
||||||
|
end
|
|
@ -41,7 +41,7 @@ describe 'gitlab:app namespace rake task' do
|
||||||
|
|
||||||
context 'gitlab version' do
|
context 'gitlab version' do
|
||||||
before do
|
before do
|
||||||
allow(Dir).to receive(:glob).and_return([])
|
allow(Dir).to receive(:glob).and_return(['1_gitlab_backup.tar'])
|
||||||
allow(Dir).to receive(:chdir)
|
allow(Dir).to receive(:chdir)
|
||||||
allow(File).to receive(:exist?).and_return(true)
|
allow(File).to receive(:exist?).and_return(true)
|
||||||
allow(Kernel).to receive(:system).and_return(true)
|
allow(Kernel).to receive(:system).and_return(true)
|
||||||
|
|
Loading…
Reference in a new issue