kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Merge branch 'backup-cron-mode' into 'master'

Browse source 
Backup cron mode

If you have your server configured to receive emails containing the output of
cron jobs it is annoying to get long spammy emails from the backup script. This
change adds a 'cron mode' that makes the backup script silent unless something
goes wrong during the backup.

See merge request !1268
This commit is contained in:
Valery Sizov 2014-11-24 09:33:39 +00:00
commit 5f7574cf05
6 changed files with 77 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG
View file

@ -1,7 +1,7 @@
v 7.6.0
- Fork repository to groups
- New rugged version
-
- Add CRON=1 backup setting for quiet backups
-
-
-

5
doc/raketasks/backup_restore.md
View file

@ -203,5 +203,8 @@ Add the following lines at the bottom:
```
# Create a full backup of the GitLab repositories and SQL database every day at 4am
0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production
0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production CRON=1
```
The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors.
This is recommended to reduce cron spam.

12
lib/backup/database.rb
View file

@ -13,10 +13,10 @@ module Backup
def dump
success = case config["adapter"]
when /^mysql/ then
print "Dumping MySQL database #{config['database']} ... "
$progress.print "Dumping MySQL database #{config['database']} ... "
system('mysqldump', *mysql_args, config['database'], out: db_file_name)
when "postgresql" then
print "Dumping PostgreSQL database #{config['database']} ... "
$progress.print "Dumping PostgreSQL database #{config['database']} ... "
pg_env
system('pg_dump', config['database'], out: db_file_name)
end
@ -27,10 +27,10 @@ module Backup
def restore
success = case config["adapter"]
when /^mysql/ then
print "Restoring MySQL database #{config['database']} ... "
$progress.print "Restoring MySQL database #{config['database']} ... "
system('mysql', *mysql_args, config['database'], in: db_file_name)
when "postgresql" then
print "Restoring PostgreSQL database #{config['database']} ... "
$progress.print "Restoring PostgreSQL database #{config['database']} ... "
# Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
# statements like MySQL.
Rake::Task["gitlab:db:drop_all_tables"].invoke
@ -69,9 +69,9 @@ module Backup
def report_success(success)
if success
puts '[DONE]'.green
$progress.puts '[DONE]'.green
else
puts '[FAILED]'.red
$progress.puts '[FAILED]'.red
end
end
end

32
lib/backup/manager.rb
View file

@ -18,11 +18,11 @@ module Backup
end
# create archive
print "Creating backup archive: #{tar_file} ... "
$progress.print "Creating backup archive: #{tar_file} ... "
if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS)
puts "done".green
$progress.puts "done".green
else
puts "failed".red
puts "creating archive #{tar_file} failed".red
abort 'Backup failed'
end
@ -31,37 +31,37 @@ module Backup
def upload(tar_file)
remote_directory = Gitlab.config.backup.upload.remote_directory
print "Uploading backup archive to remote storage #{remote_directory} ... "
$progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
connection_settings = Gitlab.config.backup.upload.connection
if connection_settings.blank?
puts "skipped".yellow
$progress.puts "skipped".yellow
return
end
connection = ::Fog::Storage.new(connection_settings)
directory = connection.directories.get(remote_directory)
if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
puts "done".green
$progress.puts "done".green
else
puts "failed".red
puts "uploading backup to #{remote_directory} failed".red
abort 'Backup failed'
end
end
def cleanup
print "Deleting tmp directories ... "
$progress.print "Deleting tmp directories ... "
if Kernel.system('rm', '-rf', *BACKUP_CONTENTS)
puts "done".green
$progress.puts "done".green
else
puts "failed".red
puts "deleting tmp directory failed".red
abort 'Backup failed'
end
end
def remove_old
# delete backups
print "Deleting old backups ... "
$progress.print "Deleting old backups ... "
keep_time = Gitlab.config.backup.keep_time.to_i
path = Gitlab.config.backup.path
@ -76,9 +76,9 @@ module Backup
end
end
end
puts "done. (#{removed} removed)".green
$progress.puts "done. (#{removed} removed)".green
else
puts "skipping".yellow
$progress.puts "skipping".yellow
end
end
@ -101,12 +101,12 @@ module Backup
exit 1
end
print "Unpacking backup ... "
$progress.print "Unpacking backup ... "
unless Kernel.system(*%W(tar -xf #{tar_file}))
puts "failed".red
puts "unpacking backup failed".red
exit 1
else
puts "done".green
$progress.puts "done".green
end
settings = YAML.load_file("backup_information.yml")

41
lib/backup/repository.rb
View file

@ -8,19 +8,21 @@ module Backup
prepare
Project.find_each(batch_size: 1000) do |project|
print " * #{project.path_with_namespace} ... "
$progress.print " * #{project.path_with_namespace} ... "
# Create namespace dir if missing
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
if project.empty_repo?
puts "[SKIPPED]".cyan
$progress.puts "[SKIPPED]".cyan
else
output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all))
cmd = %W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
puts "[DONE]".green
$progress.puts "[DONE]".green
else
puts "[FAILED]".red
puts "failed: #{cmd.join(' ')}"
puts output
abort 'Backup failed'
end
@ -29,15 +31,17 @@ module Backup
wiki = ProjectWiki.new(project)
if File.exists?(path_to_repo(wiki))
print " * #{wiki.path_with_namespace} ... "
$progress.print " * #{wiki.path_with_namespace} ... "
if wiki.repository.empty?
puts " [SKIPPED]".cyan
$progress.puts " [SKIPPED]".cyan
else
output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all))
cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
puts " [DONE]".green
$progress.puts " [DONE]".green
else
puts " [FAILED]".red
puts "failed: #{cmd.join(' ')}"
abort 'Backup failed'
end
end
@ -55,7 +59,7 @@ module Backup
FileUtils.mkdir_p(repos_path)
Project.find_each(batch_size: 1000) do |project|
print "#{project.path_with_namespace} ... "
$progress.print "#{project.path_with_namespace} ... "
project.namespace.ensure_dir_exist if project.namespace
@ -66,30 +70,35 @@ module Backup
end
if system(*cmd, silent)
puts "[DONE]".green
$progress.puts "[DONE]".green
else
puts "[FAILED]".red
puts "failed: #{cmd.join(' ')}"
abort 'Restore failed'
end
wiki = ProjectWiki.new(project)
if File.exists?(path_to_bundle(wiki))
print " * #{wiki.path_with_namespace} ... "
if system(*%W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}), silent)
puts " [DONE]".green
$progress.print " * #{wiki.path_with_namespace} ... "
cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
if system(*cmd, silent)
$progress.puts " [DONE]".green
else
puts " [FAILED]".red
puts "failed: #{cmd.join(' ')}"
abort 'Restore failed'
end
end
end
print 'Put GitLab hooks in repositories dirs'.yellow
if system("#{Gitlab.config.gitlab_shell.path}/bin/create-hooks")
puts " [DONE]".green
$progress.print 'Put GitLab hooks in repositories dirs'.yellow
cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks"
if system(cmd)
$progress.puts " [DONE]".green
else
puts " [FAILED]".red
puts "failed: #{cmd}"
end
end

37
lib/tasks/gitlab/backup.rake
View file

@ -6,6 +6,7 @@ namespace :gitlab do
desc "GITLAB | Create a backup of the GitLab system"
task create: :environment do
warn_user_is_not_gitlab
configure_cron_mode
Rake::Task["gitlab:backup:db:create"].invoke
Rake::Task["gitlab:backup:repo:create"].invoke
@ -21,6 +22,7 @@ namespace :gitlab do
desc "GITLAB | Restore a previously created backup"
task restore: :environment do
warn_user_is_not_gitlab
configure_cron_mode
backup = Backup::Manager.new
backup.unpack
@ -35,43 +37,54 @@ namespace :gitlab do
namespace :repo do
task create: :environment do
puts "Dumping repositories ...".blue
$progress.puts "Dumping repositories ...".blue
Backup::Repository.new.dump
puts "done".green
$progress.puts "done".green
end
task restore: :environment do
puts "Restoring repositories ...".blue
$progress.puts "Restoring repositories ...".blue
Backup::Repository.new.restore
puts "done".green
$progress.puts "done".green
end
end
namespace :db do
task create: :environment do
puts "Dumping database ... ".blue
$progress.puts "Dumping database ... ".blue
Backup::Database.new.dump
puts "done".green
$progress.puts "done".green
end
task restore: :environment do
puts "Restoring database ... ".blue
$progress.puts "Restoring database ... ".blue
Backup::Database.new.restore
puts "done".green
$progress.puts "done".green
end
end
namespace :uploads do
task create: :environment do
puts "Dumping uploads ... ".blue
$progress.puts "Dumping uploads ... ".blue
Backup::Uploads.new.dump
puts "done".green
$progress.puts "done".green
end
task restore: :environment do
puts "Restoring uploads ... ".blue
$progress.puts "Restoring uploads ... ".blue
Backup::Uploads.new.restore
puts "done".green
$progress.puts "done".green
end
end
def configure_cron_mode
if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a
# StringIO.
require 'stringio'
$progress = StringIO.new
else
$progress = $stdout
end
end
end # namespace end: backup