From 67c83489cac6029ac0eb072b12fd6a9955343cd0 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Fri, 27 Mar 2015 15:18:59 +0200 Subject: [PATCH] ability to skip some items in backup --- CHANGELOG | 1 + doc/raketasks/backup_restore.md | 7 ++++ lib/backup/manager.rb | 42 ++++++++++++++++++---- lib/tasks/gitlab/backup.rake | 33 ++++++++++++----- spec/tasks/gitlab/backup_rake_spec.rb | 51 +++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 15 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 259e1a30072..6d6b114a8b9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -52,6 +52,7 @@ v 7.10.0 (unreleased) - Don't show commit comment button when user is not signed in. - Fix admin user projects lists. - Don't leak private group existence by redirecting from namespace controller to group controller. + - Ability to skip some items from backup (database, respositories or uploads) v 7.9.2 - Contains no changes diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 99cdfff0ac6..2e41fad89e7 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -17,6 +17,13 @@ sudo gitlab-rake gitlab:backup:create sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ``` +Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, +uploads (attachments), repositories. Use a comma to specify several options at the same time. + +``` +sudo gitlab-rake gitlab:backup:create SKIP=db,uploads +``` + Example output: ``` diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index d445150c55e..b69aebf9fe1 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,7 +1,5 @@ module Backup class Manager - BACKUP_CONTENTS = %w{repositories/ db/ uploads/ backup_information.yml} - def pack # saving additional informations s = {} @@ -9,6 +7,7 @@ module Backup s[:backup_created_at] = Time.now s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version + s[:skipped] = ENV["SKIP"] tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar" Dir.chdir(Gitlab.config.backup.path) do @@ -17,12 +16,12 @@ module Backup file << s.to_yaml.gsub(/^---\n/,'') end - FileUtils.chmod(0700, %w{db uploads repositories}) + FileUtils.chmod(0700, folders_to_backup) # create archive $progress.print "Creating backup archive: #{tar_file} ... " orig_umask = File.umask(0077) - if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS) + if Kernel.system('tar', '-cf', tar_file, *backup_contents) $progress.puts "done".green else puts "creating archive #{tar_file} failed".red @@ -46,6 +45,7 @@ module Backup connection = ::Fog::Storage.new(connection_settings) directory = connection.directories.get(remote_directory) + if directory.files.create(key: tar_file, body: File.open(tar_file), public: false) $progress.puts "done".green else @@ -56,7 +56,10 @@ module Backup def cleanup $progress.print "Deleting tmp directories ... " - BACKUP_CONTENTS.each do |dir| + + backup_contents.each do |dir| + next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) $progress.puts "done".green else @@ -73,6 +76,7 @@ module Backup if keep_time > 0 removed = 0 + Dir.chdir(Gitlab.config.backup.path) do file_list = Dir.glob('*_gitlab_backup.tar') file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } @@ -84,6 +88,7 @@ module Backup end end end + $progress.puts "done. (#{removed} removed)".green else $progress.puts "skipping".yellow @@ -96,6 +101,7 @@ module Backup # check for existing backups in the backup dir file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } puts "no backups found" if file_list.count == 0 + if file_list.count > 1 && ENV["BACKUP"].nil? puts "Found more than one backup, please specify which one you want to restore:" puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" @@ -110,6 +116,7 @@ module Backup end $progress.print "Unpacking backup ... " + unless Kernel.system(*%W(tar -xf #{tar_file})) puts "unpacking backup failed".red exit 1 @@ -117,7 +124,6 @@ module Backup $progress.puts "done".green end - settings = YAML.load_file("backup_information.yml") ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems @@ -136,5 +142,29 @@ module Backup tar_version, _ = Gitlab::Popen.popen(%W(tar --version)) tar_version.force_encoding('locale').split("\n").first end + + def skipped?(item) + settings[:skipped] && settings[:skipped].include?(item) + end + + private + + def backup_contents + folders_to_backup + ["backup_information.yml"] + end + + def folders_to_backup + folders = %w{repositories db uploads} + + if ENV["SKIP"] + return folders.reject{ |folder| ENV["SKIP"].include?(folder) } + end + + folders + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 0230fbb010b..84445b3bf2f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -27,9 +27,9 @@ namespace :gitlab do backup = Backup::Manager.new backup.unpack - Rake::Task["gitlab:backup:db:restore"].invoke - Rake::Task["gitlab:backup:repo:restore"].invoke - Rake::Task["gitlab:backup:uploads:restore"].invoke + Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") + Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") + Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -38,8 +38,13 @@ namespace :gitlab do namespace :repo do task create: :environment do $progress.puts "Dumping repositories ...".blue - Backup::Repository.new.dump - $progress.puts "done".green + + if ENV["SKIP"] && ENV["SKIP"].include?("repositories") + $progress.puts "[SKIPPED]".cyan + else + Backup::Repository.new.dump + $progress.puts "done".green + end end task restore: :environment do @@ -52,8 +57,13 @@ namespace :gitlab do namespace :db do task create: :environment do $progress.puts "Dumping database ... ".blue - Backup::Database.new.dump - $progress.puts "done".green + + if ENV["SKIP"] && ENV["SKIP"].include?("db") + $progress.puts "[SKIPPED]".cyan + else + Backup::Database.new.dump + $progress.puts "done".green + end end task restore: :environment do @@ -66,8 +76,13 @@ namespace :gitlab do namespace :uploads do task create: :environment do $progress.puts "Dumping uploads ... ".blue - Backup::Uploads.new.dump - $progress.puts "done".green + + if ENV["SKIP"] && ENV["SKIP"].include?("uploads") + $progress.puts "[SKIPPED]".cyan + else + Backup::Uploads.new.dump + $progress.puts "done".green + end end task restore: :environment do diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 3d5d21c2a13..60144be5510 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -98,4 +98,55 @@ describe 'gitlab:app namespace rake task' do expect(temp_dirs).to be_empty end end # backup_create task + + describe "Skipping items" do + def tars_glob + Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar')) + end + + before :all do + @origin_cd = Dir.pwd + + Rake::Task["gitlab:backup:db:create"].reenable + Rake::Task["gitlab:backup:repo:create"].reenable + Rake::Task["gitlab:backup:uploads:create"].reenable + + # Record the existing backup tars so we don't touch them + existing_tars = tars_glob + + # Redirect STDOUT and run the rake task + orig_stdout = $stdout + $stdout = StringIO.new + ENV["SKIP"] = "repositories" + run_rake_task('gitlab:backup:create') + $stdout = orig_stdout + + @backup_tar = (tars_glob - existing_tars).first + end + + after :all do + FileUtils.rm(@backup_tar) + Dir.chdir @origin_cd + end + + it "does not contain skipped item" do + tar_contents, exit_status = Gitlab::Popen.popen( + %W{tar -tvf #{@backup_tar} db uploads repositories} + ) + + expect(tar_contents).to match('db/') + expect(tar_contents).to match('uploads/') + expect(tar_contents).not_to match('repositories/') + end + + it 'does not invoke repositories restore' do + Rake::Task["gitlab:shell:setup"].stub invoke: true + allow($stdout).to receive :write + + expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke + expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke + expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke + expect { run_rake_task('gitlab:backup:restore') }.to_not raise_error + end + end end # gitlab:app namespace