Merge branch 'sh-fix-backup-specific-rake-task' into 'master'

Fix backup creation and restore for specific Rake tasks

Closes #40617

See merge request gitlab-org/gitlab-ce!19122

changelogs/unreleased/sh-fix-backup-specific-rake-task.yml

@@ -0,0 +1,5 @@
+---
+title: Fix backup creation and restore for specific Rake tasks
+merge_request:
+author:
+type: fixed

lib/backup/artifacts.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Artifacts < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('artifacts', JobArtifactUploader.root)
     end
   end

lib/backup/builds.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Builds < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('builds', Settings.gitlab_ci.builds_path)
     end
   end

lib/backup/database.rb

@@ -2,9 +2,11 @@ require 'yaml'
 
 module Backup
   class Database
+    attr_reader :progress
     attr_reader :config, :db_file_name
 
-    def initialize
+    def initialize(progress)
+      @progress = progress
       @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
@@ -19,12 +21,12 @@ module Backup
       dump_pid =
         case config["adapter"]
         when /^mysql/ then
-          $progress.print "Dumping MySQL database #{config['database']} ... "
+          progress.print "Dumping MySQL database #{config['database']} ... "
           # Workaround warnings from MySQL 5.6 about passwords on cmd line
           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
           spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
         when "postgresql" then
-          $progress.print "Dumping PostgreSQL database #{config['database']} ... "
+          progress.print "Dumping PostgreSQL database #{config['database']} ... "
           pg_env
           pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
           if Gitlab.config.backup.pg_schema
@@ -53,12 +55,12 @@ module Backup
       restore_pid =
         case config["adapter"]
         when /^mysql/ then
-          $progress.print "Restoring MySQL database #{config['database']} ... "
+          progress.print "Restoring MySQL database #{config['database']} ... "
           # Workaround warnings from MySQL 5.6 about passwords on cmd line
           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
           spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
         when "postgresql" then
-          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+          progress.print "Restoring PostgreSQL database #{config['database']} ... "
           pg_env
           spawn('psql', config['database'], in: decompress_rd)
         end
@@ -111,9 +113,9 @@ module Backup
 
     def report_success(success)
       if success
-        $progress.puts '[DONE]'.color(:green)
+        progress.puts '[DONE]'.color(:green)
       else
-        $progress.puts '[FAILED]'.color(:red)
+        progress.puts '[FAILED]'.color(:red)
       end
     end
   end

lib/backup/lfs.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Lfs < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('lfs', Settings.lfs.storage_path)
     end
   end

lib/backup/manager.rb

@@ -4,6 +4,12 @@ module Backup
     FOLDERS_TO_BACKUP = %w[repositories db].freeze
     FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze
 
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+    end
+
     def pack
       # Make sure there is a connection
       ActiveRecord::Base.connection.reconnect!
@@ -14,11 +20,11 @@ module Backup
         end
 
         # create archive
-        $progress.print "Creating backup archive: #{tar_file} ... "
+        progress.print "Creating backup archive: #{tar_file} ... "
         # Set file permissions on open to prevent chmod races.
         tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
         if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
-          $progress.puts "done".color(:green)
+          progress.puts "done".color(:green)
         else
           puts "creating archive #{tar_file} failed".color(:red)
           abort 'Backup failed'
@@ -29,11 +35,11 @@ module Backup
     end
 
     def upload
-      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
+      progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
 
       connection_settings = Gitlab.config.backup.upload.connection
       if connection_settings.blank?
-        $progress.puts "skipped".color(:yellow)
+        progress.puts "skipped".color(:yellow)
         return
       end
 
@@ -43,7 +49,7 @@ module Backup
                                 multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
                                 encryption: Gitlab.config.backup.upload.encryption,
                                 storage_class: Gitlab.config.backup.upload.storage_class)
-        $progress.puts "done".color(:green)
+        progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
         abort 'Backup failed'
@@ -51,13 +57,13 @@ module Backup
     end
 
     def cleanup
-      $progress.print "Deleting tmp directories ... "
+      progress.print "Deleting tmp directories ... "
 
       backup_contents.each do |dir|
         next unless File.exist?(File.join(backup_path, dir))
 
         if FileUtils.rm_rf(File.join(backup_path, dir))
-          $progress.puts "done".color(:green)
+          progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
           abort 'Backup failed'
@@ -67,7 +73,7 @@ module Backup
 
     def remove_old
       # delete backups
-      $progress.print "Deleting old backups ... "
+      progress.print "Deleting old backups ... "
       keep_time = Gitlab.config.backup.keep_time.to_i
 
       if keep_time > 0
@@ -88,31 +94,32 @@ module Backup
                 FileUtils.rm(file)
                 removed += 1
               rescue => e
-                $progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
+                progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
               end
             end
           end
         end
 
-        $progress.puts "done. (#{removed} removed)".color(:green)
+        progress.puts "done. (#{removed} removed)".color(:green)
       else
-        $progress.puts "skipping".color(:yellow)
+        progress.puts "skipping".color(:yellow)
       end
     end
 
+    # rubocop: disable Metrics/AbcSize
     def unpack
       Dir.chdir(backup_path) do
         # check for existing backups in the backup dir
         if backup_file_list.empty?
-          $progress.puts "No backups found in #{backup_path}"
-          $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
+          progress.puts "No backups found in #{backup_path}"
+          progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
           exit 1
         elsif backup_file_list.many? && ENV["BACKUP"].nil?
-          $progress.puts 'Found more than one backup:'
+          progress.puts 'Found more than one backup:'
           # print list of available backups
-          $progress.puts " " + available_timestamps.join("\n ")
-          $progress.puts 'Please specify which one you want to restore:'
-          $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
+          progress.puts " " + available_timestamps.join("\n ")
+          progress.puts 'Please specify which one you want to restore:'
+          progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
           exit 1
         end
 
@@ -123,31 +130,31 @@ module Backup
                    end
 
         unless File.exist?(tar_file)
-          $progress.puts "The backup file #{tar_file} does not exist!"
+          progress.puts "The backup file #{tar_file} does not exist!"
           exit 1
         end
 
-        $progress.print 'Unpacking backup ... '
+        progress.print 'Unpacking backup ... '
 
         unless Kernel.system(*%W(tar -xf #{tar_file}))
-          $progress.puts 'unpacking backup failed'.color(:red)
+          progress.puts 'unpacking backup failed'.color(:red)
           exit 1
         else
-          $progress.puts 'done'.color(:green)
+          progress.puts 'done'.color(:green)
         end
 
         ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
 
         # restoring mismatching backups can lead to unexpected problems
         if settings[:gitlab_version] != Gitlab::VERSION
-          $progress.puts(<<~HEREDOC.color(:red))
+          progress.puts(<<~HEREDOC.color(:red))
             GitLab version mismatch:
               Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
               Please switch to the following version and try again:
               version: #{settings[:gitlab_version]}
           HEREDOC
-          $progress.puts
-          $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
+          progress.puts
+          progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
           exit 1
         end
       end

lib/backup/pages.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Pages < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('pages', Gitlab.config.pages.path)
     end
   end

lib/backup/registry.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Registry < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('registry', Settings.registry.path)
     end
   end

lib/backup/repository.rb

@@ -6,6 +6,12 @@ module Backup
     include Backup::Helper
     # rubocop:disable Metrics/AbcSize
 
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+    end
+
     def dump
       prepare
 
@@ -217,10 +223,6 @@ module Backup
       Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
     end
 
-    def progress
-      $progress
-    end
-
     def display_repo_path(project)
       project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
     end

lib/backup/uploads.rb

@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Uploads < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('uploads', Rails.root.join('public/uploads'))
     end
   end

lib/tasks/gitlab/backup.rake

@@ -6,7 +6,6 @@ namespace :gitlab do
     desc "GitLab | Create a backup of the GitLab system"
     task create: :gitlab_environment do
       warn_user_is_not_gitlab
-      configure_cron_mode
 
       Rake::Task["gitlab:backup:db:create"].invoke
       Rake::Task["gitlab:backup:repo:create"].invoke
@@ -17,7 +16,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:lfs:create"].invoke
       Rake::Task["gitlab:backup:registry:create"].invoke
 
-      backup = Backup::Manager.new
+      backup = Backup::Manager.new(progress)
       backup.pack
       backup.cleanup
       backup.remove_old
@@ -27,9 +26,8 @@ namespace :gitlab do
     desc 'GitLab | Restore a previously created backup'
     task restore: :gitlab_environment do
       warn_user_is_not_gitlab
-      configure_cron_mode
 
-      backup = Backup::Manager.new
+      backup = Backup::Manager.new(progress)
       backup.unpack
 
       unless backup.skipped?('db')
@@ -49,9 +47,9 @@ namespace :gitlab do
 
           # Drop all tables Load the schema to ensure we don't have any newer tables
           # hanging out from a failed upgrade
-          $progress.puts 'Cleaning the database ... '.color(:blue)
+          progress.puts 'Cleaning the database ... '.color(:blue)
           Rake::Task['gitlab:db:drop_tables'].invoke
-          $progress.puts 'done'.color(:green)
+          progress.puts 'done'.color(:green)
           Rake::Task['gitlab:backup:db:restore'].invoke
         rescue Gitlab::TaskAbortedByUserError
           puts "Quitting...".color(:red)
@@ -74,173 +72,173 @@ namespace :gitlab do
 
     namespace :repo do
       task create: :gitlab_environment do
-        $progress.puts "Dumping repositories ...".color(:blue)
+        progress.puts "Dumping repositories ...".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Repository.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Repository.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring repositories ...".color(:blue)
-        Backup::Repository.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring repositories ...".color(:blue)
+        Backup::Repository.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :db do
       task create: :gitlab_environment do
-        $progress.puts "Dumping database ... ".color(:blue)
+        progress.puts "Dumping database ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("db")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Database.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Database.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring database ... ".color(:blue)
-        Backup::Database.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring database ... ".color(:blue)
+        Backup::Database.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :builds do
       task create: :gitlab_environment do
-        $progress.puts "Dumping builds ... ".color(:blue)
+        progress.puts "Dumping builds ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("builds")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Builds.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Builds.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring builds ... ".color(:blue)
-        Backup::Builds.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring builds ... ".color(:blue)
+        Backup::Builds.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :uploads do
       task create: :gitlab_environment do
-        $progress.puts "Dumping uploads ... ".color(:blue)
+        progress.puts "Dumping uploads ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Uploads.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Uploads.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring uploads ... ".color(:blue)
-        Backup::Uploads.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring uploads ... ".color(:blue)
+        Backup::Uploads.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :artifacts do
       task create: :gitlab_environment do
-        $progress.puts "Dumping artifacts ... ".color(:blue)
+        progress.puts "Dumping artifacts ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Artifacts.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Artifacts.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring artifacts ... ".color(:blue)
-        Backup::Artifacts.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring artifacts ... ".color(:blue)
+        Backup::Artifacts.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :pages do
       task create: :gitlab_environment do
-        $progress.puts "Dumping pages ... ".color(:blue)
+        progress.puts "Dumping pages ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("pages")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Pages.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Pages.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring pages ... ".color(:blue)
-        Backup::Pages.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring pages ... ".color(:blue)
+        Backup::Pages.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :lfs do
       task create: :gitlab_environment do
-        $progress.puts "Dumping lfs objects ... ".color(:blue)
+        progress.puts "Dumping lfs objects ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Lfs.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Lfs.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring lfs objects ... ".color(:blue)
-        Backup::Lfs.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring lfs objects ... ".color(:blue)
+        Backup::Lfs.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :registry do
       task create: :gitlab_environment do
-        $progress.puts "Dumping container registry images ... ".color(:blue)
+        progress.puts "Dumping container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
           if ENV["SKIP"] && ENV["SKIP"].include?("registry")
-            $progress.puts "[SKIPPED]".color(:cyan)
+            progress.puts "[SKIPPED]".color(:cyan)
           else
-            Backup::Registry.new.dump
-            $progress.puts "done".color(:green)
+            Backup::Registry.new(progress).dump
+            progress.puts "done".color(:green)
           end
         else
-          $progress.puts "[DISABLED]".color(:cyan)
+          progress.puts "[DISABLED]".color(:cyan)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring container registry images ... ".color(:blue)
+        progress.puts "Restoring container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
-          Backup::Registry.new.restore
-          $progress.puts "done".color(:green)
+          Backup::Registry.new(progress).restore
+          progress.puts "done".color(:green)
         else
-          $progress.puts "[DISABLED]".color(:cyan)
+          progress.puts "[DISABLED]".color(:cyan)
         end
       end
     end
 
-    def configure_cron_mode
+    def progress
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
         # StringIO.
         require 'stringio'
-        $progress = StringIO.new
+        StringIO.new
       else
-        $progress = $stdout
+        $stdout
       end
     end
   end # namespace end: backup

spec/lib/backup/manager_spec.rb

@@ -5,6 +5,8 @@ describe Backup::Manager do
 
   let(:progress) { StringIO.new }
 
+  subject { described_class.new(progress) }
+
   before do
     allow(progress).to receive(:puts)
     allow(progress).to receive(:print)

spec/lib/backup/repository_spec.rb

@@ -3,6 +3,7 @@ require 'spec_helper'
 describe Backup::Repository do
   let(:progress) { StringIO.new }
   let!(:project) { create(:project, :wiki_repo) }
+  subject { described_class.new(progress) }
 
   before do
     allow(progress).to receive(:puts)
@@ -24,14 +25,12 @@ describe Backup::Repository do
       end
 
       it 'does not raise error' do
-        expect { described_class.new.dump }.not_to raise_error
+        expect { subject.dump }.not_to raise_error
       end
     end
   end
 
   describe '#restore' do
-    subject { described_class.new }
-
     let(:timestamp) { Time.utc(2017, 3, 22) }
     let(:temp_dirs) do
       Gitlab.config.repositories.storages.map do |name, storage|
@@ -102,20 +101,20 @@ describe Backup::Repository do
       it 'invalidates the emptiness cache' do
         expect(wiki.repository).to receive(:expire_emptiness_caches).once
 
-        described_class.new.send(:empty_repo?, wiki)
+        subject.send(:empty_repo?, wiki)
       end
 
       context 'wiki repo has content' do
         let!(:wiki_page) { create(:wiki_page, wiki: wiki) }
 
         it 'returns true, regardless of bad cache value' do
-          expect(described_class.new.send(:empty_repo?, wiki)).to be(false)
+          expect(subject.send(:empty_repo?, wiki)).to be(false)
         end
       end
 
       context 'wiki repo does not have content' do
         it 'returns true, regardless of bad cache value' do
-          expect(described_class.new.send(:empty_repo?, wiki)).to be_truthy
+          expect(subject.send(:empty_repo?, wiki)).to be_truthy
         end
       end
     end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -125,6 +125,16 @@ describe 'gitlab:app namespace rake task' do
           expect(Dir.entries(File.join(project.repository.path, 'custom_hooks'))).to include("dummy.txt")
         end
       end
+
+      context 'specific backup tasks' do
+        let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) }
+
+        it 'prints a progress message to stdout' do
+          task_list.each do |task|
+            expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout
+          end
+        end
+      end
     end
 
     context 'tar creation' do