Merge branch 'lfs_default_and_backup' into 'master'

Lfs default and backup

Part of gitlab-org/gitlab-ce#2955

See merge request !1823

config/gitlab.yml.example

@@ -126,7 +126,7 @@ production: &base
 
   ## Git LFS
   lfs:
-    enabled: false
+    enabled: true
     # The location where LFS objects are stored (default: shared/lfs-objects).
     # storage_path: shared/lfs-objects
 

config/initializers/1_settings.rb

@@ -203,7 +203,7 @@ Settings.incoming_email['mailbox']    = "inbox" if Settings.incoming_email['mail
 # Git LFS
 #
 Settings['lfs'] ||= Settingslogic.new({})
-Settings.lfs['enabled']      = false if Settings.lfs['enabled'].nil?
+Settings.lfs['enabled']      = true if Settings.lfs['enabled'].nil?
 Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root)
 
 #

doc/raketasks/backup_restore.md

@@ -29,7 +29,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
 ```
 
 Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
-uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
+uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts), lfs (LFS objects).
 Use a comma to specify several options at the same time.
 
 ```

lib/backup/lfs.rb

@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Lfs < Files
+    def initialize
+      super('lfs', Settings.lfs.storage_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end

lib/backup/manager.rb

@@ -154,7 +154,7 @@ module Backup
     end
 
     def archives_to_backup
-      %w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
+      %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
     end
 
     def folders_to_backup

lib/tasks/gitlab/backup.rake

@@ -13,6 +13,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
       Rake::Task["gitlab:backup:artifacts:create"].invoke
+      Rake::Task["gitlab:backup:lfs:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -34,6 +35,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
       Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
+      Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -134,6 +136,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :lfs do
+      task create: :environment do
+        $progress.puts "Dumping lfs objects ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Lfs.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring lfs objects ... ".blue
+        Backup::Lfs.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a

spec/tasks/gitlab/backup_rake_spec.rb

@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
   end
 
   def reenable_backup_sub_tasks
-    %w{db repo uploads builds artifacts}.each do |subtask|
+    %w{db repo uploads builds artifacts lfs}.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -49,7 +49,7 @@ describe 'gitlab:app namespace rake task' do
           to raise_error(SystemExit)
       end
 
-      it 'should invoke restoration on mach' do
+      it 'should invoke restoration on match' do
         allow(YAML).to receive(:load_file).
           and_return({ gitlab_version: gitlab_version })
         expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke)
@@ -57,6 +57,7 @@ describe 'gitlab:app namespace rake task' do
         expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
+        expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
         expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
       end
@@ -114,7 +115,7 @@ describe 'gitlab:app namespace rake task' do
 
     it 'should set correct permissions on the tar contents' do
       tar_contents, exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz}
       )
       expect(exit_status).to eq(0)
       expect(tar_contents).to match('db/')
@@ -122,12 +123,13 @@ describe 'gitlab:app namespace rake task' do
       expect(tar_contents).to match('repositories/')
       expect(tar_contents).to match('builds.tar.gz')
       expect(tar_contents).to match('artifacts.tar.gz')
+      expect(tar_contents).to match('lfs.tar.gz')
       expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
     end
 
     it 'should delete temp directories' do
       temp_dirs = Dir.glob(
-        File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}')
+        File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs}')
       )
 
       expect(temp_dirs).to be_empty
@@ -163,13 +165,14 @@ describe 'gitlab:app namespace rake task' do
 
     it "does not contain skipped item" do
       tar_contents, _exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz}
       )
 
       expect(tar_contents).to match('db/')
       expect(tar_contents).to match('uploads.tar.gz')
       expect(tar_contents).to match('builds.tar.gz')
       expect(tar_contents).to match('artifacts.tar.gz')
+      expect(tar_contents).to match('lfs.tar.gz')
       expect(tar_contents).not_to match('repositories/')
     end
 
@@ -183,6 +186,7 @@ describe 'gitlab:app namespace rake task' do
       expect(Rake::Task["gitlab:backup:uploads:restore"]).not_to receive :invoke
       expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
+      expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
     end