From 060e59f05436f29640f6890c69426be1cd79e5cd Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 11:06:12 +0100 Subject: [PATCH 1/3] Lfs on by default. --- config/gitlab.yml.example | 2 +- config/initializers/1_settings.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 8fdb2603ce8..1788d4c2c7c 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -126,7 +126,7 @@ production: &base ## Git LFS lfs: - enabled: false + enabled: true # The location where LFS objects are stored (default: shared/lfs-objects). # storage_path: shared/lfs-objects diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 6b7990c0ab0..b498fb1e1da 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -203,7 +203,7 @@ Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mail # Git LFS # Settings['lfs'] ||= Settingslogic.new({}) -Settings.lfs['enabled'] = false if Settings.lfs['enabled'].nil? +Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root) # From 3cc2b48a82aae8b535ddf11e9057a29f2242524c Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 11:39:26 +0100 Subject: [PATCH 2/3] Backup LFS objects same as any upload. --- doc/raketasks/backup_restore.md | 2 +- lib/backup/lfs.rb | 13 +++++++++++++ lib/backup/manager.rb | 2 +- lib/tasks/gitlab/backup.rake | 21 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 lib/backup/lfs.rb diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 1a5442cdac7..4e645b21a85 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -29,7 +29,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ``` Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, -uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts). +uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts), lfs (LFS objects). Use a comma to specify several options at the same time. ``` diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb new file mode 100644 index 00000000000..4153467fbee --- /dev/null +++ b/lib/backup/lfs.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Lfs < Files + def initialize + super('lfs', Settings.lfs.storage_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index e7eda7c6f45..099062eeb8b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -154,7 +154,7 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3c46bcea40e..cb4abe13799 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:lfs:create"].invoke backup = Backup::Manager.new backup.pack @@ -34,6 +35,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") + Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -134,6 +136,25 @@ namespace :gitlab do end end + namespace :lfs do + task create: :environment do + $progress.puts "Dumping lfs objects ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("lfs") + $progress.puts "[SKIPPED]".cyan + else + Backup::Lfs.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring lfs objects ... ".blue + Backup::Lfs.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a From 2219743d5c6bffd80eaab55db76c0f7d19a8ae61 Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 13:21:57 +0100 Subject: [PATCH 3/3] Add lfs to backup specs. --- spec/tasks/gitlab/backup_rake_spec.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index fb5e74af648..63bed2414df 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do end def reenable_backup_sub_tasks - %w{db repo uploads builds artifacts}.each do |subtask| + %w{db repo uploads builds artifacts lfs}.each do |subtask| Rake::Task["gitlab:backup:#{subtask}:create"].reenable end end @@ -49,7 +49,7 @@ describe 'gitlab:app namespace rake task' do to raise_error(SystemExit) end - it 'should invoke restoration on mach' do + it 'should invoke restoration on match' do allow(YAML).to receive(:load_file). and_return({ gitlab_version: gitlab_version }) expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke) @@ -57,6 +57,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke) + expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end @@ -114,7 +115,7 @@ describe 'gitlab:app namespace rake task' do it 'should set correct permissions on the tar contents' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz} ) expect(exit_status).to eq(0) expect(tar_contents).to match('db/') @@ -122,12 +123,13 @@ describe 'gitlab:app namespace rake task' do expect(tar_contents).to match('repositories/') expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('artifacts.tar.gz') + expect(tar_contents).to match('lfs.tar.gz') expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/) end it 'should delete temp directories' do temp_dirs = Dir.glob( - File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}') + File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs}') ) expect(temp_dirs).to be_empty @@ -163,13 +165,14 @@ describe 'gitlab:app namespace rake task' do it "does not contain skipped item" do tar_contents, _exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz} ) expect(tar_contents).to match('db/') expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('artifacts.tar.gz') + expect(tar_contents).to match('lfs.tar.gz') expect(tar_contents).not_to match('repositories/') end @@ -183,6 +186,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task["gitlab:backup:uploads:restore"]).not_to receive :invoke expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke + expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive :invoke expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end