Merge branch 'bvl-export-import-lfs' into 'master'

Export and import LFS objects

Closes #40643

See merge request gitlab-org/gitlab-ce!18115

app/controllers/projects/lfs_api_controller.rb

@@ -41,7 +41,7 @@ class Projects::LfsApiController < Projects::GitHttpClientController
 
   def existing_oids
     @existing_oids ||= begin
-      storage_project.lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
+      project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
     end
   end
 

app/models/project.rb

@@ -1066,6 +1066,16 @@ class Project < ActiveRecord::Base
     end
   end
 
+  # This will return all `lfs_objects` that are accessible to the project.
+  # So this might be `self.lfs_objects` if the project is not part of a fork
+  # network, or it is the base of the fork network.
+  #
+  # TODO: refactor this to get the correct lfs objects when implementing
+  #       https://gitlab.com/gitlab-org/gitlab-ce/issues/39769
+  def all_lfs_objects
+    lfs_storage_project.lfs_objects
+  end
+
   def personal?
     !group
   end

app/services/projects/import_export/export_service.rb

@@ -28,7 +28,7 @@ module Projects
       end
 
       def save_services
-        [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
+        [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver].all?(&:save)
       end
 
       def version_saver
@@ -55,6 +55,10 @@ module Projects
         Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
       end
 
+      def lfs_saver
+        Gitlab::ImportExport::LfsSaver.new(project: project, shared: @shared)
+      end
+
       def cleanup_and_notify_error
         Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}")
 

app/views/projects/_export.html.haml

@@ -21,11 +21,11 @@
           %li Project uploads
           %li Project configuration including web hooks and services
           %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
+          %li LFS objects
         %p
           The following items will NOT be exported:
         %ul
           %li Job traces and artifacts
-          %li LFS objects
           %li Container registry images
           %li CI variables
           %li Any encrypted tokens

changelogs/unreleased/bvl-export-import-lfs.yml

@@ -0,0 +1,5 @@
+---
+title: Support LFS objects when importing/exporting GitLab project archives
+merge_request: 18115
+author:
+type: added

doc/user/project/settings/import_export.md

@@ -57,11 +57,11 @@ The following items will be exported:
 - Project configuration including web hooks and services
 - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets,
   and other project entities
+- LFS objects
 
 The following items will NOT be exported:
 
 - Build traces and artifacts
-- LFS objects
 - Container registry images
 - CI variables
 - Any encrypted tokens

lib/gitlab/checks/lfs_integrity.rb

@@ -15,8 +15,7 @@ module Gitlab
 
         return false unless new_lfs_pointers.present?
 
-        existing_count = @project.lfs_storage_project
-                                 .lfs_objects
+        existing_count = @project.all_lfs_objects
                                  .where(oid: new_lfs_pointers.map(&:lfs_oid))
                                  .count
 

lib/gitlab/import_export/importer.rb

@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def execute
-        if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && restorers.all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
@@ -24,6 +24,11 @@ module Gitlab
 
       private
 
+      def restorers
+        [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
+         uploads_restorer, lfs_restorer]
+      end
+
       def import_file
         Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
                                                   shared: @shared)
@@ -60,6 +65,10 @@ module Gitlab
         Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
       end
 
+      def lfs_restorer
+        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
       def path_with_namespace
         File.join(@project.namespace.full_path, @project.path)
       end

lib/gitlab/import_export/lfs_restorer.rb

@@ -0,0 +1,43 @@
+module Gitlab
+  module ImportExport
+    class LfsRestorer
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def restore
+        return true if lfs_file_paths.empty?
+
+        lfs_file_paths.each do |file_path|
+          link_or_create_lfs_object!(file_path)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def link_or_create_lfs_object!(path)
+        size = File.size(path)
+        oid = LfsObject.calculate_oid(path)
+
+        lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
+        lfs_object.file = File.open(path) unless lfs_object.file&.exists?
+
+        @project.all_lfs_objects << lfs_object
+      end
+
+      def lfs_file_paths
+        @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*")
+      end
+
+      def lfs_storage_path
+        File.join(@shared.export_path, 'lfs-objects')
+      end
+    end
+  end
+end

lib/gitlab/import_export/lfs_saver.rb

@@ -0,0 +1,55 @@
+module Gitlab
+  module ImportExport
+    class LfsSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        @project.all_lfs_objects.each do |lfs_object|
+          save_lfs_object(lfs_object)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+
+        false
+      end
+
+      private
+
+      def save_lfs_object(lfs_object)
+        if lfs_object.local_store?
+          copy_file_for_lfs_object(lfs_object)
+        else
+          download_file_for_lfs_object(lfs_object)
+        end
+      end
+
+      def download_file_for_lfs_object(lfs_object)
+        destination = destination_path_for_object(lfs_object)
+        mkdir_p(File.dirname(destination))
+
+        File.open(destination, 'w') do |file|
+          IO.copy_stream(URI.parse(lfs_object.file.url).open, file)
+        end
+      end
+
+      def copy_file_for_lfs_object(lfs_object)
+        copy_files(lfs_object.file.path, destination_path_for_object(lfs_object))
+      end
+
+      def destination_path_for_object(lfs_object)
+        File.join(lfs_export_path, lfs_object.oid)
+      end
+
+      def lfs_export_path
+        File.join(@shared.export_path, 'lfs-objects')
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/importer_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::Importer do
+  let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
+  let(:shared) { project.import_export_shared }
+  let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
+
+  subject(:importer) { described_class.new(project) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
+    FileUtils.mkdir_p(shared.export_path)
+    FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
+  end
+
+  after do
+    FileUtils.rm_rf(test_path)
+  end
+
+  describe '#execute' do
+    it 'succeeds' do
+      importer.execute
+
+      expect(shared.errors).to be_empty
+    end
+
+    it 'extracts the archive'  do
+      expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
+
+      importer.execute
+    end
+
+    it 'checks the version' do
+      expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
+
+      importer.execute
+    end
+
+    context 'all restores are executed' do
+      [
+        Gitlab::ImportExport::AvatarRestorer,
+        Gitlab::ImportExport::RepoRestorer,
+        Gitlab::ImportExport::WikiRestorer,
+        Gitlab::ImportExport::UploadsRestorer,
+        Gitlab::ImportExport::LfsRestorer
+      ].each do |restorer|
+        it "calls the #{restorer}" do
+          fake_restorer = double(restorer.to_s)
+
+          expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
+          expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
+
+          importer.execute
+        end
+      end
+
+      it 'restores the ProjectTree' do
+        expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
+
+        importer.execute
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/lfs_restorer_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::LfsRestorer do
+  include UploadHelpers
+
+  let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" }
+  let(:project) { create(:project) }
+  let(:shared) { project.import_export_shared }
+  subject(:restorer) { described_class.new(project: project, shared: shared) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    FileUtils.mkdir_p(shared.export_path)
+  end
+
+  after do
+    FileUtils.rm_rf(shared.export_path)
+  end
+
+  describe '#restore' do
+    context 'when the archive contains lfs files' do
+      let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') }
+
+      def create_lfs_object_with_content(content)
+        dummy_lfs_file = Tempfile.new('existing')
+        File.write(dummy_lfs_file.path, content)
+        size = dummy_lfs_file.size
+        oid = LfsObject.calculate_oid(dummy_lfs_file.path)
+        LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file)
+      end
+
+      before do
+        FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path))
+        File.write(dummy_lfs_file_path, 'not very large')
+        allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path])
+      end
+
+      it 'creates an lfs object for the project' do
+        expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1)
+      end
+
+      it 'assigns the file correctly' do
+        restorer.restore
+
+        expect(project.lfs_objects.first.file.read).to eq('not very large')
+      end
+
+      it 'links an existing LFS object if it existed' do
+        lfs_object = create_lfs_object_with_content('not very large')
+
+        restorer.restore
+
+        expect(project.lfs_objects).to include(lfs_object)
+      end
+
+      it 'succeeds' do
+        expect(restorer.restore).to be_truthy
+        expect(shared.errors).to be_empty
+      end
+
+      it 'stores the upload' do
+        expect_any_instance_of(LfsObjectUploader).to receive(:store!)
+
+        restorer.restore
+      end
+    end
+
+    context 'without any LFS-objects' do
+      it 'succeeds' do
+        expect(restorer.restore).to be_truthy
+        expect(shared.errors).to be_empty
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/lfs_saver_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::LfsSaver do
+  let(:shared) { project.import_export_shared }
+  let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
+  let(:project) { create(:project) }
+
+  subject(:saver) { described_class.new(project: project, shared: shared) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    FileUtils.mkdir_p(shared.export_path)
+  end
+
+  after do
+    FileUtils.rm_rf(shared.export_path)
+  end
+
+  describe '#save' do
+    context 'when the project has LFS objects locally stored' do
+      let(:lfs_object) { create(:lfs_object, :with_file) }
+
+      before do
+        project.lfs_objects << lfs_object
+      end
+
+      it 'does not cause errors' do
+        saver.save
+
+        expect(shared.errors).to be_empty
+      end
+
+      it 'copies the file in the correct location when there is an lfs object' do
+        saver.save
+
+        expect(File).to exist("#{shared.export_path}/lfs-objects/#{lfs_object.oid}")
+      end
+    end
+
+    context 'when the LFS objects are stored in object storage' do
+      let(:lfs_object) { create(:lfs_object, :object_storage) }
+
+      before do
+        allow(LfsObjectUploader).to receive(:object_store_enabled?).and_return(true)
+        allow(lfs_object.file).to receive(:url).and_return('http://my-object-storage.local')
+        project.lfs_objects << lfs_object
+      end
+
+      it 'downloads the file to include in an archive' do
+        fake_uri = double
+        exported_file_path = "#{shared.export_path}/lfs-objects/#{lfs_object.oid}"
+
+        expect(fake_uri).to receive(:open).and_return(StringIO.new('LFS file content'))
+        expect(URI).to receive(:parse).with('http://my-object-storage.local').and_return(fake_uri)
+
+        saver.save
+
+        expect(File.read(exported_file_path)).to eq('LFS file content')
+      end
+    end
+  end
+end

spec/models/project_spec.rb

@@ -2022,6 +2022,22 @@ describe Project do
         expect(forked_project.lfs_storage_project).to eq forked_project
       end
     end
+
+    describe '#all_lfs_objects' do
+      let(:lfs_object) { create(:lfs_object) }
+
+      before do
+        project.lfs_objects << lfs_object
+      end
+
+      it 'returns the lfs object for a project' do
+        expect(project.all_lfs_objects).to contain_exactly(lfs_object)
+      end
+
+      it 'returns the lfs object for a fork' do
+        expect(forked_project.all_lfs_objects).to contain_exactly(lfs_object)
+      end
+    end
   end
 
   describe '#pushes_since_gc' do

spec/services/projects/import_export/export_service_spec.rb

@@ -8,6 +8,49 @@ describe Projects::ImportExport::ExportService do
     let(:service) { described_class.new(project, user) }
     let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new }
 
+    it 'saves the version' do
+      expect(Gitlab::ImportExport::VersionSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the avatar' do
+      expect(Gitlab::ImportExport::AvatarSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the models' do
+      expect(Gitlab::ImportExport::ProjectTreeSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the uploads' do
+      expect(Gitlab::ImportExport::UploadsSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the repo' do
+      # once for the normal repo, once for the wiki
+      expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original
+
+      service.execute
+    end
+
+    it 'saves the lfs objects' do
+      expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the wiki repo' do
+      expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
     context 'when all saver services succeed' do
       before do
         allow(service).to receive(:save_services).and_return(true)