diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 47ceff18af6..fd569e1c9ae 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -5,10 +5,7 @@ module Projects def execute(options = {}) @shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work')) # TODO handle errors - save_project_tree - bundle_repo - bundle_wiki_repo - save_all + save_all if [save_project_tree, bundle_repo, bundle_wiki_repo].all? end private diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 539eae13f33..aa69f7c44a5 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -6,7 +6,7 @@ module Gitlab File.join(storage_path, relative_path) end - def project_atts + def project_attributes %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived) end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 5ff72f5ff8d..4a1018072c6 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,36 +1,31 @@ module Gitlab module ImportExport module CommandLineUtil - def tar_cf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'cf') + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') end def untar_zxf(archive:, dir:) untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def untar_xf(archive:, dir:) - untar_with_options(archive: archive, dir: dir, options: 'xf') - end - - def tar_czf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'czf') - end - def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) - cmd = %W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) end def tar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir} .) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? + execute(%W(tar -#{options} #{archive} -C #{dir} .)) end def untar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir}) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) _output, status = Gitlab::Popen.popen(cmd) status.zero? end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 92f492e9013..ca433e72d5f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,22 +1,23 @@ -# Class relationships to be included in the project import/export -:project_tree: - - :issues: +# Model relationships to be included in the project import/export +project_tree: + - issues: - :notes - :labels - :milestones - :snippets - :releases - :events - - :project_members: + - project_members: - :user - - :merge_requests: + - merge_requests: - :merge_request_diff - :notes - - :ci_commits: + - ci_commits: - :statuses -:attributes_only: - :project: +# Only include the following attributes for the models specified. +included_attributes: + project: - :name - :path - :description @@ -27,11 +28,12 @@ - :snippets_enabled - :visibility_level - :archived - :user: + user: - :id - :email - :username -:attributes_except: - :snippets: +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: - :expired_at \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 14049cb1bd2..77db6cabe38 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -4,7 +4,7 @@ module Gitlab extend self def project_tree - { only: atts_only[:project], include: build_hash(tree) } + { only: included_attributes[:project], include: build_hash(tree) } end def tree @@ -14,24 +14,24 @@ module Gitlab private def config - @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml') + @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access end - def atts_only - config[:attributes_only] + def included_attributes + config[:included_attributes] || {} end - def atts_except - config[:attributes_except] + def excluded_attributes + config[:excluded_attributes] || {} end def build_hash(array) - array.map do |el| - if el.is_a?(Hash) - process_include(el) + array.map do |model_object| + if model_object.is_a?(Hash) + process_include(model_object) else - only_except_hash = check_only_and_except(el) - only_except_hash.empty? ? el : { el => only_except_hash } + only_except_hash = check_only_and_except(model_object) + only_except_hash.empty? ? model_object : { model_object => only_except_hash } end end end @@ -82,12 +82,12 @@ module Gitlab def check_only(value) key = key_from_hash(value) - atts_only[key].nil? ? {} : { only: atts_only[key] } + included_attributes[key].nil? ? {} : { only: included_attributes[key] } end def check_except(value) key = key_from_hash(value) - atts_except[key].nil? ? {} : { except: atts_except[key] } + excluded_attributes[key].nil? ? {} : { except: excluded_attributes[key] } end def key_from_hash(value) diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 394411a56c2..29d715c16d3 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,19 +3,13 @@ module Gitlab class ProjectTreeSaver attr_reader :full_path - def initialize(project: , shared: ) + def initialize(project:, shared:) @project = project @export_path = shared.export_path + @full_path = File.join(@export_path, project_filename) end def save - @full_path = File.join(@export_path, project_filename) - save_to_disk - end - - private - - def save_to_disk FileUtils.mkdir_p(@export_path) File.write(full_path, project_json_tree) true @@ -24,6 +18,8 @@ module Gitlab false end + private + # TODO remove magic keyword and move it to a shared config def project_filename "project.json" diff --git a/spec/lib/gitlab/import_export/import_export_reader_spec.rb b/spec/lib/gitlab/import_export/import_export_reader_spec.rb index e71c7b60b80..0b2da73a225 100644 --- a/spec/lib/gitlab/import_export/import_export_reader_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_reader_spec.rb @@ -1,4 +1,4 @@ -require 'rspec' +require 'spec_helper' describe Gitlab::ImportExport::ImportExportReader do @@ -17,7 +17,7 @@ describe Gitlab::ImportExport::ImportExportReader do end it 'should generate hash from project tree config' do - allow(described_class).to receive(:config).and_return(YAML.load_file(test_config)) + allow(described_class).to receive(:config).and_return(YAML.load_file(test_config).deep_symbolize_keys) expect(described_class.project_tree).to eq(project_tree_hash) end diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb index 5c1b9af2c13..67e6267b741 100644 --- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb @@ -1,42 +1,20 @@ require 'spec_helper' describe Gitlab::ImportExport::ProjectTreeSaver, services: true do - describe :save do + describe 'saves the project tree into a json object' do - # TODO refactor this into a setup method - - let(:user) { create(:user) } - let(:issue) { create(:issue, assignee: user) } - let(:merge_request) { create(:merge_request) } - let(:label) { create(:label) } - let(:snippet) { create(:project_snippet) } - let(:commit_status) { create(:commit_status) } - let(:release) { create(:release) } - let!(:project) do - create(:project, - :public, - name: 'searchable_project', - issues: [issue], - merge_requests: [merge_request], - labels: [label], - snippets: [snippet], - releases: [release] - ) - end - let!(:ci_commit) { create(:ci_commit, project: project, sha: merge_request.last_commit.id, ref: merge_request.source_branch, statuses: [commit_status]) } - let!(:milestone) { create(:milestone, title: "Milestone v1.2", project: project) } - let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) } - let(:project_tree_saver) { Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: shared) } - let!(:issue_note) { create(:note, note: ":+1: issue", noteable: issue) } - let!(:merge_request_note) { create(:note, note: ":+1: merge_request", noteable: merge_request) } + let(:project_tree_saver) { described_class.new(project: project, shared: shared) } + let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" } + let(:user) { create(:user) } + let(:project) { setup_project } - before(:each) do + before do project.team << [user, :master] allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) end - after(:each) do + after do FileUtils.rm_rf(export_path) end @@ -109,6 +87,30 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do end end + def setup_project + issue = create(:issue, assignee: user) + merge_request = create(:merge_request) + label = create(:label) + snippet = create(:project_snippet) + commit_status = create(:commit_status) + release = create(:release) + + project = create(:project, + :public, + issues: [issue], + merge_requests: [merge_request], + labels: [label], + snippets: [snippet], + releases: [release] + ) + + create(:ci_commit, project: project, sha: merge_request.last_commit.id, ref: merge_request.source_branch, statuses: [commit_status]) + create(:milestone, project: project) + create(:note, noteable: issue) + create(:note, noteable: merge_request) + project + end + def project_json(filename) JSON.parse(IO.read(filename)) end diff --git a/spec/lib/gitlab/import_export/repo_bundler_spec.rb b/spec/lib/gitlab/import_export/repo_bundler_spec.rb index 23447d878a8..fa926bab823 100644 --- a/spec/lib/gitlab/import_export/repo_bundler_spec.rb +++ b/spec/lib/gitlab/import_export/repo_bundler_spec.rb @@ -1,20 +1,20 @@ require 'spec_helper' describe Gitlab::ImportExport::RepoBundler, services: true do - describe :bundle do + describe 'bundle a project Git repo' do let(:user) { create(:user) } let!(:project) { create(:project, :public, name: 'searchable_project') } let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) } - let(:bundler) { Gitlab::ImportExport::RepoBundler.new(project: project, shared: shared) } + let(:bundler) { described_class.new(project: project, shared: shared) } - before(:each) do + before do project.team << [user, :master] allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) end - after(:each) do + after do FileUtils.rm_rf(export_path) end diff --git a/spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb b/spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb index 6cc95edc217..60a0145c1a0 100644 --- a/spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb +++ b/spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb @@ -1,23 +1,23 @@ require 'spec_helper' describe Gitlab::ImportExport::WikiRepoBundler, services: true do - describe :bundle do + describe 'bundle a wiki Git repo' do let(:user) { create(:user) } let!(:project) { create(:project, :public, name: 'searchable_project') } let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) } - let(:wiki_bundler) { Gitlab::ImportExport::WikiRepoBundler.new(project: project, shared: shared) } + let(:wiki_bundler) { described_class.new(project: project, shared: shared) } let!(:project_wiki) { ProjectWiki.new(project, user) } - before(:each) do + before do project.team << [user, :master] allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) project_wiki.wiki project_wiki.create_page("index", "test content") end - after(:each) do + after do FileUtils.rm_rf(export_path) end diff --git a/spec/support/import_export/import_export.yml b/spec/support/import_export/import_export.yml index e6985d6413c..3ceec506401 100644 --- a/spec/support/import_export/import_export.yml +++ b/spec/support/import_export/import_export.yml @@ -1,20 +1,20 @@ # Class relationships to be included in the project import/export -:project_tree: +project_tree: - :issues - :labels - - :merge_requests: + - merge_requests: - :merge_request_diff - :merge_request_test - - :commit_statuses: + - commit_statuses: - :commit -:attributes_only: - :project: +included_attributes: + project: - :name - :path - :merge_requests: + merge_requests: - :id -:attributes_except: - :merge_requests: +excluded_attributes: + merge_requests: - :iid \ No newline at end of file