From 9504a529b758b0352b9c60d67fda8b4ee2a5fec0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 14:36:33 -0200 Subject: [PATCH 01/19] Write project full path to .git/config when creating projects We'd need to keep track of project full path otherwise directory tree created with hashed storage enabled cannot be usefully imported using the import rake task. --- app/models/project.rb | 5 +++++ app/services/projects/create_service.rb | 5 +++++ spec/services/projects/create_service_spec.rb | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index 6ebb083aeb4..eac78de1ac9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1432,6 +1432,11 @@ class Project < ActiveRecord::Base Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) end + def write_repository_config(key, value, prefix: :gitlab) + key = [prefix, key].compact.join('.') + repo.config[key] = value + end + def rename_repo_notify! send_move_instructions(full_path_was) expires_full_path_cache diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index dc7b1f1f5cc..4c7e5370bbe 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -87,6 +87,11 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + @project.write_repository_config(:fullpath, @project.full_path) + unless @project.gitlab_project_import? @project.create_wiki unless skip_wiki? create_services_from_active_templates(@project) diff --git a/spec/services/projects/create_service_spec.rb b/spec/services/projects/create_service_spec.rb index dc89fdebce7..1833078f37c 100644 --- a/spec/services/projects/create_service_spec.rb +++ b/spec/services/projects/create_service_spec.rb @@ -252,6 +252,12 @@ describe Projects::CreateService, '#execute' do end end + it 'writes project full path to .git/config' do + project = create_project(user, opts) + + expect(project.repo.config['gitlab.fullpath']).to eq project.full_path + end + def create_project(user, opts) Projects::CreateService.new(user, opts).execute end From 64fe954dcebaadd6f686f30eb4ff0be5ebcf172d Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 14:53:59 -0200 Subject: [PATCH 02/19] Update project full path in .git/config when renaming a repository --- app/models/project.rb | 5 +++++ spec/models/project_spec.rb | 14 ++++++++++++++ spec/services/projects/update_service_spec.rb | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/models/project.rb b/app/models/project.rb index eac78de1ac9..1182dbda0c0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1420,6 +1420,11 @@ class Project < ActiveRecord::Base end def after_rename_repo + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + write_repository_config(:fullpath, full_path) + path_before_change = previous_changes['path'].first # We need to check if project had been rolled out to move resource to hashed storage or not and decide diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 7338e341359..10634d22b39 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2626,6 +2626,14 @@ describe Project do project.rename_repo end end + + it 'updates project full path in .git/config' do + allow(project_storage).to receive(:rename_repo).and_return(true) + + expect(project).to receive(:write_repository_config).with(:fullpath, project.full_path) + + project.rename_repo + end end describe '#pages_path' do @@ -2781,6 +2789,12 @@ describe Project do end end end + + it 'updates project full path in .git/config' do + expect(project).to receive(:write_repository_config).with(:fullpath, project.full_path) + + project.rename_repo + end end describe '#pages_path' do diff --git a/spec/services/projects/update_service_spec.rb b/spec/services/projects/update_service_spec.rb index d887f70efae..fc6aa713d6f 100644 --- a/spec/services/projects/update_service_spec.rb +++ b/spec/services/projects/update_service_spec.rb @@ -61,7 +61,7 @@ describe Projects::UpdateService do end end - context 'When project visibility is higher than parent group' do + context 'when project visibility is higher than parent group' do let(:group) { create(:group, visibility_level: Gitlab::VisibilityLevel::INTERNAL) } before do From bd90330740e0ea5c0ce0672fd605a463fcdfc898 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 17:18:26 -0200 Subject: [PATCH 03/19] Update project full path in .git/config when transfering a project --- app/services/projects/transfer_service.rb | 10 ++++++++++ spec/services/projects/transfer_service_spec.rb | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index e5cd6fcdfe3..e742df5f696 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -75,6 +75,8 @@ module Projects project.old_path_with_namespace = @old_path project.expires_full_path_cache + write_repository_config(@new_path) + execute_system_hooks end rescue Exception # rubocop:disable Lint/RescueException @@ -98,6 +100,13 @@ module Projects project.save! end + def write_repository_config(full_path) + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + project.write_repository_config(:fullpath, full_path) + end + def refresh_permissions # This ensures we only schedule 1 job for every user that has access to # the namespaces. @@ -110,6 +119,7 @@ module Projects def rollback_side_effects rollback_folder_move update_namespace_and_visibility(@old_namespace) + write_repository_config(@old_path) end def rollback_folder_move diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 2b1337bee7e..7377c748698 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -54,6 +54,12 @@ describe Projects::TransferService do expect(project.disk_path).not_to eq(old_path) expect(project.disk_path).to start_with(group.path) end + + it 'updates project full path in .git/config' do + transfer_project(project, user, group) + + expect(project.repo.config['gitlab.fullpath']).to eq "#{group.full_path}/#{project.path}" + end end context 'when transfer fails' do @@ -86,6 +92,12 @@ describe Projects::TransferService do expect(original_path).to eq current_path end + it 'rolls back project full path in .git/config' do + attempt_project_transfer + + expect(project.repo.config['gitlab.fullpath']).to eq project.full_path + end + it "doesn't send move notifications" do expect_any_instance_of(NotificationService).not_to receive(:project_was_moved) From ca089f59687fb8616bcbd3d5501fbc6006893e8f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 17:42:51 -0200 Subject: [PATCH 04/19] Update project full path in .git/config when renaming namespace --- app/models/concerns/storage/legacy_namespace.rb | 2 ++ app/models/namespace.rb | 7 +++++++ spec/models/namespace_spec.rb | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index b3020484738..22b9ef4e338 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -34,6 +34,8 @@ module Storage # So we basically we mute exceptions in next actions begin send_update_instructions + write_projects_full_path_config + true rescue # Returning false does not rollback after_* transaction but gives diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 0ff169d4531..d983b2f106b 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -268,4 +268,11 @@ class Namespace < ActiveRecord::Base def namespace_previously_created_with_same_path? RedirectRoute.permanent.exists?(path: path) end + + def write_projects_full_path_config + all_projects.each do |project| + project.expires_full_path_cache # we need to clear cache to validate renames correctly + project.write_repository_config(:fullpath, project.full_path) + end + end end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index b7c6286fd83..0a99485ec8e 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -240,6 +240,20 @@ describe Namespace do end end end + + it 'updates project full path in .git/config for each project inside namespace' do + parent = create(:group, name: 'mygroup', path: 'mygroup') + subgroup = create(:group, name: 'mysubgroup', path: 'mysubgroup', parent: parent) + project_in_parent_group = create(:project, :repository, namespace: parent, name: 'foo1') + hashed_project_in_subgroup = create(:project, :repository, :hashed, namespace: subgroup, name: 'foo2') + legacy_project_in_subgroup = create(:project, :repository, namespace: subgroup, name: 'foo3') + + parent.update(path: 'mygroup_new') + + expect(project_in_parent_group.repo.config['gitlab.fullpath']).to eq "mygroup_new/#{project_in_parent_group.path}" + expect(hashed_project_in_subgroup.repo.config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{hashed_project_in_subgroup.path}" + expect(legacy_project_in_subgroup.repo.config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{legacy_project_in_subgroup.path}" + end end describe '#rm_dir', 'callback' do From 2f2233774c3d6416f5571a2e83b367d34bad3f5f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 18:04:48 -0200 Subject: [PATCH 05/19] Write project full path to .git/config when migrating legacy storage --- .../projects/hashed_storage/migrate_repository_service.rb | 7 +++++++ .../hashed_storage/migrate_repository_service_spec.rb | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb index 7212e7524ab..c076ce06278 100644 --- a/app/services/projects/hashed_storage/migrate_repository_service.rb +++ b/app/services/projects/hashed_storage/migrate_repository_service.rb @@ -39,6 +39,13 @@ module Projects yield end + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + if result + project.write_repository_config(:fullpath, project.full_path) + end + result end diff --git a/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb index 3a3e47fd9c0..ded864beb1d 100644 --- a/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb +++ b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Projects::HashedStorage::MigrateRepositoryService do let(:gitlab_shell) { Gitlab::Shell.new } - let(:project) { create(:project, :empty_repo, :wiki_repo) } + let(:project) { create(:project, :repository, :wiki_repo) } let(:service) { described_class.new(project) } let(:legacy_storage) { Storage::LegacyProject.new(project) } let(:hashed_storage) { Storage::HashedProject.new(project) } @@ -33,6 +33,12 @@ describe Projects::HashedStorage::MigrateRepositoryService do service.execute end + + it 'writes project full path to .git/config' do + service.execute + + expect(project.repo.config['gitlab.fullpath']).to eq project.full_path + end end context 'when one move fails' do From d3d617354e0f0da7c8930dd9c089f437603dea20 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 19:54:04 -0200 Subject: [PATCH 06/19] Does not write to .git/config when importing bare repositories --- app/services/projects/create_service.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 4c7e5370bbe..24ae50f8dc4 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -87,12 +87,12 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - @project.write_repository_config(:fullpath, @project.full_path) - unless @project.gitlab_project_import? + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + @project.write_repository_config(:fullpath, @project.full_path) + @project.create_wiki unless skip_wiki? create_services_from_active_templates(@project) From 582678b5f5e1399603610b20149acf1d305309d3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 23:58:54 -0200 Subject: [PATCH 07/19] Import directory tree created with hashed storage using import rake task --- lib/gitlab/bare_repository_import/importer.rb | 9 +- .../bare_repository_import/repository.rb | 36 +++-- .../bare_repository_import/importer_spec.rb | 14 +- .../bare_repository_import/repository_spec.rb | 133 ++++++++++++------ 4 files changed, 136 insertions(+), 56 deletions(-) diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 64e41d42709..1f0fdc6685e 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -14,13 +14,13 @@ module Gitlab repos_to_import.each do |repo_path| bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - if bare_repo.hashed? || bare_repo.wiki? + unless bare_repo.processable? log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) next end - log "Processing #{repo_path}".color(:yellow) + log "Processing #{repo_path} -> #{bare_repo.project_full_path}".color(:yellow) new(user, bare_repo).create_project_if_needed end @@ -62,6 +62,11 @@ module Gitlab if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + project.write_repository_config(:fullpath, project.full_path) + ProjectCacheWorker.perform_async(project.id) else log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fa7891c8dcc..b5907875460 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -6,39 +6,55 @@ module Gitlab def initialize(root_path, repo_path) @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') + full_path = + if hashed? && !wiki? + repository.config.get('gitlab.fullpath') + else + repo_relative_path + end + # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = repo_relative_path.rpartition('/') + @group_path, _, @project_name = full_path.to_s.rpartition('/') end def wiki_exists? File.exist?(wiki_path) end - def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') - end - def wiki_path @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') end - def hashed? - @hashed ||= group_path.start_with?('@hashed') - end - def project_full_path @project_full_path ||= "#{group_path}/#{project_name}" end + def processable? + return false if wiki? + + group_path.present? && project_name.present? + end + private + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def hashed? + @hashed ||= repo_relative_path.include?('@hashed') + end + def repo_relative_path # Remove root path and `.git` at the end repo_path[@root_path.size...-4] end + + def repository + @repository ||= Rugged::Repository.new(repo_path) + end end end end diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 82ac3c424d4..99cc9c4bd41 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -1,6 +1,7 @@ require 'spec_helper' describe Gitlab::BareRepositoryImport::Importer, repository: true do + let(:gitlab_shell) { Gitlab::Shell.new } let!(:admin) { create(:admin) } let!(:base_dir) { Dir.mktmpdir + '/' } let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) } @@ -75,7 +76,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do end it 'creates the Git repo in disk' do - FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + create_bare_repository("#{project_path}.git") importer.create_project_if_needed @@ -130,7 +131,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do end it 'creates the Git repo in disk' do - FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + create_bare_repository("#{project_path}.git") importer.create_project_if_needed @@ -165,8 +166,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do it_behaves_like 'importing a repository' it 'creates the Wiki git repo in disk' do - FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) - FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git")) + create_bare_repository("#{project_path}.git") + create_bare_repository("#{project_path}.wiki.git") expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true, import_type: 'bare_repository')).and_call_original @@ -192,4 +193,9 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do end end end + + def create_bare_repository(project_path) + repo_path = File.join(base_dir, project_path) + Gitlab::Git::Repository.create(repo_path, bare: true) + end end diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index 61b73abcba4..770e26bbf6a 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -1,58 +1,111 @@ require 'spec_helper' describe ::Gitlab::BareRepositoryImport::Repository do - let(:project_repo_path) { described_class.new('/full/path/', '/full/path/to/repo.git') } + context 'legacy storage' do + subject { described_class.new('/full/path/', '/full/path/to/repo.git') } - it 'stores the repo path' do - expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git') - end - - it 'stores the group path' do - expect(project_repo_path.group_path).to eq('to') - end - - it 'stores the project name' do - expect(project_repo_path.project_name).to eq('repo') - end - - it 'stores the wiki path' do - expect(project_repo_path.wiki_path).to eq('/full/path/to/repo.wiki.git') - end - - describe '#wiki?' do - it 'returns true if it is a wiki' do - wiki_path = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git') - - expect(wiki_path.wiki?).to eq(true) + it 'stores the repo path' do + expect(subject.repo_path).to eq('/full/path/to/repo.git') end - it 'returns false if it is not a wiki' do - expect(project_repo_path.wiki?).to eq(false) + it 'stores the group path' do + expect(subject.group_path).to eq('to') + end + + it 'stores the project name' do + expect(subject.project_name).to eq('repo') + end + + it 'stores the wiki path' do + expect(subject.wiki_path).to eq('/full/path/to/repo.wiki.git') + end + + describe '#processable?' do + it 'returns false if it is a wiki' do + subject = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git') + + expect(subject.processable?).to eq(false) + end + + it 'returns true when group and project name are present' do + expect(subject.processable?).to eq(true) + end + end + + describe '#project_full_path' do + it 'returns the project full path' do + expect(subject.repo_path).to eq('/full/path/to/repo.git') + expect(subject.project_full_path).to eq('to/repo') + end + + it 'with no trailing slash in the root path' do + repo_path = described_class.new('/full/path', '/full/path/to/repo.git') + + expect(repo_path.project_full_path).to eq('to/repo') + end end end - describe '#hashed?' do - it 'returns true if it is a hashed folder' do - path = described_class.new('/full/path/', '/full/path/@hashed/my.repo.git') + context 'hashed storage' do + let(:gitlab_shell) { Gitlab::Shell.new } + let(:repository_storage) { 'default' } + let(:root_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } + let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" } + let(:repo_path) { File.join(root_path, "#{hashed_path}.git") } + let(:wiki_path) { File.join(root_path, "#{hashed_path}.wiki.git") } - expect(path.hashed?).to eq(true) + before do + gitlab_shell.add_repository(repository_storage, hashed_path) + repository = Rugged::Repository.new(repo_path) + repository.config['gitlab.fullpath'] = 'to/repo' end - it 'returns false if it is not a hashed folder' do - expect(project_repo_path.hashed?).to eq(false) - end - end - - describe '#project_full_path' do - it 'returns the project full path' do - expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git') - expect(project_repo_path.project_full_path).to eq('to/repo') + after do + gitlab_shell.remove_repository(root_path, hashed_path) end - it 'with no trailing slash in the root path' do - repo_path = described_class.new('/full/path', '/full/path/to/repo.git') + subject { described_class.new(root_path, repo_path) } - expect(repo_path.project_full_path).to eq('to/repo') + it 'stores the repo path' do + expect(subject.repo_path).to eq(repo_path) + end + + it 'stores the wiki path' do + expect(subject.wiki_path).to eq(wiki_path) + end + + it 'reads the group path from .git/config' do + expect(subject.group_path).to eq('to') + end + + it 'reads the project name from .git/config' do + expect(subject.project_name).to eq('repo') + end + + describe '#processable?' do + it 'returns false if it is a wiki' do + subject = described_class.new(root_path, wiki_path) + + expect(subject.processable?).to eq(false) + end + + it 'returns false when group and project name are missing' do + repository = Rugged::Repository.new(repo_path) + repository.config.delete('gitlab.fullpath') + + expect(subject.processable?).to eq(false) + end + + it 'returns true when group and project name are present' do + expect(subject.processable?).to eq(true) + end + end + + describe '#project_full_path' do + it 'returns the project full path' do + expect(subject.project_full_path).to eq('to/repo') + end end end end From c328a7db75b601bed449ad04100e12779f5bdb36 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 20 Dec 2017 14:22:13 -0200 Subject: [PATCH 08/19] Handle exceptions when writing to .git/config --- app/models/project.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index 1182dbda0c0..47ca62aa5bb 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1440,6 +1440,9 @@ class Project < ActiveRecord::Base def write_repository_config(key, value, prefix: :gitlab) key = [prefix, key].compact.join('.') repo.config[key] = value + rescue Gitlab::Git::Repository::NoRepository => e + Rails.logger.error("Error writing key #{key} to .git/config for project #{full_path} (#{id}): #{e.message}.") + nil end def rename_repo_notify! From 603fcb6b9ca0aec1a5f8d75fa16d626dd8058269 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 20 Dec 2017 15:40:59 -0200 Subject: [PATCH 09/19] Add CHANGELOG --- ...da-handle-hashed-storage-repos-using-repo-import-task.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/da-handle-hashed-storage-repos-using-repo-import-task.yml diff --git a/changelogs/unreleased/da-handle-hashed-storage-repos-using-repo-import-task.yml b/changelogs/unreleased/da-handle-hashed-storage-repos-using-repo-import-task.yml new file mode 100644 index 00000000000..74a00d49ab3 --- /dev/null +++ b/changelogs/unreleased/da-handle-hashed-storage-repos-using-repo-import-task.yml @@ -0,0 +1,5 @@ +--- +title: Handle GitLab hashed storage repositories using the repo import task +merge_request: +author: +type: added From 9d575acc5b46be7e0b76ccc763997412cd278ef0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 19 Dec 2017 17:06:38 -0200 Subject: [PATCH 10/19] Fix TestEnv.copy_repo to use disk_path instead of full_path --- spec/models/namespace_spec.rb | 2 +- spec/models/project_spec.rb | 2 +- spec/support/test_env.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 0a99485ec8e..0678cae9b93 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -203,7 +203,7 @@ describe Namespace do context 'with subgroups' do let(:parent) { create(:group, name: 'parent', path: 'parent') } let(:child) { create(:group, name: 'child', path: 'child', parent: parent) } - let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child) } + let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) } let(:uploads_dir) { File.join(CarrierWave.root, FileUploader.base_dir) } let(:pages_dir) { File.join(TestEnv.pages_path) } diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 10634d22b39..62029f385a2 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2676,7 +2676,7 @@ describe Project do end context 'hashed storage' do - let(:project) { create(:project, :repository) } + let(:project) { create(:project, :repository, skip_disk_validation: true) } let(:gitlab_shell) { Gitlab::Shell.new } let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index ffc051a3fff..1d99746b09f 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -215,7 +215,7 @@ module TestEnv end def copy_repo(project, bare_repo:, refs:) - target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.full_path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.disk_path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path From 93eba91df9af083ea80b3b8ab01986efdeec43a0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Dec 2017 13:58:36 -0200 Subject: [PATCH 11/19] Refactoring Project#write_repository_config --- app/models/namespace.rb | 2 +- app/models/project.rb | 15 ++++++------- app/services/projects/create_service.rb | 6 +---- .../migrate_repository_service.rb | 9 ++------ app/services/projects/transfer_service.rb | 5 +---- lib/gitlab/bare_repository_import/importer.rb | 7 ++---- spec/models/project_spec.rb | 22 +++++++++---------- 7 files changed, 25 insertions(+), 41 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index d983b2f106b..efbfc607040 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -272,7 +272,7 @@ class Namespace < ActiveRecord::Base def write_projects_full_path_config all_projects.each do |project| project.expires_full_path_cache # we need to clear cache to validate renames correctly - project.write_repository_config(:fullpath, project.full_path) + project.write_repository_config end end end diff --git a/app/models/project.rb b/app/models/project.rb index 47ca62aa5bb..9c0bbf697e2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1420,10 +1420,7 @@ class Project < ActiveRecord::Base end def after_rename_repo - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - write_repository_config(:fullpath, full_path) + write_repository_config path_before_change = previous_changes['path'].first @@ -1437,11 +1434,13 @@ class Project < ActiveRecord::Base Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) end - def write_repository_config(key, value, prefix: :gitlab) - key = [prefix, key].compact.join('.') - repo.config[key] = value + def write_repository_config(gl_full_path: full_path) + # We'd need to keep track of project full path otherwise directory tree + # created with hashed storage enabled cannot be usefully imported using + # the import rake task. + repo.config['gitlab.fullpath'] = gl_full_path rescue Gitlab::Git::Repository::NoRepository => e - Rails.logger.error("Error writing key #{key} to .git/config for project #{full_path} (#{id}): #{e.message}.") + Rails.logger.error("Error writing to .git/config for project #{full_path} (#{id}): #{e.message}.") nil end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 24ae50f8dc4..01838ec6b5d 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -88,11 +88,7 @@ module Projects log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") unless @project.gitlab_project_import? - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - @project.write_repository_config(:fullpath, @project.full_path) - + @project.write_repository_config @project.create_wiki unless skip_wiki? create_services_from_active_templates(@project) diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb index c076ce06278..b6763c9436f 100644 --- a/app/services/projects/hashed_storage/migrate_repository_service.rb +++ b/app/services/projects/hashed_storage/migrate_repository_service.rb @@ -30,6 +30,8 @@ module Projects unless result rollback_folder_move project.storage_version = nil + else + project.write_repository_config end project.repository_read_only = false @@ -39,13 +41,6 @@ module Projects yield end - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - if result - project.write_repository_config(:fullpath, project.full_path) - end - result end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index e742df5f696..14cf9f82bdb 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -101,10 +101,7 @@ module Projects end def write_repository_config(full_path) - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - project.write_repository_config(:fullpath, full_path) + project.write_repository_config(:gl_fullpath, full_path) end def refresh_permissions diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 1f0fdc6685e..709a901aa77 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -20,7 +20,7 @@ module Gitlab next end - log "Processing #{repo_path} -> #{bare_repo.project_full_path}".color(:yellow) + log "Processing #{repo_path}".color(:yellow) new(user, bare_repo).create_project_if_needed end @@ -62,10 +62,7 @@ module Gitlab if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) - # We'd need to keep track of project full path otherwise directory tree - # created with hashed storage enabled cannot be usefully imported using - # the import rake task. - project.write_repository_config(:fullpath, project.full_path) + project.write_repository_config ProjectCacheWorker.perform_async(project.id) else diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 62029f385a2..1d4b68bdf8d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2630,9 +2630,9 @@ describe Project do it 'updates project full path in .git/config' do allow(project_storage).to receive(:rename_repo).and_return(true) - expect(project).to receive(:write_repository_config).with(:fullpath, project.full_path) - project.rename_repo + + expect(project.repo.config['gitlab.fullpath']).to eq(project.full_path) end end @@ -2678,12 +2678,10 @@ describe Project do context 'hashed storage' do let(:project) { create(:project, :repository, skip_disk_validation: true) } let(:gitlab_shell) { Gitlab::Shell.new } - let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } + let(:hash) { Digest::SHA2.hexdigest(project.id.to_s) } before do stub_application_setting(hashed_storage_enabled: true) - allow(Digest::SHA2).to receive(:hexdigest) { hash } - allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) end describe '#legacy_storage?' do @@ -2706,13 +2704,13 @@ describe Project do describe '#base_dir' do it 'returns base_dir based on hash of project id' do - expect(project.base_dir).to eq('@hashed/6b/86') + expect(project.base_dir).to eq("@hashed/#{hash[0..1]}/#{hash[2..3]}") end end describe '#disk_path' do it 'returns disk_path based on hash of project id' do - hashed_path = '@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' + hashed_path = "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}" expect(project.disk_path).to eq(hashed_path) end @@ -2720,7 +2718,9 @@ describe Project do describe '#ensure_storage_path_exists' do it 'delegates to gitlab_shell to ensure namespace is created' do - expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '@hashed/6b/86') + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, "@hashed/#{hash[0..1]}/#{hash[2..3]}") project.ensure_storage_path_exists end @@ -2780,7 +2780,7 @@ describe Project do end context 'when not rolled out' do - let(:project) { create(:project, :repository, storage_version: 1) } + let(:project) { create(:project, :repository, storage_version: 1, skip_disk_validation: true) } it 'moves pages folder to new location' do expect_any_instance_of(Gitlab::UploadsTransfer).to receive(:rename_project) @@ -2791,9 +2791,9 @@ describe Project do end it 'updates project full path in .git/config' do - expect(project).to receive(:write_repository_config).with(:fullpath, project.full_path) - project.rename_repo + + expect(project.repo.config['gitlab.fullpath']).to eq(project.full_path) end end From fcb967ac672e224737f6e170693e45331eb4d636 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Dec 2017 15:32:08 -0200 Subject: [PATCH 12/19] Write projects config to all projects inside namespace in batches --- app/models/concerns/storage/legacy_namespace.rb | 2 +- app/models/namespace.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 22b9ef4e338..99dbd4fbacf 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -34,7 +34,7 @@ module Storage # So we basically we mute exceptions in next actions begin send_update_instructions - write_projects_full_path_config + write_projects_repository_config true rescue diff --git a/app/models/namespace.rb b/app/models/namespace.rb index efbfc607040..bdcc9159d26 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -269,8 +269,8 @@ class Namespace < ActiveRecord::Base RedirectRoute.permanent.exists?(path: path) end - def write_projects_full_path_config - all_projects.each do |project| + def write_projects_repository_config + all_projects.find_each do |project| project.expires_full_path_cache # we need to clear cache to validate renames correctly project.write_repository_config end From 2af3400c4eeb0227ca6f38117323a18e9fbd7d9b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Dec 2017 16:04:58 -0200 Subject: [PATCH 13/19] Add spec for Project#write_repository_config --- spec/models/project_spec.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 1d4b68bdf8d..cea22bbd184 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3155,4 +3155,26 @@ describe Project do it { is_expected.to eq(platform_kubernetes) } end end + + describe '#write_repository_config' do + set(:project) { create(:project, :repository) } + + it 'writes full path in .git/config when key is missing' do + project.write_repository_config + + expect(project.repo.config['gitlab.fullpath']).to eq project.full_path + end + + it 'updates full path in .git/config when key is present' do + project.write_repository_config(gl_full_path: 'old/path') + + expect { project.write_repository_config }.to change { project.repo.config['gitlab.fullpath'] }.from('old/path').to(project.full_path) + end + + it 'does not raise an error with an empty repository' do + project = create(:project_empty_repo) + + expect { project.write_repository_config }.not_to raise_error + end + end end From 58c2f3b5796cd8349dbd90404ed6ad0ca3ee6caf Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Dec 2017 16:49:32 -0200 Subject: [PATCH 14/19] Fix Repository#processable? to allow .git repos in the root folder --- lib/gitlab/bare_repository_import/repository.rb | 3 ++- .../gitlab/bare_repository_import/repository_spec.rb | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index b5907875460..85b79362196 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -33,8 +33,9 @@ module Gitlab def processable? return false if wiki? + return false if hashed? && (group_path.blank? || project_name.blank?) - group_path.present? && project_name.present? + true end private diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index 770e26bbf6a..e0b7d16ebb7 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -27,7 +27,13 @@ describe ::Gitlab::BareRepositoryImport::Repository do expect(subject.processable?).to eq(false) end - it 'returns true when group and project name are present' do + it 'returns true if group path is missing' do + subject = described_class.new('/full/path/', '/full/path/repo.git') + + expect(subject.processable?).to eq(true) + end + + it 'returns true when group path and project name are present' do expect(subject.processable?).to eq(true) end end @@ -97,7 +103,7 @@ describe ::Gitlab::BareRepositoryImport::Repository do expect(subject.processable?).to eq(false) end - it 'returns true when group and project name are present' do + it 'returns true when group path and project name are present' do expect(subject.processable?).to eq(true) end end From 6c95c771fb9a3c71d176aa996b82096a8e0a3f7a Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 21 Dec 2017 17:08:00 -0200 Subject: [PATCH 15/19] Fix Projects::TransferService#write_repository_config method --- app/services/projects/transfer_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 14cf9f82bdb..26765e5c3f3 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -101,7 +101,7 @@ module Projects end def write_repository_config(full_path) - project.write_repository_config(:gl_fullpath, full_path) + project.write_repository_config(gl_full_path: full_path) end def refresh_permissions From 62ee2ccfcc1d765cf2b80ba8f7a226855f2f8a2f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 22 Dec 2017 11:28:46 -0200 Subject: [PATCH 16/19] Refactoring spec for Gitlab::BareRepositoryImport::Repository --- .../bare_repository_import/repository_spec.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index e0b7d16ebb7..39f2cfe6175 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -39,15 +39,14 @@ describe ::Gitlab::BareRepositoryImport::Repository do end describe '#project_full_path' do - it 'returns the project full path' do - expect(subject.repo_path).to eq('/full/path/to/repo.git') + it 'returns the project full path with trailing slash in the root path' do expect(subject.project_full_path).to eq('to/repo') end - it 'with no trailing slash in the root path' do - repo_path = described_class.new('/full/path', '/full/path/to/repo.git') + it 'returns the project full path with no trailing slash in the root path' do + subject = described_class.new('/full/path', '/full/path/to/repo.git') - expect(repo_path.project_full_path).to eq('to/repo') + expect(subject.project_full_path).to eq('to/repo') end end end @@ -109,7 +108,13 @@ describe ::Gitlab::BareRepositoryImport::Repository do end describe '#project_full_path' do - it 'returns the project full path' do + it 'returns the project full path with trailing slash in the root path' do + expect(subject.project_full_path).to eq('to/repo') + end + + it 'returns the project full path with no trailing slash in the root path' do + subject = described_class.new(root_path[0...-1], repo_path) + expect(subject.project_full_path).to eq('to/repo') end end From 3f5403ab74b2f60c1a306a2f617d1cd323854c7a Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 3 Jan 2018 16:17:11 -0200 Subject: [PATCH 17/19] Remove unused variable from bare repository importer spec --- spec/lib/gitlab/bare_repository_import/importer_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 99cc9c4bd41..b5d86df09d2 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -1,7 +1,6 @@ require 'spec_helper' describe Gitlab::BareRepositoryImport::Importer, repository: true do - let(:gitlab_shell) { Gitlab::Shell.new } let!(:admin) { create(:admin) } let!(:base_dir) { Dir.mktmpdir + '/' } let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) } From 08de4746dc03e7f090546063711153e99de344ae Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 3 Jan 2018 16:22:00 -0200 Subject: [PATCH 18/19] Refactoring Gitlab::BareRepositoryImport::Repository --- .../gitlab/bare_repository_import/repository_spec.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index 39f2cfe6175..9f42cf1dfca 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -24,17 +24,17 @@ describe ::Gitlab::BareRepositoryImport::Repository do it 'returns false if it is a wiki' do subject = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git') - expect(subject.processable?).to eq(false) + expect(subject).not_to be_processable end it 'returns true if group path is missing' do subject = described_class.new('/full/path/', '/full/path/repo.git') - expect(subject.processable?).to eq(true) + expect(subject).to be_processable end it 'returns true when group path and project name are present' do - expect(subject.processable?).to eq(true) + expect(subject).to be_processable end end @@ -92,18 +92,18 @@ describe ::Gitlab::BareRepositoryImport::Repository do it 'returns false if it is a wiki' do subject = described_class.new(root_path, wiki_path) - expect(subject.processable?).to eq(false) + expect(subject).not_to be_processable end it 'returns false when group and project name are missing' do repository = Rugged::Repository.new(repo_path) repository.config.delete('gitlab.fullpath') - expect(subject.processable?).to eq(false) + expect(subject).not_to be_processable end it 'returns true when group path and project name are present' do - expect(subject.processable?).to eq(true) + expect(subject).to be_processable end end From 14336c0bda493a870ffeed86379b274c522fe804 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 3 Jan 2018 16:26:59 -0200 Subject: [PATCH 19/19] Use if instead of unless on Projects::HashedStorage::MigrateRepositoryService --- .../projects/hashed_storage/migrate_repository_service.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb index b6763c9436f..67178de75de 100644 --- a/app/services/projects/hashed_storage/migrate_repository_service.rb +++ b/app/services/projects/hashed_storage/migrate_repository_service.rb @@ -27,11 +27,11 @@ module Projects result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki") end - unless result + if result + project.write_repository_config + else rollback_folder_move project.storage_version = nil - else - project.write_repository_config end project.repository_read_only = false