kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Merge branch 'da-handle-hashed-storage-repos-using-repo-import-task' into 'master'

Browse source 
Handle GitLab hashed storage repositories using the repo import task

Closes #39870

See merge request gitlab-org/gitlab-ce!16027
This commit is contained in:
Robert Speicher 2018-01-03 20:18:49 +00:00
commit 034a21ba4d
18 changed files with 266 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/models/concerns/storage/legacy_namespace.rb
View file

@ -34,6 +34,8 @@ module Storage
# So we basically we mute exceptions in next actions
begin
send_update_instructions
write_projects_repository_config
true
rescue
# Returning false does not rollback after_* transaction but gives

7
app/models/namespace.rb
View file

@ -268,4 +268,11 @@ class Namespace < ActiveRecord::Base
def namespace_previously_created_with_same_path?
RedirectRoute.permanent.exists?(path: path)
end
def write_projects_repository_config
all_projects.find_each do |project|
project.expires_full_path_cache # we need to clear cache to validate renames correctly
project.write_repository_config
end
end
end

12
app/models/project.rb
View file

@ -1420,6 +1420,8 @@ class Project < ActiveRecord::Base
end
def after_rename_repo
write_repository_config
path_before_change = previous_changes['path'].first
# We need to check if project had been rolled out to move resource to hashed storage or not and decide
@ -1432,6 +1434,16 @@ class Project < ActiveRecord::Base
Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
end
def write_repository_config(gl_full_path: full_path)
# We'd need to keep track of project full path otherwise directory tree
# created with hashed storage enabled cannot be usefully imported using
# the import rake task.
repo.config['gitlab.fullpath'] = gl_full_path
rescue Gitlab::Git::Repository::NoRepository => e
Rails.logger.error("Error writing to .git/config for project #{full_path} (#{id}): #{e.message}.")
nil
end
def rename_repo_notify!
send_move_instructions(full_path_was)
expires_full_path_cache

1
app/services/projects/create_service.rb
View file

@ -88,6 +88,7 @@ module Projects
log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"")
unless @project.gitlab_project_import?
@project.write_repository_config
@project.create_wiki unless skip_wiki?
create_services_from_active_templates(@project)

4
app/services/projects/hashed_storage/migrate_repository_service.rb
View file

@ -27,7 +27,9 @@ module Projects
result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki")
end
unless result
if result
project.write_repository_config
else
rollback_folder_move
project.storage_version = nil
end

7
app/services/projects/transfer_service.rb
View file

@ -75,6 +75,8 @@ module Projects
project.old_path_with_namespace = @old_path
project.expires_full_path_cache
write_repository_config(@new_path)
execute_system_hooks
end
rescue Exception # rubocop:disable Lint/RescueException
@ -98,6 +100,10 @@ module Projects
project.save!
end
def write_repository_config(full_path)
project.write_repository_config(gl_full_path: full_path)
end
def refresh_permissions
# This ensures we only schedule 1 job for every user that has access to
# the namespaces.
@ -110,6 +116,7 @@ module Projects
def rollback_side_effects
rollback_folder_move
update_namespace_and_visibility(@old_namespace)
write_repository_config(@old_path)
end
def rollback_folder_move

5
changelogs/unreleased/da-handle-hashed-storage-repos-using-repo-import-task.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Handle GitLab hashed storage repositories using the repo import task
merge_request:
author:
type: added

4
lib/gitlab/bare_repository_import/importer.rb
View file

@ -14,7 +14,7 @@ module Gitlab
repos_to_import.each do |repo_path|
bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path)
if bare_repo.hashed? || bare_repo.wiki?
unless bare_repo.processable?
log " * Skipping repo #{bare_repo.repo_path}".color(:yellow)
next
@ -62,6 +62,8 @@ module Gitlab
if project.persisted? && mv_repo(project)
log " * Created #{project.name} (#{project_full_path})".color(:green)
project.write_repository_config
ProjectCacheWorker.perform_async(project.id)
else
log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red)

37
lib/gitlab/bare_repository_import/repository.rb
View file

@ -6,39 +6,56 @@ module Gitlab
def initialize(root_path, repo_path)
@root_path = root_path
@repo_path = repo_path
@root_path << '/' unless root_path.ends_with?('/')
full_path =
if hashed? && !wiki?
repository.config.get('gitlab.fullpath')
else
repo_relative_path
end
# Split path into 'all/the/namespaces' and 'project_name'
@group_path, _, @project_name = repo_relative_path.rpartition('/')
@group_path, _, @project_name = full_path.to_s.rpartition('/')
end
def wiki_exists?
File.exist?(wiki_path)
end
def wiki?
@wiki ||= repo_path.end_with?('.wiki.git')
end
def wiki_path
@wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git')
end
def hashed?
@hashed ||= group_path.start_with?('@hashed')
end
def project_full_path
@project_full_path ||= "#{group_path}/#{project_name}"
end
def processable?
return false if wiki?
return false if hashed? && (group_path.blank? || project_name.blank?)
true
end
private
def wiki?
@wiki ||= repo_path.end_with?('.wiki.git')
end
def hashed?
@hashed ||= repo_relative_path.include?('@hashed')
end
def repo_relative_path
# Remove root path and `.git` at the end
repo_path[@root_path.size...-4]
end
def repository
@repository ||= Rugged::Repository.new(repo_path)
end
end
end
end

13
spec/lib/gitlab/bare_repository_import/importer_spec.rb
View file

@ -75,7 +75,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
end
it 'creates the Git repo in disk' do
FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
create_bare_repository("#{project_path}.git")
importer.create_project_if_needed
@ -130,7 +130,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
end
it 'creates the Git repo in disk' do
FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
create_bare_repository("#{project_path}.git")
importer.create_project_if_needed
@ -165,8 +165,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
it_behaves_like 'importing a repository'
it 'creates the Wiki git repo in disk' do
FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git"))
create_bare_repository("#{project_path}.git")
create_bare_repository("#{project_path}.wiki.git")
expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true,
import_type: 'bare_repository')).and_call_original
@ -192,4 +192,9 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
end
end
end
def create_bare_repository(project_path)
repo_path = File.join(base_dir, project_path)
Gitlab::Git::Repository.create(repo_path, bare: true)
end
end

144
spec/lib/gitlab/bare_repository_import/repository_spec.rb
View file

@ -1,58 +1,122 @@
require 'spec_helper'
describe ::Gitlab::BareRepositoryImport::Repository do
let(:project_repo_path) { described_class.new('/full/path/', '/full/path/to/repo.git') }
context 'legacy storage' do
subject { described_class.new('/full/path/', '/full/path/to/repo.git') }
it 'stores the repo path' do
expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git')
end
it 'stores the group path' do
expect(project_repo_path.group_path).to eq('to')
end
it 'stores the project name' do
expect(project_repo_path.project_name).to eq('repo')
end
it 'stores the wiki path' do
expect(project_repo_path.wiki_path).to eq('/full/path/to/repo.wiki.git')
end
describe '#wiki?' do
it 'returns true if it is a wiki' do
wiki_path = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git')
expect(wiki_path.wiki?).to eq(true)
it 'stores the repo path' do
expect(subject.repo_path).to eq('/full/path/to/repo.git')
end
it 'returns false if it is not a wiki' do
expect(project_repo_path.wiki?).to eq(false)
it 'stores the group path' do
expect(subject.group_path).to eq('to')
end
it 'stores the project name' do
expect(subject.project_name).to eq('repo')
end
it 'stores the wiki path' do
expect(subject.wiki_path).to eq('/full/path/to/repo.wiki.git')
end
describe '#processable?' do
it 'returns false if it is a wiki' do
subject = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git')
expect(subject).not_to be_processable
end
it 'returns true if group path is missing' do
subject = described_class.new('/full/path/', '/full/path/repo.git')
expect(subject).to be_processable
end
it 'returns true when group path and project name are present' do
expect(subject).to be_processable
end
end
describe '#project_full_path' do
it 'returns the project full path with trailing slash in the root path' do
expect(subject.project_full_path).to eq('to/repo')
end
it 'returns the project full path with no trailing slash in the root path' do
subject = described_class.new('/full/path', '/full/path/to/repo.git')
expect(subject.project_full_path).to eq('to/repo')
end
end
end
describe '#hashed?' do
it 'returns true if it is a hashed folder' do
path = described_class.new('/full/path/', '/full/path/@hashed/my.repo.git')
context 'hashed storage' do
let(:gitlab_shell) { Gitlab::Shell.new }
let(:repository_storage) { 'default' }
let(:root_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" }
let(:repo_path) { File.join(root_path, "#{hashed_path}.git") }
let(:wiki_path) { File.join(root_path, "#{hashed_path}.wiki.git") }
expect(path.hashed?).to eq(true)
before do
gitlab_shell.add_repository(repository_storage, hashed_path)
repository = Rugged::Repository.new(repo_path)
repository.config['gitlab.fullpath'] = 'to/repo'
end
it 'returns false if it is not a hashed folder' do
expect(project_repo_path.hashed?).to eq(false)
end
end
describe '#project_full_path' do
it 'returns the project full path' do
expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git')
expect(project_repo_path.project_full_path).to eq('to/repo')
after do
gitlab_shell.remove_repository(root_path, hashed_path)
end
it 'with no trailing slash in the root path' do
repo_path = described_class.new('/full/path', '/full/path/to/repo.git')
subject { described_class.new(root_path, repo_path) }
expect(repo_path.project_full_path).to eq('to/repo')
it 'stores the repo path' do
expect(subject.repo_path).to eq(repo_path)
end
it 'stores the wiki path' do
expect(subject.wiki_path).to eq(wiki_path)
end
it 'reads the group path from .git/config' do
expect(subject.group_path).to eq('to')
end
it 'reads the project name from .git/config' do
expect(subject.project_name).to eq('repo')
end
describe '#processable?' do
it 'returns false if it is a wiki' do
subject = described_class.new(root_path, wiki_path)
expect(subject).not_to be_processable
end
it 'returns false when group and project name are missing' do
repository = Rugged::Repository.new(repo_path)
repository.config.delete('gitlab.fullpath')
expect(subject).not_to be_processable
end
it 'returns true when group path and project name are present' do
expect(subject).to be_processable
end
end
describe '#project_full_path' do
it 'returns the project full path with trailing slash in the root path' do
expect(subject.project_full_path).to eq('to/repo')
end
it 'returns the project full path with no trailing slash in the root path' do
subject = described_class.new(root_path[0...-1], repo_path)
expect(subject.project_full_path).to eq('to/repo')
end
end
end
end

16
spec/models/namespace_spec.rb
View file

@ -203,7 +203,7 @@ describe Namespace do
context 'with subgroups' do
let(:parent) { create(:group, name: 'parent', path: 'parent') }
let(:child) { create(:group, name: 'child', path: 'child', parent: parent) }
let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child) }
let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) }
let(:uploads_dir) { File.join(CarrierWave.root, FileUploader.base_dir) }
let(:pages_dir) { File.join(TestEnv.pages_path) }
@ -240,6 +240,20 @@ describe Namespace do
end
end
end
it 'updates project full path in .git/config for each project inside namespace' do
parent = create(:group, name: 'mygroup', path: 'mygroup')
subgroup = create(:group, name: 'mysubgroup', path: 'mysubgroup', parent: parent)
project_in_parent_group = create(:project, :repository, namespace: parent, name: 'foo1')
hashed_project_in_subgroup = create(:project, :repository, :hashed, namespace: subgroup, name: 'foo2')
legacy_project_in_subgroup = create(:project, :repository, namespace: subgroup, name: 'foo3')
parent.update(path: 'mygroup_new')
expect(project_in_parent_group.repo.config['gitlab.fullpath']).to eq "mygroup_new/#{project_in_parent_group.path}"
expect(hashed_project_in_subgroup.repo.config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{hashed_project_in_subgroup.path}"
expect(legacy_project_in_subgroup.repo.config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{legacy_project_in_subgroup.path}"
end
end
describe '#rm_dir', 'callback' do

52
spec/models/project_spec.rb
View file

@ -2626,6 +2626,14 @@ describe Project do
project.rename_repo
end
end
it 'updates project full path in .git/config' do
allow(project_storage).to receive(:rename_repo).and_return(true)
project.rename_repo
expect(project.repo.config['gitlab.fullpath']).to eq(project.full_path)
end
end
describe '#pages_path' do
@ -2668,14 +2676,12 @@ describe Project do
end
context 'hashed storage' do
let(:project) { create(:project, :repository) }
let(:project) { create(:project, :repository, skip_disk_validation: true) }
let(:gitlab_shell) { Gitlab::Shell.new }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
let(:hash) { Digest::SHA2.hexdigest(project.id.to_s) }
before do
stub_application_setting(hashed_storage_enabled: true)
allow(Digest::SHA2).to receive(:hexdigest) { hash }
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
end
describe '#legacy_storage?' do
@ -2698,13 +2704,13 @@ describe Project do
describe '#base_dir' do
it 'returns base_dir based on hash of project id' do
expect(project.base_dir).to eq('@hashed/6b/86')
expect(project.base_dir).to eq("@hashed/#{hash[0..1]}/#{hash[2..3]}")
end
end
describe '#disk_path' do
it 'returns disk_path based on hash of project id' do
hashed_path = '@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b'
hashed_path = "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}"
expect(project.disk_path).to eq(hashed_path)
end
@ -2712,7 +2718,9 @@ describe Project do
describe '#ensure_storage_path_exists' do
it 'delegates to gitlab_shell to ensure namespace is created' do
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '@hashed/6b/86')
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, "@hashed/#{hash[0..1]}/#{hash[2..3]}")
project.ensure_storage_path_exists
end
@ -2772,7 +2780,7 @@ describe Project do
end
context 'when not rolled out' do
let(:project) { create(:project, :repository, storage_version: 1) }
let(:project) { create(:project, :repository, storage_version: 1, skip_disk_validation: true) }
it 'moves pages folder to new location' do
expect_any_instance_of(Gitlab::UploadsTransfer).to receive(:rename_project)
@ -2781,6 +2789,12 @@ describe Project do
end
end
end
it 'updates project full path in .git/config' do
project.rename_repo
expect(project.repo.config['gitlab.fullpath']).to eq(project.full_path)
end
end
describe '#pages_path' do
@ -3141,4 +3155,26 @@ describe Project do
it { is_expected.to eq(platform_kubernetes) }
end
end
describe '#write_repository_config' do
set(:project) { create(:project, :repository) }
it 'writes full path in .git/config when key is missing' do
project.write_repository_config
expect(project.repo.config['gitlab.fullpath']).to eq project.full_path
end
it 'updates full path in .git/config when key is present' do
project.write_repository_config(gl_full_path: 'old/path')
expect { project.write_repository_config }.to change { project.repo.config['gitlab.fullpath'] }.from('old/path').to(project.full_path)
end
it 'does not raise an error with an empty repository' do
project = create(:project_empty_repo)
expect { project.write_repository_config }.not_to raise_error
end
end
end

6
spec/services/projects/create_service_spec.rb
View file

@ -252,6 +252,12 @@ describe Projects::CreateService, '#execute' do
end
end
it 'writes project full path to .git/config' do
project = create_project(user, opts)
expect(project.repo.config['gitlab.fullpath']).to eq project.full_path
end
def create_project(user, opts)
Projects::CreateService.new(user, opts).execute
end

8
spec/services/projects/hashed_storage/migrate_repository_service_spec.rb
View file

@ -2,7 +2,7 @@ require 'spec_helper'
describe Projects::HashedStorage::MigrateRepositoryService do
let(:gitlab_shell) { Gitlab::Shell.new }
let(:project) { create(:project, :empty_repo, :wiki_repo) }
let(:project) { create(:project, :repository, :wiki_repo) }
let(:service) { described_class.new(project) }
let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) }
@ -33,6 +33,12 @@ describe Projects::HashedStorage::MigrateRepositoryService do
service.execute
end
it 'writes project full path to .git/config' do
service.execute
expect(project.repo.config['gitlab.fullpath']).to eq project.full_path
end
end
context 'when one move fails' do

12
spec/services/projects/transfer_service_spec.rb
View file

@ -54,6 +54,12 @@ describe Projects::TransferService do
expect(project.disk_path).not_to eq(old_path)
expect(project.disk_path).to start_with(group.path)
end
it 'updates project full path in .git/config' do
transfer_project(project, user, group)
expect(project.repo.config['gitlab.fullpath']).to eq "#{group.full_path}/#{project.path}"
end
end
context 'when transfer fails' do
@ -86,6 +92,12 @@ describe Projects::TransferService do
expect(original_path).to eq current_path
end
it 'rolls back project full path in .git/config' do
attempt_project_transfer
expect(project.repo.config['gitlab.fullpath']).to eq project.full_path
end
it "doesn't send move notifications" do
expect_any_instance_of(NotificationService).not_to receive(:project_was_moved)

2
spec/services/projects/update_service_spec.rb
View file

@ -61,7 +61,7 @@ describe Projects::UpdateService do
end
end
context 'When project visibility is higher than parent group' do
context 'when project visibility is higher than parent group' do
let(:group) { create(:group, visibility_level: Gitlab::VisibilityLevel::INTERNAL) }
before do

2
spec/support/test_env.rb
View file

@ -215,7 +215,7 @@ module TestEnv
end
def copy_repo(project, bare_repo:, refs:)
target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.full_path}.git")
target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.disk_path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path