Merge branch 'ce-gitaly-remote-mirror-prep' into 'master'
[CE] Move git operations for UpdateRemoteMirrorService into Gitlab::Git See merge request gitlab-org/gitlab-ce!16023
This commit is contained in:
commit
58a705aaac
5 changed files with 212 additions and 100 deletions
75
lib/gitlab/git/remote_mirror.rb
Normal file
75
lib/gitlab/git/remote_mirror.rb
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
module Gitlab
|
||||||
|
module Git
|
||||||
|
class RemoteMirror
|
||||||
|
def initialize(repository, ref_name)
|
||||||
|
@repository = repository
|
||||||
|
@ref_name = ref_name
|
||||||
|
end
|
||||||
|
|
||||||
|
def update(only_branches_matching: [], only_tags_matching: [])
|
||||||
|
local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
|
||||||
|
remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
|
||||||
|
|
||||||
|
updated_branches = changed_refs(local_branches, remote_branches)
|
||||||
|
push_branches(updated_branches.keys) if updated_branches.present?
|
||||||
|
|
||||||
|
delete_refs(local_branches, remote_branches)
|
||||||
|
|
||||||
|
local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching)
|
||||||
|
remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching)
|
||||||
|
|
||||||
|
updated_tags = changed_refs(local_tags, remote_tags)
|
||||||
|
@repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
|
||||||
|
|
||||||
|
delete_refs(local_tags, remote_tags)
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def refs_obj(refs, only_refs_matching: [])
|
||||||
|
refs.each_with_object({}) do |ref, refs|
|
||||||
|
next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
|
||||||
|
|
||||||
|
refs[ref.name] = ref
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def changed_refs(local_refs, remote_refs)
|
||||||
|
local_refs.select do |ref_name, ref|
|
||||||
|
remote_ref = remote_refs[ref_name]
|
||||||
|
|
||||||
|
remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def push_branches(branches)
|
||||||
|
default_branch, branches = branches.partition do |branch|
|
||||||
|
@repository.root_ref == branch
|
||||||
|
end
|
||||||
|
|
||||||
|
# Push the default branch first so it works fine when remote mirror is empty.
|
||||||
|
branches.unshift(*default_branch)
|
||||||
|
|
||||||
|
@repository.push_remote_branches(@ref_name, branches)
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_refs(local_refs, remote_refs)
|
||||||
|
refs = refs_to_delete(local_refs, remote_refs)
|
||||||
|
|
||||||
|
@repository.delete_remote_branches(@ref_name, refs.keys) if refs.present?
|
||||||
|
end
|
||||||
|
|
||||||
|
def refs_to_delete(local_refs, remote_refs)
|
||||||
|
default_branch_id = @repository.commit.id
|
||||||
|
|
||||||
|
remote_refs.select do |remote_ref_name, remote_ref|
|
||||||
|
next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo
|
||||||
|
|
||||||
|
remote_ref_id = remote_ref.dereferenced_target.try(:id)
|
||||||
|
|
||||||
|
remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -21,6 +21,7 @@ module Gitlab
|
||||||
REBASE_WORKTREE_PREFIX = 'rebase'.freeze
|
REBASE_WORKTREE_PREFIX = 'rebase'.freeze
|
||||||
SQUASH_WORKTREE_PREFIX = 'squash'.freeze
|
SQUASH_WORKTREE_PREFIX = 'squash'.freeze
|
||||||
GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
|
GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
|
||||||
|
GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
|
||||||
|
|
||||||
NoRepository = Class.new(StandardError)
|
NoRepository = Class.new(StandardError)
|
||||||
InvalidBlobName = Class.new(StandardError)
|
InvalidBlobName = Class.new(StandardError)
|
||||||
|
@ -83,7 +84,7 @@ module Gitlab
|
||||||
# Rugged repo object
|
# Rugged repo object
|
||||||
attr_reader :rugged
|
attr_reader :rugged
|
||||||
|
|
||||||
attr_reader :storage, :gl_repository, :relative_path
|
attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
|
||||||
|
|
||||||
# This initializer method is only used on the client side (gitlab-ce).
|
# This initializer method is only used on the client side (gitlab-ce).
|
||||||
# Gitaly-ruby uses a different initializer.
|
# Gitaly-ruby uses a different initializer.
|
||||||
|
@ -93,6 +94,12 @@ module Gitlab
|
||||||
@gl_repository = gl_repository
|
@gl_repository = gl_repository
|
||||||
|
|
||||||
storage_path = Gitlab.config.repositories.storages[@storage]['path']
|
storage_path = Gitlab.config.repositories.storages[@storage]['path']
|
||||||
|
@gitlab_projects = Gitlab::Git::GitlabProjects.new(
|
||||||
|
storage_path,
|
||||||
|
relative_path,
|
||||||
|
global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
|
||||||
|
logger: Rails.logger
|
||||||
|
)
|
||||||
@path = File.join(storage_path, @relative_path)
|
@path = File.join(storage_path, @relative_path)
|
||||||
@name = @relative_path.split("/").last
|
@name = @relative_path.split("/").last
|
||||||
@attributes = Gitlab::Git::Attributes.new(path)
|
@attributes = Gitlab::Git::Attributes.new(path)
|
||||||
|
@ -1266,6 +1273,24 @@ module Gitlab
|
||||||
fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
|
fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def push_remote_branches(remote_name, branch_names, forced: true)
|
||||||
|
success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names)
|
||||||
|
|
||||||
|
success || gitlab_projects_error
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_remote_branches(remote_name, branch_names)
|
||||||
|
success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
|
||||||
|
|
||||||
|
success || gitlab_projects_error
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_remote_branches(remote_name, branch_names)
|
||||||
|
success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
|
||||||
|
|
||||||
|
success || gitlab_projects_error
|
||||||
|
end
|
||||||
|
|
||||||
def gitaly_repository
|
def gitaly_repository
|
||||||
Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
|
Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
|
||||||
end
|
end
|
||||||
|
@ -1944,6 +1969,10 @@ module Gitlab
|
||||||
def fetch_remote(remote_name = 'origin', env: nil)
|
def fetch_remote(remote_name = 'origin', env: nil)
|
||||||
run_git(['fetch', remote_name], env: env).last.zero?
|
run_git(['fetch', remote_name], env: env).last.zero?
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def gitlab_projects_error
|
||||||
|
raise CommandError, @gitlab_projects.output
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -306,47 +306,6 @@ module Gitlab
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Push branch to remote repository
|
|
||||||
#
|
|
||||||
# storage - project's storage path
|
|
||||||
# project_name - project's disk path
|
|
||||||
# remote_name - remote name
|
|
||||||
# branch_names - remote branch names to push
|
|
||||||
# forced - should we use --force flag
|
|
||||||
#
|
|
||||||
# Ex.
|
|
||||||
# push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
|
|
||||||
#
|
|
||||||
def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
|
|
||||||
cmd = gitlab_projects(storage, "#{project_name}.git")
|
|
||||||
|
|
||||||
success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
|
|
||||||
|
|
||||||
raise Error, cmd.output unless success
|
|
||||||
|
|
||||||
success
|
|
||||||
end
|
|
||||||
|
|
||||||
# Delete branch from remote repository
|
|
||||||
#
|
|
||||||
# storage - project's storage path
|
|
||||||
# project_name - project's disk path
|
|
||||||
# remote_name - remote name
|
|
||||||
# branch_names - remote branch names
|
|
||||||
#
|
|
||||||
# Ex.
|
|
||||||
# delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
|
|
||||||
#
|
|
||||||
def delete_remote_branches(storage, project_name, remote_name, branch_names)
|
|
||||||
cmd = gitlab_projects(storage, "#{project_name}.git")
|
|
||||||
|
|
||||||
success = cmd.delete_remote_branches(remote_name, branch_names)
|
|
||||||
|
|
||||||
raise Error, cmd.output unless success
|
|
||||||
|
|
||||||
success
|
|
||||||
end
|
|
||||||
|
|
||||||
protected
|
protected
|
||||||
|
|
||||||
def gitlab_shell_path
|
def gitlab_shell_path
|
||||||
|
|
|
@ -18,9 +18,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
|
||||||
end
|
end
|
||||||
|
|
||||||
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
|
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
|
||||||
|
let(:storage_path) { TestEnv.repos_path }
|
||||||
|
|
||||||
describe '.create_hooks' do
|
describe '.create_hooks' do
|
||||||
let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') }
|
let(:repo_path) { File.join(storage_path, 'hook-test.git') }
|
||||||
let(:hooks_dir) { File.join(repo_path, 'hooks') }
|
let(:hooks_dir) { File.join(repo_path, 'hooks') }
|
||||||
let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
|
let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
|
||||||
let(:existing_target) { File.join(repo_path, 'foobar') }
|
let(:existing_target) { File.join(repo_path, 'foobar') }
|
||||||
|
@ -645,7 +646,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
|
||||||
end
|
end
|
||||||
|
|
||||||
after do
|
after do
|
||||||
Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
|
Gitlab::Shell.new.remove_repository(storage_path, 'my_project')
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'fetches a repository as a mirror remote' do
|
it 'fetches a repository as a mirror remote' do
|
||||||
|
@ -1906,6 +1907,110 @@ describe Gitlab::Git::Repository, seed_helper: true do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe '#gitlab_projects' do
|
||||||
|
subject { repository.gitlab_projects }
|
||||||
|
|
||||||
|
it { expect(subject.shard_path).to eq(storage_path) }
|
||||||
|
it { expect(subject.repository_relative_path).to eq(repository.relative_path) }
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'gitlab_projects commands' do
|
||||||
|
let(:gitlab_projects) { repository.gitlab_projects }
|
||||||
|
let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }
|
||||||
|
|
||||||
|
describe '#push_remote_branches' do
|
||||||
|
subject do
|
||||||
|
repository.push_remote_branches('downstream-remote', ['master'])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'executes the command' do
|
||||||
|
expect(gitlab_projects).to receive(:push_branches)
|
||||||
|
.with('downstream-remote', timeout, true, ['master'])
|
||||||
|
.and_return(true)
|
||||||
|
|
||||||
|
is_expected.to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'raises an error if the command fails' do
|
||||||
|
allow(gitlab_projects).to receive(:output) { 'error' }
|
||||||
|
expect(gitlab_projects).to receive(:push_branches)
|
||||||
|
.with('downstream-remote', timeout, true, ['master'])
|
||||||
|
.and_return(false)
|
||||||
|
|
||||||
|
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#delete_remote_branches' do
|
||||||
|
subject do
|
||||||
|
repository.delete_remote_branches('downstream-remote', ['master'])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'executes the command' do
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(true)
|
||||||
|
|
||||||
|
is_expected.to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'raises an error if the command fails' do
|
||||||
|
allow(gitlab_projects).to receive(:output) { 'error' }
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(false)
|
||||||
|
|
||||||
|
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#delete_remote_branches' do
|
||||||
|
subject do
|
||||||
|
repository.delete_remote_branches('downstream-remote', ['master'])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'executes the command' do
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(true)
|
||||||
|
|
||||||
|
is_expected.to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'raises an error if the command fails' do
|
||||||
|
allow(gitlab_projects).to receive(:output) { 'error' }
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(false)
|
||||||
|
|
||||||
|
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#delete_remote_branches' do
|
||||||
|
subject do
|
||||||
|
repository.delete_remote_branches('downstream-remote', ['master'])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'executes the command' do
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(true)
|
||||||
|
|
||||||
|
is_expected.to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'raises an error if the command fails' do
|
||||||
|
allow(gitlab_projects).to receive(:output) { 'error' }
|
||||||
|
expect(gitlab_projects).to receive(:delete_remote_branches)
|
||||||
|
.with('downstream-remote', ['master'])
|
||||||
|
.and_return(false)
|
||||||
|
|
||||||
|
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
|
def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
|
||||||
source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
|
source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
|
||||||
rugged = repository.rugged
|
rugged = repository.rugged
|
||||||
|
|
|
@ -347,62 +347,6 @@ describe Gitlab::Shell do
|
||||||
end.to raise_error(Gitlab::Shell::Error, "error")
|
end.to raise_error(Gitlab::Shell::Error, "error")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '#push_remote_branches' do
|
|
||||||
subject(:result) do
|
|
||||||
gitlab_shell.push_remote_branches(
|
|
||||||
project.repository_storage_path,
|
|
||||||
project.disk_path,
|
|
||||||
'downstream-remote',
|
|
||||||
['master']
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'executes the command' do
|
|
||||||
expect(gitlab_projects).to receive(:push_branches)
|
|
||||||
.with('downstream-remote', timeout, true, ['master'])
|
|
||||||
.and_return(true)
|
|
||||||
|
|
||||||
is_expected.to be_truthy
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'fails to execute the command' do
|
|
||||||
allow(gitlab_projects).to receive(:output) { 'error' }
|
|
||||||
expect(gitlab_projects).to receive(:push_branches)
|
|
||||||
.with('downstream-remote', timeout, true, ['master'])
|
|
||||||
.and_return(false)
|
|
||||||
|
|
||||||
expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
describe '#delete_remote_branches' do
|
|
||||||
subject(:result) do
|
|
||||||
gitlab_shell.delete_remote_branches(
|
|
||||||
project.repository_storage_path,
|
|
||||||
project.disk_path,
|
|
||||||
'downstream-remote',
|
|
||||||
['master']
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'executes the command' do
|
|
||||||
expect(gitlab_projects).to receive(:delete_remote_branches)
|
|
||||||
.with('downstream-remote', ['master'])
|
|
||||||
.and_return(true)
|
|
||||||
|
|
||||||
is_expected.to be_truthy
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'fails to execute the command' do
|
|
||||||
allow(gitlab_projects).to receive(:output) { 'error' }
|
|
||||||
expect(gitlab_projects).to receive(:delete_remote_branches)
|
|
||||||
.with('downstream-remote', ['master'])
|
|
||||||
.and_return(false)
|
|
||||||
|
|
||||||
expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
describe 'namespace actions' do
|
describe 'namespace actions' do
|
||||||
|
|
Loading…
Reference in a new issue