diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb new file mode 100644 index 00000000000..38e9d2a8554 --- /dev/null +++ b/lib/gitlab/git/remote_mirror.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class RemoteMirror + def initialize(repository, ref_name) + @repository = repository + @ref_name = ref_name + end + + def update(only_branches_matching: [], only_tags_matching: []) + local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) + remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + + updated_branches = changed_refs(local_branches, remote_branches) + push_branches(updated_branches.keys) if updated_branches.present? + + delete_refs(local_branches, remote_branches) + + local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) + remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + + updated_tags = changed_refs(local_tags, remote_tags) + @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? + + delete_refs(local_tags, remote_tags) + end + + private + + def refs_obj(refs, only_refs_matching: []) + refs.each_with_object({}) do |ref, refs| + next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) + + refs[ref.name] = ref + end + end + + def changed_refs(local_refs, remote_refs) + local_refs.select do |ref_name, ref| + remote_ref = remote_refs[ref_name] + + remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target + end + end + + def push_branches(branches) + default_branch, branches = branches.partition do |branch| + @repository.root_ref == branch + end + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + @repository.push_remote_branches(@ref_name, branches) + end + + def delete_refs(local_refs, remote_refs) + refs = refs_to_delete(local_refs, remote_refs) + + @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? + end + + def refs_to_delete(local_refs, remote_refs) + default_branch_id = @repository.commit.id + + remote_refs.select do |remote_ref_name, remote_ref| + next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo + + remote_ref_id = remote_ref.dereferenced_target.try(:id) + + remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 176bd953ca1..aec85f971ca 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -21,6 +21,7 @@ module Gitlab REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -83,7 +84,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -93,6 +94,12 @@ module Gitlab @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @gitlab_projects = Gitlab::Git::GitlabProjects.new( + storage_path, + relative_path, + global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, + logger: Rails.logger + ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) @@ -1266,6 +1273,24 @@ module Gitlab fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) end + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1944,6 +1969,10 @@ module Gitlab def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..40650fc5ee7 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -306,47 +306,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 2531e1e7507..faccc2c8e00 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -18,9 +18,10 @@ describe Gitlab::Git::Repository, seed_helper: true do end let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') } + let(:storage_path) { TestEnv.repos_path } describe '.create_hooks' do - let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') } + let(:repo_path) { File.join(storage_path, 'hook-test.git') } let(:hooks_dir) { File.join(repo_path, 'hooks') } let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path } let(:existing_target) { File.join(repo_path, 'foobar') } @@ -645,7 +646,7 @@ describe Gitlab::Git::Repository, seed_helper: true do end after do - Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project') + Gitlab::Shell.new.remove_repository(storage_path, 'my_project') end it 'fetches a repository as a mirror remote' do @@ -1906,6 +1907,110 @@ describe Gitlab::Git::Repository, seed_helper: true do end end + describe '#gitlab_projects' do + subject { repository.gitlab_projects } + + it { expect(subject.shard_path).to eq(storage_path) } + it { expect(subject.repository_relative_path).to eq(repository.relative_path) } + end + + context 'gitlab_projects commands' do + let(:gitlab_projects) { repository.gitlab_projects } + let(:timeout) { Gitlab.config.gitlab_shell.git_timeout } + + describe '#push_remote_branches' do + subject do + repository.push_remote_branches('downstream-remote', ['master']) + end + + it 'executes the command' do + expect(gitlab_projects).to receive(:push_branches) + .with('downstream-remote', timeout, true, ['master']) + .and_return(true) + + is_expected.to be_truthy + end + + it 'raises an error if the command fails' do + allow(gitlab_projects).to receive(:output) { 'error' } + expect(gitlab_projects).to receive(:push_branches) + .with('downstream-remote', timeout, true, ['master']) + .and_return(false) + + expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error') + end + end + + describe '#delete_remote_branches' do + subject do + repository.delete_remote_branches('downstream-remote', ['master']) + end + + it 'executes the command' do + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(true) + + is_expected.to be_truthy + end + + it 'raises an error if the command fails' do + allow(gitlab_projects).to receive(:output) { 'error' } + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(false) + + expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error') + end + end + + describe '#delete_remote_branches' do + subject do + repository.delete_remote_branches('downstream-remote', ['master']) + end + + it 'executes the command' do + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(true) + + is_expected.to be_truthy + end + + it 'raises an error if the command fails' do + allow(gitlab_projects).to receive(:output) { 'error' } + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(false) + + expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error') + end + end + + describe '#delete_remote_branches' do + subject do + repository.delete_remote_branches('downstream-remote', ['master']) + end + + it 'executes the command' do + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(true) + + is_expected.to be_truthy + end + + it 'raises an error if the command fails' do + allow(gitlab_projects).to receive(:output) { 'error' } + expect(gitlab_projects).to receive(:delete_remote_branches) + .with('downstream-remote', ['master']) + .and_return(false) + + expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error') + end + end + end + def create_remote_branch(repository, remote_name, branch_name, source_branch_name) source_branch = repository.branches.find { |branch| branch.name == source_branch_name } rugged = repository.rugged diff --git a/spec/lib/gitlab/shell_spec.rb b/spec/lib/gitlab/shell_spec.rb index dd779b04741..81d9e6a8f82 100644 --- a/spec/lib/gitlab/shell_spec.rb +++ b/spec/lib/gitlab/shell_spec.rb @@ -347,62 +347,6 @@ describe Gitlab::Shell do end.to raise_error(Gitlab::Shell::Error, "error") end end - - describe '#push_remote_branches' do - subject(:result) do - gitlab_shell.push_remote_branches( - project.repository_storage_path, - project.disk_path, - 'downstream-remote', - ['master'] - ) - end - - it 'executes the command' do - expect(gitlab_projects).to receive(:push_branches) - .with('downstream-remote', timeout, true, ['master']) - .and_return(true) - - is_expected.to be_truthy - end - - it 'fails to execute the command' do - allow(gitlab_projects).to receive(:output) { 'error' } - expect(gitlab_projects).to receive(:push_branches) - .with('downstream-remote', timeout, true, ['master']) - .and_return(false) - - expect { result }.to raise_error(Gitlab::Shell::Error, 'error') - end - end - - describe '#delete_remote_branches' do - subject(:result) do - gitlab_shell.delete_remote_branches( - project.repository_storage_path, - project.disk_path, - 'downstream-remote', - ['master'] - ) - end - - it 'executes the command' do - expect(gitlab_projects).to receive(:delete_remote_branches) - .with('downstream-remote', ['master']) - .and_return(true) - - is_expected.to be_truthy - end - - it 'fails to execute the command' do - allow(gitlab_projects).to receive(:output) { 'error' } - expect(gitlab_projects).to receive(:delete_remote_branches) - .with('downstream-remote', ['master']) - .and_return(false) - - expect { result }.to raise_error(Gitlab::Shell::Error, 'error') - end - end end describe 'namespace actions' do