diff --git a/Gemfile b/Gemfile index 6654b285a72..bbc777fec9e 100644 --- a/Gemfile +++ b/Gemfile @@ -417,7 +417,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly' +gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly' gem 'grpc', '~> 1.19.0' diff --git a/Gemfile.lock b/Gemfile.lock index ceece1da8d7..0ee59793469 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -283,7 +283,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (1.22.0) + gitaly-proto (1.26.0) grpc (~> 1.0) github-markup (1.7.0) gitlab-default_value_for (3.1.1) @@ -1056,7 +1056,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 1.22.0) + gitaly-proto (~> 1.26.0) github-markup (~> 1.7.0) gitlab-default_value_for (~> 3.1.1) gitlab-labkit (~> 0.1.2) diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 8eb3c28ab70..d0577d7a4ff 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -40,6 +40,10 @@ module Gitlab @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end + def fetch + object_pool_service.fetch(source_repository) + end + private def object_pool_service diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index ce1fb4d68ae..d7fac26bc13 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -33,6 +33,15 @@ module Gitlab GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, request, timeout: GitalyClient.fast_timeout) end + + def fetch(repository) + request = Gitaly::FetchIntoObjectPoolRequest.new( + object_pool: object_pool, + origin: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) + end end end end diff --git a/spec/lib/gitlab/git/object_pool_spec.rb b/spec/lib/gitlab/git/object_pool_spec.rb index 0d5069568e1..6511c2b61bf 100644 --- a/spec/lib/gitlab/git/object_pool_spec.rb +++ b/spec/lib/gitlab/git/object_pool_spec.rb @@ -3,8 +3,12 @@ require 'spec_helper' describe Gitlab::Git::ObjectPool do + include RepoHelpers + let(:pool_repository) { create(:pool_repository) } let(:source_repository) { pool_repository.source_project.repository } + let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) } + let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) } subject { pool_repository.object_pool } @@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do end end end + + describe '#fetch' do + let(:commit_count) { source_repository.commit_count } + + context "when the object's pool repository exists" do + it 'does not raise an error' do + expect { subject.fetch }.not_to raise_error + end + end + + context "when the object's pool repository does not exist" do + before do + subject.delete + end + + it "re-creates the object pool's repository" do + subject.fetch + + expect(subject.repository.exists?).to be(true) + end + + it 'does not raise an error' do + expect { subject.fetch }.not_to raise_error + end + + it 'fetches objects from the source repository' do + new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid + + expect(subject.repository.exists?).to be false + + subject.fetch + + expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count) + expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id) + end + end + end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 5f8a2848944..0f6aac9b6de 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -3,6 +3,7 @@ require "spec_helper" describe Gitlab::Git::Repository, :seed_helper do include Gitlab::EncodingHelper + include RepoHelpers using RSpec::Parameterized::TableSyntax shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method| @@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) end - # Build the options hash that's passed to Rugged::Commit#create - def commit_options(repo, index, target, ref, message) - options = {} - options[:tree] = index.write_tree(repo) - options[:author] = { - email: "test@example.com", - name: "Test Author", - time: Time.gm(2014, "mar", 3, 20, 15, 1) - } - options[:committer] = { - email: "test@example.com", - name: "Test Author", - time: Time.gm(2014, "mar", 3, 20, 15, 1) - } - options[:message] ||= message - options[:parents] = repo.empty? ? [] : [target].compact - options[:update_ref] = ref - - options - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of CHANGELOG with a single new line of text. - def new_commit_edit_old_file(repo) - oid = repo.write("I replaced the changelog with this text", :blob) - index = repo.index - index.read_tree(repo.head.target.tree) - index.add(path: "CHANGELOG", oid: oid, mode: 0100644) - - options = commit_options( - repo, - index, - repo.head.target, - "HEAD", - "Edit CHANGELOG in its original location" - ) - - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of the specified file_path with new text. - def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head) - oid = repo.write(text, :blob) - index = repo.index - index.read_tree(branch.target.tree) - index.add(path: file_path, oid: oid, mode: 0100644) - options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message) - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the - # contents of encoding/CHANGELOG with new text. - def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text) - branch = repo.branches[branch_name] - new_commit_edit_new_file(repo, file_path, commit_message, text, branch) - end - - # Writes a new commit to the repo and returns a Rugged::Commit. Moves the - # CHANGELOG file to the encoding/ directory. - def new_commit_move_file(repo) - blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid] - file_content = repo.lookup(blob_oid).content - oid = repo.write(file_content, :blob) - index = repo.index - index.read_tree(repo.head.target.tree) - index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) - index.remove("CHANGELOG") - - options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/") - - sha = Rugged::Commit.create(repo, options) - repo.lookup(sha) - end - def refs(dir) IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line| line.split("\t").last diff --git a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb index 149b7ec5bb0..0e0c3d329b5 100644 --- a/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb @@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do end end end + + describe '#fetch' do + before do + subject.delete + end + + it 'restores the pool repository objects' do + subject.fetch(project.repository) + + expect(object_pool.repository.exists?).to be(true) + end + + context 'when called twice' do + it "doesn't raise an error" do + subject.delete + + expect { subject.fetch(project.repository) }.not_to raise_error + end + end + end end diff --git a/spec/support/helpers/repo_helpers.rb b/spec/support/helpers/repo_helpers.rb index 4af90f4af79..44d95a029af 100644 --- a/spec/support/helpers/repo_helpers.rb +++ b/spec/support/helpers/repo_helpers.rb @@ -11,6 +11,8 @@ module RepoHelpers # blob.path # => 'files/js/commit.js.coffee' # blob.data # => 'class Commit...' # + # Build the options hash that's passed to Rugged::Commit#create + def sample_blob OpenStruct.new( oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3', @@ -129,4 +131,80 @@ eos file_content: content ).execute end + + def commit_options(repo, index, target, ref, message) + options = {} + options[:tree] = index.write_tree(repo) + options[:author] = { + email: "test@example.com", + name: "Test Author", + time: Time.gm(2014, "mar", 3, 20, 15, 1) + } + options[:committer] = { + email: "test@example.com", + name: "Test Author", + time: Time.gm(2014, "mar", 3, 20, 15, 1) + } + options[:message] ||= message + options[:parents] = repo.empty? ? [] : [target].compact + options[:update_ref] = ref + + options + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of CHANGELOG with a single new line of text. + def new_commit_edit_old_file(repo) + oid = repo.write("I replaced the changelog with this text", :blob) + index = repo.index + index.read_tree(repo.head.target.tree) + index.add(path: "CHANGELOG", oid: oid, mode: 0100644) + + options = commit_options( + repo, + index, + repo.head.target, + "HEAD", + "Edit CHANGELOG in its original location" + ) + + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of the specified file_path with new text. + def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head) + oid = repo.write(text, :blob) + index = repo.index + index.read_tree(branch.target.tree) + index.add(path: file_path, oid: oid, mode: 0100644) + options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message) + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the + # contents of encoding/CHANGELOG with new text. + def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text) + branch = repo.branches[branch_name] + new_commit_edit_new_file(repo, file_path, commit_message, text, branch) + end + + # Writes a new commit to the repo and returns a Rugged::Commit. Moves the + # CHANGELOG file to the encoding/ directory. + def new_commit_move_file(repo) + blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid] + file_content = repo.lookup(blob_oid).content + oid = repo.write(file_content, :blob) + index = repo.index + index.read_tree(repo.head.target.tree) + index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) + index.remove("CHANGELOG") + + options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/") + + sha = Rugged::Commit.create(repo, options) + repo.lookup(sha) + end end