From 24625323a826d70991fdf27fbbe1f39bee845c03 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Thu, 18 Jan 2018 17:33:35 +0000 Subject: [PATCH] Migrate repository bundling to Gitaly Closes gitaly#929 --- GITALY_SERVER_VERSION | 2 +- Gemfile | 2 +- Gemfile.lock | 4 +-- app/models/repository.rb | 1 + lib/gitlab/git/repository.rb | 12 ++++++++ .../gitaly_client/repository_service.rb | 17 +++++++++++ lib/gitlab/import_export/command_line_util.rb | 4 --- lib/gitlab/import_export/repo_saver.rb | 2 +- lib/gitlab/import_export/wiki_repo_saver.rb | 2 +- spec/lib/gitlab/git/repository_spec.rb | 28 +++++++++++++++++++ 10 files changed, 64 insertions(+), 10 deletions(-) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 7375dee5f49..31b648bd6fa 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.72.0 +0.73.0 diff --git a/Gemfile b/Gemfile index a4c3db4bf5f..346182b3852 100644 --- a/Gemfile +++ b/Gemfile @@ -406,7 +406,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.74.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.76.0', require: 'gitaly' gem 'toml-rb', '~> 0.3.15', require: false diff --git a/Gemfile.lock b/Gemfile.lock index d69c532b309..1cbeab8d6b5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -285,7 +285,7 @@ GEM po_to_json (>= 1.0.0) rails (>= 3.2.0) gherkin-ruby (0.3.2) - gitaly-proto (0.74.0) + gitaly-proto (0.76.0) google-protobuf (~> 3.1) grpc (~> 1.0) github-linguist (4.7.6) @@ -1056,7 +1056,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.2.0) - gitaly-proto (~> 0.74.0) + gitaly-proto (~> 0.76.0) github-linguist (~> 4.7.0) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-markup (~> 1.6.2) diff --git a/app/models/repository.rb b/app/models/repository.rb index 73c4899cb9b..824e18bec78 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -20,6 +20,7 @@ class Repository attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository + delegate :bundle_to_disk, to: :raw_repository CreateTreeError = Class.new(StandardError) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d666362de0d..9864e4251f1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1268,6 +1268,18 @@ module Gitlab success || gitlab_projects_error end + def bundle_to_disk(save_path) + gitaly_migrate(:bundle_to_disk) do |is_enabled| + if is_enabled + gitaly_repository_client.create_bundle(save_path) + else + run_git!(%W(bundle create #{save_path} --all)) + end + end + + true + end + # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 12016aee2a6..654a3c314f1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -161,6 +161,23 @@ module Gitlab return response.error.b, 1 end end + + def create_bundle(save_path) + request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + :create_bundle, + request, + timeout: GitalyClient.default_timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + end end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index dd5d35feab9..25399f307f2 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -11,10 +11,6 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def git_bundle(repo_path:, bundle_path:) - execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) - end - def git_clone_bundle(repo_path:, bundle_path:) execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path})) Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index a7028a32570..695462c7dd2 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -21,7 +21,7 @@ module Gitlab def bundle_to_disk mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + @project.repository.bundle_to_disk(@full_path) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1e6722a7bba..5fa2e101e29 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -10,7 +10,7 @@ module Gitlab def bundle_to_disk(full_path) mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: full_path) + @wiki.repository.bundle_to_disk(full_path) rescue => e @shared.error(e) false diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index aec7cde6df8..36ca3980de9 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1926,6 +1926,34 @@ describe Gitlab::Git::Repository, seed_helper: true do it { expect(subject.repository_relative_path).to eq(repository.relative_path) } end + describe '#bundle_to_disk' do + shared_examples 'bundling to disk' do + let(:save_path) { File.join(Dir.tmpdir, "repo-#{SecureRandom.hex}.bundle") } + + after do + FileUtils.rm_rf(save_path) + end + + it 'saves a bundle to disk' do + repository.bundle_to_disk(save_path) + + success = system( + *%W(#{Gitlab.config.git.bin_path} -C #{repository.path} bundle verify #{save_path}), + [:out, :err] => '/dev/null' + ) + expect(success).to be true + end + end + + context 'when Gitaly bundle_to_disk feature is enabled' do + it_behaves_like 'bundling to disk' + end + + context 'when Gitaly bundle_to_disk feature is disabled', :disable_gitaly do + it_behaves_like 'bundling to disk' + end + end + context 'gitlab_projects commands' do let(:gitlab_projects) { repository.gitlab_projects } let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }