From 76cd2d7881da69697f5c6584be04abbfb9411a22 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 6 Apr 2018 13:14:05 -0300 Subject: [PATCH] Move checksum calculation to Gitlab::Git::Repository --- lib/gitlab/git/checksum.rb | 97 -------------------------- lib/gitlab/git/repository.rb | 40 +++++++++++ spec/lib/gitlab/git/checksum_spec.rb | 49 ------------- spec/lib/gitlab/git/repository_spec.rb | 49 +++++++++++++ 4 files changed, 89 insertions(+), 146 deletions(-) delete mode 100644 lib/gitlab/git/checksum.rb delete mode 100644 spec/lib/gitlab/git/checksum_spec.rb diff --git a/lib/gitlab/git/checksum.rb b/lib/gitlab/git/checksum.rb deleted file mode 100644 index 218651906bf..00000000000 --- a/lib/gitlab/git/checksum.rb +++ /dev/null @@ -1,97 +0,0 @@ -module Gitlab - module Git - class Checksum - include Gitlab::Git::Popen - - EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze - - Failure = Class.new(StandardError) - - attr_reader :path, :relative_path, :storage, :storage_path, :gl_repository - - def initialize(storage, relative_path, gl_repository) - @storage = storage - @storage_path = Gitlab.config.repositories.storages[storage].legacy_disk_path - @relative_path = "#{relative_path}.git" - @path = File.join(storage_path, @relative_path) - @gl_repository = gl_repository - end - - def calculate - unless repository_exists? - failure!(Gitlab::Git::Repository::NoRepository, 'No repository for such path') - end - - raw_repository.gitaly_migrate(:calculate_checksum) do |is_enabled| - if is_enabled - calculate_checksum_gitaly - else - calculate_checksum_by_shelling_out - end - end - end - - private - - def repository_exists? - raw_repository.exists? - end - - def calculate_checksum_gitaly - gitaly_repository_client.calculate_checksum - end - - def calculate_checksum_by_shelling_out - args = %W(--git-dir=#{path} show-ref --heads --tags) - output, status = run_git(args) - - if status&.zero? - refs = output.split("\n") - - result = refs.inject(nil) do |checksum, ref| - value = Digest::SHA1.hexdigest(ref).hex - - if checksum.nil? - value - else - checksum ^ value - end - end - - result.to_s(16) - else - # Empty repositories return with a non-zero status and an empty output. - if output&.empty? - EMPTY_REPOSITORY_CHECKSUM - else - failure!(Gitlab::Git::Checksum::Failure, output) - end - end - end - - def failure!(klass, message) - Gitlab::GitLogger.error("'git show-ref --heads --tags' in #{path}: #{message}") - - raise klass.new("Could not calculate the checksum for #{path}: #{message}") - end - - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) - end - - def raw_repository - @raw_repository ||= Gitlab::Git::Repository.new(storage, relative_path, gl_repository) - end - - def gitaly_repository_client - raw_repository.gitaly_repository_client - end - - def run_git(args) - circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) - end - end - end - end -end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5678c28cf3a..79cacd9f6f5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -23,6 +23,7 @@ module Gitlab SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -31,6 +32,7 @@ module Gitlab DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the repository path @@ -1502,6 +1504,16 @@ module Gitlab FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) end + def checksum + gitaly_migrate(:calculate_checksum) do |is_enabled| + if is_enabled + gitaly_repository_client.calculate_checksum + else + calculate_checksum_by_shelling_out + end + end + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -2420,6 +2432,34 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def calculate_checksum_by_shelling_out + raise NoRepository unless exists? + + args = %W(--git-dir=#{path} show-ref --heads --tags) + output, status = run_git(args) + + if status.nil? || !status.zero? + # Empty repositories return with a non-zero status and an empty output. + return EMPTY_REPOSITORY_CHECKSUM if output&.empty? + + raise ChecksumError, output + end + + refs = output.split("\n") + + result = refs.inject(nil) do |checksum, ref| + value = Digest::SHA1.hexdigest(ref).hex + + if checksum.nil? + value + else + checksum ^ value + end + end + + result.to_s(16) + end end end end diff --git a/spec/lib/gitlab/git/checksum_spec.rb b/spec/lib/gitlab/git/checksum_spec.rb deleted file mode 100644 index a500b52cf20..00000000000 --- a/spec/lib/gitlab/git/checksum_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -require 'spec_helper' - -describe Gitlab::Git::Checksum, seed_helper: true do - let(:storage) { 'default' } - let(:gl_repository) { 'project-123' } - - shared_examples 'calculating checksum' do - it 'raises Gitlab::Git::Repository::NoRepository when there is no repo' do - checksum = described_class.new(storage, 'nonexistent-repo', gl_repository) - - expect { checksum.calculate }.to raise_error Gitlab::Git::Repository::NoRepository - end - - it 'pretends that checksum is 000000... when the repo is empty' do - FileUtils.rm_rf(File.join(SEED_STORAGE_PATH, 'empty-repo.git')) - - system(git_env, *%W(#{Gitlab.config.git.bin_path} init --bare empty-repo.git), - chdir: SEED_STORAGE_PATH, - out: '/dev/null', - err: '/dev/null') - - checksum = described_class.new(storage, 'empty-repo', gl_repository) - - expect(checksum.calculate).to eq '0000000000000000000000000000000000000000' - end - - it 'calculates the checksum when there is a repo' do - checksum = described_class.new(storage, 'gitlab-git-test', gl_repository) - - expect(checksum.calculate).to eq '54f21be4c32c02f6788d72207fa03ad3bce725e4' - end - end - - context 'when calculate_checksum Gitaly feature is enabled' do - it_behaves_like 'calculating checksum' - end - - context 'when calculate_checksum Gitaly feature is disabled', :disable_gitaly do - it_behaves_like 'calculating checksum' - - it "raises a Gitlab::Git::Repository::Failure error if the `popen` call to git returns a non-zero exit code" do - checksum = described_class.new(storage, 'gitlab-git-test', gl_repository) - - allow(checksum).to receive(:popen).and_return(['output', nil]) - - expect { checksum.calculate }.to raise_error Gitlab::Git::Checksum::Failure - end - end -end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index f934bf9a6b0..d3ab61746f4 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -2184,6 +2184,55 @@ describe Gitlab::Git::Repository, seed_helper: true do end end + describe '#checksum' do + shared_examples 'calculating checksum' do + it 'calculates the checksum for non-empty repo' do + expect(repository.checksum).to eq '54f21be4c32c02f6788d72207fa03ad3bce725e4' + end + + it 'returns 0000000000000000000000000000000000000000 for an empty repo' do + FileUtils.rm_rf(File.join(storage_path, 'empty-repo.git')) + + system(git_env, *%W(#{Gitlab.config.git.bin_path} init --bare empty-repo.git), + chdir: storage_path, + out: '/dev/null', + err: '/dev/null') + + empty_repo = described_class.new('default', 'empty-repo.git', '') + + expect(empty_repo.checksum).to eq '0000000000000000000000000000000000000000' + end + + it 'raises a no repository exception when there is no repo' do + broken_repo = described_class.new('default', 'a/path.git', '') + + expect { broken_repo.checksum }.to raise_error(Gitlab::Git::Repository::NoRepository) + end + end + + context 'when calculate_checksum Gitaly feature is enabled' do + it_behaves_like 'calculating checksum' + end + + context 'when calculate_checksum Gitaly feature is disabled', :disable_gitaly do + it_behaves_like 'calculating checksum' + + describe 'when storage is broken', :broken_storage do + it 'raises a storage exception when storage is not available' do + broken_repo = described_class.new('broken', 'a/path.git', '') + + expect { broken_repo.rugged }.to raise_error(Gitlab::Git::Storage::Inaccessible) + end + end + + it "raises a Gitlab::Git::Repository::Failure error if the `popen` call to git returns a non-zero exit code" do + allow(repository).to receive(:popen).and_return(['output', nil]) + + expect { repository.checksum }.to raise_error Gitlab::Git::Repository::ChecksumError + end + end + end + context 'gitlab_projects commands' do let(:gitlab_projects) { repository.gitlab_projects } let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }