From 6766a0a144bd07441b4593d25235924a14df0a91 Mon Sep 17 00:00:00 2001 From: Patrick Bajao Date: Mon, 25 Mar 2019 16:20:49 +0800 Subject: [PATCH] Download a folder from repository Add `GetArchiveRequest` to git-archive params. Modifies `Git::Repository#archive_metadata` to append `path` to `ArchivePrefix` so it'll not hit the cache of repository archive when it already exists. --- GITALY_SERVER_VERSION | 2 +- GITLAB_WORKHORSE_VERSION | 2 +- Gemfile | 2 +- Gemfile.lock | 4 +- .../projects/repositories_controller.rb | 2 +- app/helpers/projects_helper.rb | 12 ++---- app/helpers/workhorse_helper.rb | 1 - app/models/repository.rb | 5 ++- .../projects/buttons/_download.html.haml | 15 ++++++-- .../24704-download-repository-path.yml | 5 +++ lib/gitlab/git/repository.rb | 7 ++-- lib/gitlab/workhorse.rb | 38 +++++++++++++++++-- spec/lib/gitlab/git/repository_spec.rb | 11 +++++- spec/lib/gitlab/workhorse_spec.rb | 31 +++++++++------ 14 files changed, 97 insertions(+), 40 deletions(-) create mode 100644 changelogs/unreleased/24704-download-repository-path.yml diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 034552a83ee..7c343127b0f 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -1.30.0 +=24704-get-archive-by-path diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index d127a0ff9f1..32c1ab34777 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -8.3.3 +=24704-git-archive-by-path diff --git a/Gemfile b/Gemfile index c0815c4a2a6..a3a56de8099 100644 --- a/Gemfile +++ b/Gemfile @@ -419,7 +419,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 1.13.0', require: 'gitaly' +gem 'gitaly-proto', '~> 1.19.0', require: 'gitaly' gem 'grpc', '~> 1.15.0' diff --git a/Gemfile.lock b/Gemfile.lock index 4ebcc6c81b2..e8053ada8b2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -281,7 +281,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (1.13.0) + gitaly-proto (1.19.0) grpc (~> 1.0) github-markup (1.7.0) gitlab-default_value_for (3.1.1) @@ -1017,7 +1017,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 1.13.0) + gitaly-proto (~> 1.19.0) github-markup (~> 1.7.0) gitlab-default_value_for (~> 3.1.1) gitlab-markup (~> 1.7.0) diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb index 86298ef6274..3b4215b766e 100644 --- a/app/controllers/projects/repositories_controller.rb +++ b/app/controllers/projects/repositories_controller.rb @@ -23,7 +23,7 @@ class Projects::RepositoriesController < Projects::ApplicationController append_sha = false if @filename == shortname end - send_git_archive @repository, ref: @ref, subdirectory: params[:subdirectory], format: params[:format], append_sha: append_sha + send_git_archive @repository, ref: @ref, path: params[:path], format: params[:format], append_sha: append_sha rescue => ex logger.error("#{self.class.name}: #{ex}") git_not_found! diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index f5cae40aa86..009dd70c2c9 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -26,14 +26,6 @@ module ProjectsHelper image_tag(src, width: opts[:size], class: classes, alt: '', "data-src" => avatar) end - - def is_directory - @path.empty? ? false : true - end - - def get_directory_path - @path ? "#{@path}/" : '' - end def author_content_tag(author, opts = {}) default_opts = { author_class: 'author', tooltip: false, by_username: false } @@ -307,6 +299,10 @@ module ProjectsHelper }.to_json end + def directory? + @path.present? + end + private def get_project_nav_tabs(project, current_user) diff --git a/app/helpers/workhorse_helper.rb b/app/helpers/workhorse_helper.rb index 7c49a6e1c6d..bb5b1555dc4 100644 --- a/app/helpers/workhorse_helper.rb +++ b/app/helpers/workhorse_helper.rb @@ -31,7 +31,6 @@ module WorkhorseHelper # Archive a Git repository and send it through Workhorse def send_git_archive(repository, **kwargs) - kwargs.delete(:subdirectory) if kwargs[:subdirectory].nil? headers.store(*Gitlab::Workhorse.send_git_archive(repository, **kwargs)) head :ok end diff --git a/app/models/repository.rb b/app/models/repository.rb index 574ce12b309..51ab2247a03 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -299,13 +299,14 @@ class Repository end end - def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:, path: nil) raw_repository.archive_metadata( ref, storage_path, project.path, format, - append_sha: append_sha + append_sha: append_sha, + path: path ) end diff --git a/app/views/projects/buttons/_download.html.haml b/app/views/projects/buttons/_download.html.haml index 7d22a4044e4..76690f4b678 100644 --- a/app/views/projects/buttons/_download.html.haml +++ b/app/views/projects/buttons/_download.html.haml @@ -8,14 +8,23 @@ %span.sr-only= _('Select Archive Format') = sprite_icon("arrow-down") %ul.dropdown-menu.dropdown-menu-right{ role: 'menu' } - - if is_directory + - if directory? %li.dropdown-header #{ _('Directory') } %li - = link_to project_archive_path(project, subdirectory: get_directory_path, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do + = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'zip'), rel: 'nofollow', download: '' do %span= _('Download zip') + %li + = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.gz'), rel: 'nofollow', download: '' do + %span= _('Download tar.gz') + %li + = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.bz2'), rel: 'nofollow', download: '' do + %span= _('Download tar.bz2') + %li + = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar'), rel: 'nofollow', download: '' do + %span= _('Download tar') %li.dropdown-header - #{ _('Repository') } + #{ _('Source code') } %li = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do %span= _('Download zip') diff --git a/changelogs/unreleased/24704-download-repository-path.yml b/changelogs/unreleased/24704-download-repository-path.yml new file mode 100644 index 00000000000..ff3082bec45 --- /dev/null +++ b/changelogs/unreleased/24704-download-repository-path.yml @@ -0,0 +1,5 @@ +--- +title: Download a folder from repository +merge_request: 26532 +author: kiameisomabes +type: added diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7d6851a4b8d..c33d243330d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -231,12 +231,12 @@ module Gitlab end end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) + def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) + prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha, path: path) { 'ArchivePrefix' => prefix, @@ -248,13 +248,14 @@ module Gitlab # This is both the filename of the archive (missing the extension) and the # name of the top-level member of the archive under which all files go - def archive_prefix(ref, sha, project_path, append_sha:) + def archive_prefix(ref, sha, project_path, append_sha:, path:) append_sha = (ref != sha) if append_sha.nil? formatted_ref = ref.tr('/', '-') prefix_segments = [project_path, formatted_ref] prefix_segments << sha if append_sha + prefix_segments << path.tr('/', '-').gsub(%r{^/|/$}, '') if path prefix_segments.join('-') end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5d5a867c9ab..83eabb8d674 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -63,13 +63,26 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:, append_sha:) + def send_git_archive(repository, ref:, format:, append_sha:, path: nil) format ||= 'tar.gz' format = format.downcase - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) - raise "Repository or ref not found" if params.empty? + metadata = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha, path: path) - params['GitalyServer'] = gitaly_server_hash(repository) + raise "Repository or ref not found" if metadata.empty? + + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'ArchivePath' => metadata['ArchivePath'], + 'GetArchiveRequest' => encode_binary( + Gitaly::GetArchiveRequest.new( + repository: repository.gitaly_repository, + commit_id: metadata['CommitId'], + prefix: metadata['ArchivePrefix'], + format: archive_format(format), + path: path.presence || "" + ).to_proto + ) + } # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. params['DisableCache'] = true if git_archive_cache_disabled? @@ -220,6 +233,10 @@ module Gitlab Base64.urlsafe_encode64(JSON.dump(hash)) end + def encode_binary(binary) + Base64.urlsafe_encode64(binary) + end + def gitaly_server_hash(repository) { address: Gitlab::GitalyClient.address(repository.project.repository_storage), @@ -238,6 +255,19 @@ module Gitlab def git_archive_cache_disabled? ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) end + + def archive_format(format) + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + Gitaly::GetArchiveRequest::Format::TAR_BZ2 + when "tar" + Gitaly::GetArchiveRequest::Format::TAR + when "zip" + Gitaly::GetArchiveRequest::Format::ZIP + else + Gitaly::GetArchiveRequest::Format::TAR_GZ + end + end end end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 8ba6862392c..fc8f590068a 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -152,13 +152,14 @@ describe Gitlab::Git::Repository, :seed_helper do let(:append_sha) { true } let(:ref) { 'master' } let(:format) { nil } + let(:path) { nil } let(:expected_extension) { 'tar.gz' } let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" } let(:expected_path) { File.join(storage_path, cache_key, expected_filename) } let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" } - subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha) } + subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha, path: path) } it 'sets CommitId to the commit SHA' do expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID) @@ -176,6 +177,14 @@ describe Gitlab::Git::Repository, :seed_helper do expect(metadata['ArchivePath']).to eq(expected_path) end + context 'path is set' do + let(:path) { 'foo/bar' } + + it 'appends the path to the prefix' do + expect(metadata['ArchivePrefix']).to eq("#{expected_prefix}-foo-bar") + end + end + context 'append_sha varies archive path and filename' do where(:append_sha, :ref, :expected_prefix) do sha = SeedRepo::LastCommit::ID diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index d88086b01b1..fed7834e2a9 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -16,20 +16,12 @@ describe Gitlab::Workhorse do let(:ref) { 'master' } let(:format) { 'zip' } let(:storage_path) { Gitlab.config.gitlab.repository_downloads_path } - let(:base_params) { repository.archive_metadata(ref, storage_path, format, append_sha: nil) } - let(:gitaly_params) do - base_params.merge( - 'GitalyServer' => { - 'address' => Gitlab::GitalyClient.address(project.repository_storage), - 'token' => Gitlab::GitalyClient.token(project.repository_storage) - }, - 'GitalyRepository' => repository.gitaly_repository.to_h.deep_stringify_keys - ) - end + let(:path) { 'some/path' } + let(:metadata) { repository.archive_metadata(ref, storage_path, format, append_sha: nil, path: path) } let(:cache_disabled) { false } subject do - described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil) + described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil, path: path) end before do @@ -41,7 +33,22 @@ describe Gitlab::Workhorse do expect(key).to eq('Gitlab-Workhorse-Send-Data') expect(command).to eq('git-archive') - expect(params).to include(gitaly_params) + expect(params).to eq({ + 'GitalyServer' => { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + }, + 'ArchivePath' => metadata['ArchivePath'], + 'GetArchiveRequest' => Base64.urlsafe_encode64( + Gitaly::GetArchiveRequest.new( + repository: repository.gitaly_repository, + commit_id: metadata['CommitId'], + prefix: metadata['ArchivePrefix'], + format: Gitaly::GetArchiveRequest::Format::ZIP, + path: path + ).to_proto + ) + }.deep_stringify_keys) end context 'when archive caching is disabled' do