Merge branch '24704-download-repository-path' into 'master'

Download a folder from repository

Closes #24704

See merge request gitlab-org/gitlab-ce!26532

GITALY_SERVER_VERSION

@@ -1 +1 @@
-1.30.0
+1.31.0

GITLAB_WORKHORSE_VERSION

@@ -1 +1 @@
-8.3.3
+8.4.0

Gemfile

@@ -419,7 +419,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.13.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.19.0', require: 'gitaly'
 
 gem 'grpc', '~> 1.15.0'
 

Gemfile.lock

@@ -281,7 +281,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.13.0)
+    gitaly-proto (1.19.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1017,7 +1017,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.13.0)
+  gitaly-proto (~> 1.19.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-markup (~> 1.7.0)

app/assets/stylesheets/framework/dropdowns.scss

@@ -287,7 +287,7 @@
     list-style: none;
     padding: 0 1px;
 
-    a,
+    a:not(.btn),
     button,
     .menu-item {
       @include dropdown-link;
@@ -351,6 +351,10 @@
     // Expects up to 3 digits on the badge
     margin-right: 40px;
   }
+
+  .dropdown-menu-content {
+    padding: $dropdown-item-padding-y $dropdown-item-padding-x;
+  }
 }
 
 .droplab-dropdown {

app/controllers/projects/repositories_controller.rb

@@ -23,7 +23,7 @@ class Projects::RepositoriesController < Projects::ApplicationController
       append_sha = false if @filename == shortname
     end
 
-    send_git_archive @repository, ref: @ref, format: params[:format], append_sha: append_sha
+    send_git_archive @repository, ref: @ref, path: params[:path], format: params[:format], append_sha: append_sha
   rescue => ex
     logger.error("#{self.class.name}: #{ex}")
     git_not_found!

app/helpers/projects_helper.rb

@@ -299,6 +299,10 @@ module ProjectsHelper
     }.to_json
   end
 
+  def directory?
+    @path.present?
+  end
+
   private
 
   def get_project_nav_tabs(project, current_user)

app/models/repository.rb

@@ -299,13 +299,14 @@ class Repository
     end
   end
 
-  def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:)
+  def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:, path: nil)
     raw_repository.archive_metadata(
       ref,
       storage_path,
       project.path,
       format,
-      append_sha: append_sha
+      append_sha: append_sha,
+      path: path
     )
   end
 

app/views/projects/buttons/_download.html.haml

@@ -8,30 +8,20 @@
       %span.sr-only=  _('Select Archive Format')
       = sprite_icon("arrow-down")
     %ul.dropdown-menu.dropdown-menu-right{ role: 'menu' }
-      %li.dropdown-header
-        #{ _('Source code') }
-      %li
-        = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do
-          %span=  _('Download zip')
-      %li
-        = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'tar.gz'), rel: 'nofollow', download: '' do
-          %span= _('Download tar.gz')
-      %li
-        = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'tar.bz2'), rel: 'nofollow', download: '' do
-          %span= _('Download tar.bz2')
-      %li
-        = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'tar'), rel: 'nofollow', download: '' do
-          %span= _('Download tar')
-
+      %li.dropdown-bold-header= _('Download source code')
+      %li.dropdown-menu-content
+        = render 'projects/buttons/download_links', project: project, ref: ref, archive_prefix: archive_prefix, path: nil
+      - if directory?
+        %li.separator
+        %li.dropdown-bold-header= _('Download this directory')
+        %li.dropdown-menu-content
+          = render 'projects/buttons/download_links', project: project, ref: ref, archive_prefix: archive_prefix, path: @path
       - if pipeline && pipeline.latest_builds_with_artifacts.any?
-        %li.dropdown-header Artifacts
+        %li.separator
+        %li.dropdown-bold-header= _('Download artifacts')
         - unless pipeline.latest?
-          - latest_pipeline = project.pipeline_for(ref)
-          %li
-            .unclickable= ci_status_for_statuseable(latest_pipeline)
-          %li.dropdown-header Previous Artifacts
+          %span.unclickable= ci_status_for_statuseable(project.pipeline_for(ref))
+        %li.dropdown-header= _('Previous Artifacts')
         - pipeline.latest_builds_with_artifacts.each do |job|
           %li
-            = link_to latest_succeeded_project_artifacts_path(project, "#{ref}/download", job: job.name), rel: 'nofollow', download: '' do
-              %span
-                #{s_('DownloadArtifacts|Download')} '#{job.name}'
+            = link_to job.name, latest_succeeded_project_artifacts_path(project, "#{ref}/download", job: job.name), rel: 'nofollow', download: ''

app/views/projects/buttons/_download_links.html.haml

@@ -0,0 +1,9 @@
+%ul
+  %li.d-inline-block.m-0.p-0
+    = link_to 'zip', project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: 'zip'), rel: 'nofollow', download: '', class: 'btn btn-primary btn-xs'
+  %li.d-inline-block.m-0.p-0
+    = link_to 'tar.gz', project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: 'tar.gz'), rel: 'nofollow', download: '', class: 'btn btn-xs'
+  %li.d-inline-block.m-0.p-0
+    = link_to 'tar.bz2', project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: 'tar.bz2'), rel: 'nofollow', download: '', class: 'btn btn-xs'
+  %li.d-inline-block.m-0.p-0
+    = link_to 'tar', project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: 'tar'), rel: 'nofollow', download: '', class: 'btn btn-xs'

changelogs/unreleased/24704-download-repository-path.yml

@@ -0,0 +1,5 @@
+---
+title: Download a folder from repository
+merge_request: 26532
+author: kiameisomabes
+type: added

doc/user/project/repository/index.md

@@ -241,4 +241,24 @@ Projects that contain a `.xcodeproj` or `.xcworkspace` directory can now be clon
 in Xcode using the new **Open in Xcode** button, located next to the Git URL
 used for cloning your project. The button is only shown on macOS.
 
+## Download Source Code
+
+Source code stored in the repository can be downloaded.
+
+By clicking the download icon, a dropdown will open with links to download the following:
+
+![Download source code](img/download_source_code.png)
+
+- **Source Code:**
+  This allows users to download the source code on branch they're currently
+  viewing. Available zip, tar, tar.gz and tar.bz2.
+- **Directory:**
+  > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/24704) in GitLab 11.10
+
+  Only shows up when viewing a sub-directory. This allows users to download
+  the specific directory they're currently viewing. Also available in zip, tar,
+  tar.gz and tar.bz2.
+- **Artifacts:**
+  This allows users to download the artifacts of the latest CI build.
+
 [jupyter]: https://jupyter.org

lib/gitlab/git/repository.rb

@@ -231,12 +231,12 @@ module Gitlab
         end
       end
 
-      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:)
+      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil)
         ref ||= root_ref
         commit = Gitlab::Git::Commit.find(self, ref)
         return {} if commit.nil?
 
-        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha)
+        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha, path: path)
 
         {
           'ArchivePrefix' => prefix,
@@ -248,13 +248,14 @@ module Gitlab
 
       # This is both the filename of the archive (missing the extension) and the
       # name of the top-level member of the archive under which all files go
-      def archive_prefix(ref, sha, project_path, append_sha:)
+      def archive_prefix(ref, sha, project_path, append_sha:, path:)
         append_sha = (ref != sha) if append_sha.nil?
 
         formatted_ref = ref.tr('/', '-')
 
         prefix_segments = [project_path, formatted_ref]
         prefix_segments << sha if append_sha
+        prefix_segments << path.tr('/', '-').gsub(%r{^/|/$}, '') if path
 
         prefix_segments.join('-')
       end

lib/gitlab/workhorse.rb

@@ -63,13 +63,26 @@ module Gitlab
         ]
       end
 
-      def send_git_archive(repository, ref:, format:, append_sha:)
+      def send_git_archive(repository, ref:, format:, append_sha:, path: nil)
         format ||= 'tar.gz'
         format = format.downcase
-        params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha)
-        raise "Repository or ref not found" if params.empty?
+        metadata = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha, path: path)
 
-        params['GitalyServer'] = gitaly_server_hash(repository)
+        raise "Repository or ref not found" if metadata.empty?
+
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'ArchivePath' => metadata['ArchivePath'],
+          'GetArchiveRequest' => encode_binary(
+            Gitaly::GetArchiveRequest.new(
+              repository: repository.gitaly_repository,
+              commit_id: metadata['CommitId'],
+              prefix: metadata['ArchivePrefix'],
+              format: archive_format(format),
+              path: path.presence || ""
+            ).to_proto
+          )
+        }
 
         # If present DisableCache must be a Boolean. Otherwise workhorse ignores it.
         params['DisableCache'] = true if git_archive_cache_disabled?
@@ -220,6 +233,10 @@ module Gitlab
         Base64.urlsafe_encode64(JSON.dump(hash))
       end
 
+      def encode_binary(binary)
+        Base64.urlsafe_encode64(binary)
+      end
+
       def gitaly_server_hash(repository)
         {
           address: Gitlab::GitalyClient.address(repository.project.repository_storage),
@@ -238,6 +255,19 @@ module Gitlab
       def git_archive_cache_disabled?
         ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled)
       end
+
+      def archive_format(format)
+        case format
+        when "tar.bz2", "tbz", "tbz2", "tb2", "bz2"
+          Gitaly::GetArchiveRequest::Format::TAR_BZ2
+        when "tar"
+          Gitaly::GetArchiveRequest::Format::TAR
+        when "zip"
+          Gitaly::GetArchiveRequest::Format::ZIP
+        else
+          Gitaly::GetArchiveRequest::Format::TAR_GZ
+        end
+      end
     end
   end
 end

locale/gitlab.pot

@@ -3013,19 +3013,10 @@ msgstr ""
 msgid "Download asset"
 msgstr ""
 
-msgid "Download tar"
+msgid "Download source code"
 msgstr ""
 
-msgid "Download tar.bz2"
-msgstr ""
-
-msgid "Download tar.gz"
-msgstr ""
-
-msgid "Download zip"
-msgstr ""
-
-msgid "DownloadArtifacts|Download"
+msgid "Download this directory"
 msgstr ""
 
 msgid "DownloadCommit|Email Patches"
@@ -6028,6 +6019,9 @@ msgstr ""
 msgid "Preview payload"
 msgstr ""
 
+msgid "Previous Artifacts"
+msgstr ""
+
 msgid "Prioritize"
 msgstr ""
 

spec/features/projects/branches/download_buttons_spec.rb

@@ -35,7 +35,7 @@ describe 'Download buttons in branches page' do
       it 'shows download artifacts button' do
         href = latest_succeeded_project_artifacts_path(project, 'binary-encoding/download', job: 'build')
 
-        expect(page).to have_link "Download '#{build.name}'", href: href
+        expect(page).to have_link build.name, href: href
       end
     end
   end

spec/features/projects/files/download_buttons_spec.rb

@@ -30,7 +30,7 @@ describe 'Projects > Files > Download buttons in files tree' do
     it 'shows download artifacts button' do
       href = latest_succeeded_project_artifacts_path(project, "#{project.default_branch}/download", job: 'build')
 
-      expect(page).to have_link "Download '#{build.name}'", href: href
+      expect(page).to have_link build.name, href: href
     end
   end
 end

spec/features/projects/show/download_buttons_spec.rb

@@ -35,11 +35,10 @@ describe 'Projects > Show > Download buttons' do
       it 'shows download artifacts button' do
         href = latest_succeeded_project_artifacts_path(project, "#{project.default_branch}/download", job: 'build')
 
-        expect(page).to have_link "Download '#{build.name}'", href: href
+        expect(page).to have_link build.name, href: href
       end
 
       it 'download links have download attribute' do
-        expect(page).to have_selector('a', text: 'Download')
         page.all('a', text: 'Download').each do |link|
           expect(link[:download]).to eq ''
         end

spec/features/projects/tags/download_buttons_spec.rb

@@ -36,7 +36,7 @@ describe 'Download buttons in tags page' do
       it 'shows download artifacts button' do
         href = latest_succeeded_project_artifacts_path(project, "#{tag}/download", job: 'build')
 
-        expect(page).to have_link "Download '#{build.name}'", href: href
+        expect(page).to have_link build.name, href: href
       end
     end
   end

spec/lib/gitlab/git/repository_spec.rb

@@ -152,13 +152,14 @@ describe Gitlab::Git::Repository, :seed_helper do
     let(:append_sha) { true }
     let(:ref) { 'master' }
     let(:format) { nil }
+    let(:path) { nil }
 
     let(:expected_extension) { 'tar.gz' }
     let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" }
     let(:expected_path) { File.join(storage_path, cache_key, expected_filename) }
     let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" }
 
-    subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha) }
+    subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha, path: path) }
 
     it 'sets CommitId to the commit SHA' do
       expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID)
@@ -176,6 +177,14 @@ describe Gitlab::Git::Repository, :seed_helper do
       expect(metadata['ArchivePath']).to eq(expected_path)
     end
 
+    context 'path is set' do
+      let(:path) { 'foo/bar' }
+
+      it 'appends the path to the prefix' do
+        expect(metadata['ArchivePrefix']).to eq("#{expected_prefix}-foo-bar")
+      end
+    end
+
     context 'append_sha varies archive path and filename' do
       where(:append_sha, :ref, :expected_prefix) do
         sha = SeedRepo::LastCommit::ID

spec/lib/gitlab/workhorse_spec.rb

@@ -16,20 +16,12 @@ describe Gitlab::Workhorse do
     let(:ref) { 'master' }
     let(:format) { 'zip' }
     let(:storage_path) { Gitlab.config.gitlab.repository_downloads_path }
-    let(:base_params) { repository.archive_metadata(ref, storage_path, format, append_sha: nil) }
-    let(:gitaly_params) do
-      base_params.merge(
-        'GitalyServer' => {
-          'address' => Gitlab::GitalyClient.address(project.repository_storage),
-          'token' => Gitlab::GitalyClient.token(project.repository_storage)
-        },
-        'GitalyRepository' => repository.gitaly_repository.to_h.deep_stringify_keys
-      )
-    end
+    let(:path) { 'some/path' }
+    let(:metadata) { repository.archive_metadata(ref, storage_path, format, append_sha: nil, path: path) }
     let(:cache_disabled) { false }
 
     subject do
-      described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil)
+      described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil, path: path)
     end
 
     before do
@@ -41,7 +33,22 @@ describe Gitlab::Workhorse do
 
       expect(key).to eq('Gitlab-Workhorse-Send-Data')
       expect(command).to eq('git-archive')
-      expect(params).to include(gitaly_params)
+      expect(params).to eq({
+        'GitalyServer' => {
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        },
+        'ArchivePath' => metadata['ArchivePath'],
+        'GetArchiveRequest' => Base64.urlsafe_encode64(
+          Gitaly::GetArchiveRequest.new(
+            repository: repository.gitaly_repository,
+            commit_id: metadata['CommitId'],
+            prefix: metadata['ArchivePrefix'],
+            format: Gitaly::GetArchiveRequest::Format::ZIP,
+            path: path
+          ).to_proto
+        )
+      }.deep_stringify_keys)
     end
 
     context 'when archive caching is disabled' do