Merge branch 'feature/gb/download-single-job-artifact-using-api' into 'master'

Add API endpoint for downloading a single job artifact

Closes #37196

See merge request !14027
This commit is contained in:
Kamil Trzciński 2017-09-06 15:58:26 +00:00
commit 29a34b3c28
13 changed files with 477 additions and 218 deletions

View File

@ -7,7 +7,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
before_action :authorize_update_build!, only: [:keep] before_action :authorize_update_build!, only: [:keep]
before_action :extract_ref_name_and_path before_action :extract_ref_name_and_path
before_action :validate_artifacts! before_action :validate_artifacts!
before_action :set_path_and_entry, only: [:file, :raw] before_action :entry, only: [:file]
def download def download
if artifacts_file.file_storage? if artifacts_file.file_storage?
@ -41,7 +41,10 @@ class Projects::ArtifactsController < Projects::ApplicationController
end end
def raw def raw
send_artifacts_entry(build, @entry) path = Gitlab::Ci::Build::Artifacts::Path
.new(params[:path])
send_artifacts_entry(build, path)
end end
def keep def keep
@ -93,9 +96,8 @@ class Projects::ArtifactsController < Projects::ApplicationController
@artifacts_file ||= build.artifacts_file @artifacts_file ||= build.artifacts_file
end end
def set_path_and_entry def entry
@path = params[:path] @entry = build.artifacts_metadata_entry(params[:path])
@entry = build.artifacts_metadata_entry(@path)
render_404 unless @entry.exists? render_404 unless @entry.exists?
end end

View File

@ -0,0 +1,5 @@
---
title: Make it possible to download a single job artifact file using the API
merge_request: 14027
author:
type: added

View File

@ -320,11 +320,11 @@ Response:
[ce-2893]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2893 [ce-2893]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2893
## Download the artifacts file ## Download the artifacts archive
> [Introduced][ce-5347] in GitLab 8.10. > [Introduced][ce-5347] in GitLab 8.10.
Download the artifacts file from the given reference name and job provided the Download the artifacts archive from the given reference name and job provided the
job finished successfully. job finished successfully.
``` ```
@ -354,6 +354,40 @@ Example response:
[ce-5347]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5347 [ce-5347]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5347
## Download a single artifact file
> Introduced in GitLab 10.0
Download a single artifact file from within the job's artifacts archive.
Only a single file is going to be extracted from the archive and streamed to a client.
```
GET /projects/:id/jobs/:job_id/artifacts/*artifact_path
```
Parameters
| Attribute | Type | Required | Description |
|-----------------|---------|----------|-------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
| `job_id ` | integer | yes | The unique job identifier |
| `artifact_path` | string | yes | Path to a file inside the artifacts archive |
Example request:
```
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/5/artifacts/some/release/file.pdf"
```
Example response:
| Status | Description |
|-----------|--------------------------------------|
| 200 | Sends a single artifact file |
| 400 | Invalid path provided |
| 404 | Build not found or no file/artifacts |
## Get a trace file ## Get a trace file
Get a trace of a specific job of a project Get a trace of a specific job of a project

View File

@ -108,6 +108,7 @@ module API
mount ::API::Internal mount ::API::Internal
mount ::API::Issues mount ::API::Issues
mount ::API::Jobs mount ::API::Jobs
mount ::API::JobArtifacts
mount ::API::Keys mount ::API::Keys
mount ::API::Labels mount ::API::Labels
mount ::API::Lint mount ::API::Lint

View File

@ -128,6 +128,10 @@ module API
merge_request merge_request
end end
def find_build!(id)
user_project.builds.find(id.to_i)
end
def authenticate! def authenticate!
unauthorized! unless current_user && can?(initial_current_user, :access_api) unauthorized! unless current_user && can?(initial_current_user, :access_api)
end end
@ -160,6 +164,14 @@ module API
authorize! :admin_project, user_project authorize! :admin_project, user_project
end end
def authorize_read_builds!
authorize! :read_build, user_project
end
def authorize_update_builds!
authorize! :update_build, user_project
end
def require_gitlab_workhorse! def require_gitlab_workhorse!
unless env['HTTP_GITLAB_WORKHORSE'].present? unless env['HTTP_GITLAB_WORKHORSE'].present?
forbidden!('Request should be executed via GitLab Workhorse') forbidden!('Request should be executed via GitLab Workhorse')
@ -210,7 +222,7 @@ module API
def bad_request!(attribute) def bad_request!(attribute)
message = ["400 (Bad request)"] message = ["400 (Bad request)"]
message << "\"" + attribute.to_s + "\" not given" message << "\"" + attribute.to_s + "\" not given" if attribute
render_api_error!(message.join(' '), 400) render_api_error!(message.join(' '), 400)
end end
@ -432,6 +444,10 @@ module API
header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
end end
def send_artifacts_entry(build, entry)
header(*Gitlab::Workhorse.send_artifacts_entry(build, entry))
end
# The Grape Error Middleware only has access to env but no params. We workaround this by # The Grape Error Middleware only has access to env but no params. We workaround this by
# defining a method that returns the right value. # defining a method that returns the right value.
def define_params_for_grape_middleware def define_params_for_grape_middleware

80
lib/api/job_artifacts.rb Normal file
View File

@ -0,0 +1,80 @@
module API
class JobArtifacts < Grape::API
before { authenticate_non_get! }
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
desc 'Download the artifacts file from a job' do
detail 'This feature was introduced in GitLab 8.10'
end
params do
requires :ref_name, type: String, desc: 'The ref from repository'
requires :job, type: String, desc: 'The name for the job'
end
get ':id/jobs/artifacts/:ref_name/download',
requirements: { ref_name: /.+/ } do
authorize_read_builds!
builds = user_project.latest_successful_builds_for(params[:ref_name])
latest_build = builds.find_by!(name: params[:job])
present_artifacts!(latest_build.artifacts_file)
end
desc 'Download the artifacts file from a job' do
detail 'This feature was introduced in GitLab 8.5'
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
get ':id/jobs/:job_id/artifacts' do
authorize_read_builds!
build = find_build!(params[:job_id])
present_artifacts!(build.artifacts_file)
end
desc 'Download a specific file from artifacts archive' do
detail 'This feature was introduced in GitLab 10.0'
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
requires :artifact_path, type: String, desc: 'Artifact path'
end
get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do
authorize_read_builds!
build = find_build!(params[:job_id])
not_found! unless build.artifacts?
path = Gitlab::Ci::Build::Artifacts::Path
.new(params[:artifact_path])
bad_request! unless path.valid?
send_artifacts_entry(build, path)
end
desc 'Keep the artifacts to prevent them from being deleted' do
success Entities::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
post ':id/jobs/:job_id/artifacts/keep' do
authorize_update_builds!
build = find_build!(params[:job_id])
authorize!(:update_build, build)
return not_found!(build) unless build.artifacts?
build.keep_artifacts!
status 200
present build, with: Entities::Job
end
end
end
end

View File

@ -66,42 +66,11 @@ module API
get ':id/jobs/:job_id' do get ':id/jobs/:job_id' do
authorize_read_builds! authorize_read_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
present build, with: Entities::Job present build, with: Entities::Job
end end
desc 'Download the artifacts file from a job' do
detail 'This feature was introduced in GitLab 8.5'
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
get ':id/jobs/:job_id/artifacts' do
authorize_read_builds!
build = get_build!(params[:job_id])
present_artifacts!(build.artifacts_file)
end
desc 'Download the artifacts file from a job' do
detail 'This feature was introduced in GitLab 8.10'
end
params do
requires :ref_name, type: String, desc: 'The ref from repository'
requires :job, type: String, desc: 'The name for the job'
end
get ':id/jobs/artifacts/:ref_name/download',
requirements: { ref_name: /.+/ } do
authorize_read_builds!
builds = user_project.latest_successful_builds_for(params[:ref_name])
latest_build = builds.find_by!(name: params[:job])
present_artifacts!(latest_build.artifacts_file)
end
# TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
# is saved in the DB instead of file). But before that, we need to consider how to replace the value of # is saved in the DB instead of file). But before that, we need to consider how to replace the value of
# `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
@ -112,7 +81,7 @@ module API
get ':id/jobs/:job_id/trace' do get ':id/jobs/:job_id/trace' do
authorize_read_builds! authorize_read_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
content_type 'text/plain' content_type 'text/plain'
@ -131,7 +100,7 @@ module API
post ':id/jobs/:job_id/cancel' do post ':id/jobs/:job_id/cancel' do
authorize_update_builds! authorize_update_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
authorize!(:update_build, build) authorize!(:update_build, build)
build.cancel build.cancel
@ -148,7 +117,7 @@ module API
post ':id/jobs/:job_id/retry' do post ':id/jobs/:job_id/retry' do
authorize_update_builds! authorize_update_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
authorize!(:update_build, build) authorize!(:update_build, build)
return forbidden!('Job is not retryable') unless build.retryable? return forbidden!('Job is not retryable') unless build.retryable?
@ -166,7 +135,7 @@ module API
post ':id/jobs/:job_id/erase' do post ':id/jobs/:job_id/erase' do
authorize_update_builds! authorize_update_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
authorize!(:update_build, build) authorize!(:update_build, build)
return forbidden!('Job is not erasable!') unless build.erasable? return forbidden!('Job is not erasable!') unless build.erasable?
@ -174,25 +143,6 @@ module API
present build, with: Entities::Job present build, with: Entities::Job
end end
desc 'Keep the artifacts to prevent them from being deleted' do
success Entities::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
post ':id/jobs/:job_id/artifacts/keep' do
authorize_update_builds!
build = get_build!(params[:job_id])
authorize!(:update_build, build)
return not_found!(build) unless build.artifacts?
build.keep_artifacts!
status 200
present build, with: Entities::Job
end
desc 'Trigger a manual job' do desc 'Trigger a manual job' do
success Entities::Job success Entities::Job
detail 'This feature was added in GitLab 8.11' detail 'This feature was added in GitLab 8.11'
@ -203,7 +153,7 @@ module API
post ":id/jobs/:job_id/play" do post ":id/jobs/:job_id/play" do
authorize_read_builds! authorize_read_builds!
build = get_build!(params[:job_id]) build = find_build!(params[:job_id])
authorize!(:update_build, build) authorize!(:update_build, build)
bad_request!("Unplayable Job") unless build.playable? bad_request!("Unplayable Job") unless build.playable?
@ -216,14 +166,6 @@ module API
end end
helpers do helpers do
def find_build(id)
user_project.builds.find_by(id: id.to_i)
end
def get_build!(id)
find_build(id) || not_found!
end
def filter_builds(builds, scope) def filter_builds(builds, scope)
return builds if scope.nil? || scope.empty? return builds if scope.nil? || scope.empty?
@ -234,14 +176,6 @@ module API
builds.where(status: available_statuses && scope) builds.where(status: available_statuses && scope)
end end
def authorize_read_builds!
authorize! :read_build, user_project
end
def authorize_update_builds!
authorize! :update_build, user_project
end
end end
end end
end end

View File

@ -1,128 +1,128 @@
module Gitlab module Gitlab
module Ci::Build::Artifacts module Ci
class Metadata module Build
## module Artifacts
# Class that represents an entry (path and metadata) to a file or class Metadata
# directory in GitLab CI Build Artifacts binary file / archive ##
# # Class that represents an entry (path and metadata) to a file or
# This is IO-operations safe class, that does similar job to # directory in GitLab CI Build Artifacts binary file / archive
# Ruby's Pathname but without the risk of accessing filesystem. #
# # This is IO-operations safe class, that does similar job to
# This class is working only with UTF-8 encoded paths. # Ruby's Pathname but without the risk of accessing filesystem.
# #
class Entry # This class is working only with UTF-8 encoded paths.
attr_reader :path, :entries #
attr_accessor :name class Entry
attr_reader :entries
attr_accessor :name
def initialize(path, entries) def initialize(path, entries)
@path = path.dup.force_encoding('UTF-8') @entries = entries
@entries = entries @path = Artifacts::Path.new(path)
end
if path.include?("\0") delegate :empty?, to: :children
raise ArgumentError, 'Path contains zero byte character!'
def directory?
blank_node? || @path.directory?
end
def file?
!directory?
end
def blob
return unless file?
@blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
end
def has_parent?
nodes > 0
end
def parent
return nil unless has_parent?
self.class.new(@path.to_s.chomp(basename), @entries)
end
def basename
(directory? && !blank_node?) ? name + '/' : name
end
def name
@name || @path.name
end
def children
return [] unless directory?
return @children if @children
child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$}
@children = select_entries { |path| path =~ child_pattern }
end
def directories(opts = {})
return [] unless directory?
dirs = children.select(&:directory?)
return dirs unless has_parent? && opts[:parent]
dotted_parent = parent
dotted_parent.name = '..'
dirs.prepend(dotted_parent)
end
def files
return [] unless directory?
children.select(&:file?)
end
def metadata
@entries[@path.to_s] || {}
end
def nodes
@path.nodes + (file? ? 1 : 0)
end
def blank_node?
@path.to_s.empty? # "" is considered to be './'
end
def exists?
blank_node? || @entries.include?(@path.to_s)
end
def total_size
descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}}
entries.sum do |path, entry|
(entry[:size] if path =~ descendant_pattern).to_i
end
end
def path
@path.to_s
end
def to_s
@path.to_s
end
def ==(other)
path == other.path && @entries == other.entries
end
def inspect
"#{self.class.name}: #{self}"
end
private
def select_entries
selected = @entries.select { |path, _metadata| yield path }
selected.map { |path, _metadata| self.class.new(path, @entries) }
end
end end
unless path.valid_encoding?
raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
end
end
delegate :empty?, to: :children
def directory?
blank_node? || @path.end_with?('/')
end
def file?
!directory?
end
def blob
return unless file?
@blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
end
def has_parent?
nodes > 0
end
def parent
return nil unless has_parent?
self.class.new(@path.chomp(basename), @entries)
end
def basename
(directory? && !blank_node?) ? name + '/' : name
end
def name
@name || @path.split('/').last.to_s
end
def children
return [] unless directory?
return @children if @children
child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
@children = select_entries { |path| path =~ child_pattern }
end
def directories(opts = {})
return [] unless directory?
dirs = children.select(&:directory?)
return dirs unless has_parent? && opts[:parent]
dotted_parent = parent
dotted_parent.name = '..'
dirs.prepend(dotted_parent)
end
def files
return [] unless directory?
children.select(&:file?)
end
def metadata
@entries[@path] || {}
end
def nodes
@path.count('/') + (file? ? 1 : 0)
end
def blank_node?
@path.empty? # "" is considered to be './'
end
def exists?
blank_node? || @entries.include?(@path)
end
def total_size
descendant_pattern = %r{^#{Regexp.escape(@path)}}
entries.sum do |path, entry|
(entry[:size] if path =~ descendant_pattern).to_i
end
end
def to_s
@path
end
def ==(other)
@path == other.path && @entries == other.entries
end
def inspect
"#{self.class.name}: #{@path}"
end
private
def select_entries
selected = @entries.select { |path, _metadata| yield path }
selected.map { |path, _metadata| self.class.new(path, @entries) }
end end
end end
end end

View File

@ -0,0 +1,51 @@
module Gitlab
module Ci
module Build
module Artifacts
class Path
def initialize(path)
@path = path.dup.force_encoding('UTF-8')
end
def valid?
nonzero? && utf8?
end
def directory?
@path.end_with?('/')
end
def name
@path.split('/').last.to_s
end
def nodes
@path.count('/')
end
def to_s
@path.tap do |path|
unless nonzero?
raise ArgumentError, 'Path contains zero byte character!'
end
unless utf8?
raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
end
end
end
private
def nonzero?
@path.exclude?("\0")
end
def utf8?
@path.valid_encoding?
end
end
end
end
end
end

View File

@ -121,10 +121,10 @@ module Gitlab
] ]
end end
def send_artifacts_entry(build, entry) def send_artifacts_entry(build, path)
params = { params = {
'Archive' => build.artifacts_file.path, 'Archive' => build.artifacts_file.path,
'Entry' => Base64.encode64(entry.path) 'Entry' => Base64.encode64(path.to_s)
} }
[ [

View File

@ -81,14 +81,6 @@ describe Projects::ArtifactsController do
expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt')) expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt'))
end end
end end
context 'when the file does not exist' do
it 'responds Not Found' do
get :raw, namespace_id: project.namespace, project_id: project, job_id: job, path: 'unknown'
expect(response).to be_not_found
end
end
end end
describe 'GET latest_succeeded' do describe 'GET latest_succeeded' do

View File

@ -0,0 +1,64 @@
require 'spec_helper'
describe Gitlab::Ci::Build::Artifacts::Path do
describe '#valid?' do
context 'when path contains a zero character' do
it 'is not valid' do
expect(described_class.new("something/\255")).not_to be_valid
end
end
context 'when path is not utf8 string' do
it 'is not valid' do
expect(described_class.new("something/\0")).not_to be_valid
end
end
context 'when path is valid' do
it 'is valid' do
expect(described_class.new("some/file/path")).to be_valid
end
end
end
describe '#directory?' do
context 'when path ends with a directory indicator' do
it 'is a directory' do
expect(described_class.new("some/file/dir/")).to be_directory
end
end
context 'when path does not end with a directory indicator' do
it 'is not a directory' do
expect(described_class.new("some/file")).not_to be_directory
end
end
end
describe '#name' do
it 'returns a base name' do
expect(described_class.new("some/file").name).to eq 'file'
end
end
describe '#nodes' do
it 'returns number of path nodes' do
expect(described_class.new("some/dir/file").nodes).to eq 2
end
end
describe '#to_s' do
context 'when path is valid' do
it 'returns a string representation of a path' do
expect(described_class.new('some/path').to_s).to eq 'some/path'
end
end
context 'when path is invalid' do
it 'raises an error' do
expect { described_class.new("invalid/\0").to_s }
.to raise_error ArgumentError
end
end
end
end

View File

@ -1,11 +1,11 @@
require 'spec_helper' require 'spec_helper'
describe API::Jobs do describe API::Jobs do
let!(:project) do set(:project) do
create(:project, :repository, public_builds: false) create(:project, :repository, public_builds: false)
end end
let!(:pipeline) do set(:pipeline) do
create(:ci_empty_pipeline, project: project, create(:ci_empty_pipeline, project: project,
sha: project.commit.id, sha: project.commit.id,
ref: project.default_branch) ref: project.default_branch)
@ -188,6 +188,84 @@ describe API::Jobs do
end end
end end
describe 'GET /projects/:id/jobs/:job_id/artifacts/:artifact_path' do
context 'when job has artifacts' do
let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
let(:artifact) do
'other_artifacts_0.1.2/another-subdirectory/banana_sample.gif'
end
context 'when user is anonymous' do
let(:api_user) { nil }
context 'when project is public' do
it 'allows to access artifacts' do
project.update_column(:visibility_level,
Gitlab::VisibilityLevel::PUBLIC)
project.update_column(:public_builds, true)
get_artifact_file(artifact)
expect(response).to have_http_status(200)
end
end
context 'when project is public with builds access disabled' do
it 'rejects access to artifacts' do
project.update_column(:visibility_level,
Gitlab::VisibilityLevel::PUBLIC)
project.update_column(:public_builds, false)
get_artifact_file(artifact)
expect(response).to have_http_status(403)
end
end
context 'when project is private' do
it 'rejects access and hides existence of artifacts' do
project.update_column(:visibility_level,
Gitlab::VisibilityLevel::PRIVATE)
project.update_column(:public_builds, true)
get_artifact_file(artifact)
expect(response).to have_http_status(404)
end
end
end
context 'when user is authorized' do
it 'returns a specific artifact file for a valid path' do
expect(Gitlab::Workhorse)
.to receive(:send_artifacts_entry)
.and_call_original
get_artifact_file(artifact)
expect(response).to have_http_status(200)
expect(response.headers)
.to include('Content-Type' => 'application/json',
'Gitlab-Workhorse-Send-Data' => /artifacts-entry/)
end
end
end
context 'when job does not have artifacts' do
it 'does not return job artifact file' do
get_artifact_file('some/artifact')
expect(response).to have_http_status(404)
end
end
def get_artifact_file(artifact_path)
get api("/projects/#{project.id}/jobs/#{job.id}/" \
"artifacts/#{artifact_path}", api_user)
end
end
describe 'GET /projects/:id/jobs/:job_id/artifacts' do describe 'GET /projects/:id/jobs/:job_id/artifacts' do
before do before do
get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
@ -209,11 +287,12 @@ describe API::Jobs do
end end
end end
context 'unauthorized user' do context 'when anonymous user is accessing private artifacts' do
let(:api_user) { nil } let(:api_user) { nil }
it 'does not return specific job artifacts' do it 'hides artifacts and rejects request' do
expect(response).to have_http_status(401) expect(project).to be_private
expect(response).to have_http_status(404)
end end
end end
end end
@ -242,8 +321,9 @@ describe API::Jobs do
get_for_ref get_for_ref
end end
it 'gives 401' do it 'does not find a resource in a private project' do
expect(response).to have_http_status(401) expect(project).to be_private
expect(response).to have_http_status(404)
end end
end end