diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index f637a9a803b..eb010923466 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -7,7 +7,7 @@ class Projects::ArtifactsController < Projects::ApplicationController before_action :authorize_update_build!, only: [:keep] before_action :extract_ref_name_and_path before_action :validate_artifacts! - before_action :set_path_and_entry, only: [:file, :raw] + before_action :entry, only: [:file] def download if artifacts_file.file_storage? @@ -41,7 +41,10 @@ class Projects::ArtifactsController < Projects::ApplicationController end def raw - send_artifacts_entry(build, @entry) + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:path]) + + send_artifacts_entry(build, path) end def keep @@ -93,9 +96,8 @@ class Projects::ArtifactsController < Projects::ApplicationController @artifacts_file ||= build.artifacts_file end - def set_path_and_entry - @path = params[:path] - @entry = build.artifacts_metadata_entry(@path) + def entry + @entry = build.artifacts_metadata_entry(params[:path]) render_404 unless @entry.exists? end diff --git a/changelogs/unreleased/feature-gb-download-single-job-artifact-using-api.yml b/changelogs/unreleased/feature-gb-download-single-job-artifact-using-api.yml new file mode 100644 index 00000000000..920679ca166 --- /dev/null +++ b/changelogs/unreleased/feature-gb-download-single-job-artifact-using-api.yml @@ -0,0 +1,5 @@ +--- +title: Make it possible to download a single job artifact file using the API +merge_request: 14027 +author: +type: added diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 297115e94ac..d60c7c12881 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -320,11 +320,11 @@ Response: [ce-2893]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2893 -## Download the artifacts file +## Download the artifacts archive > [Introduced][ce-5347] in GitLab 8.10. -Download the artifacts file from the given reference name and job provided the +Download the artifacts archive from the given reference name and job provided the job finished successfully. ``` @@ -354,6 +354,40 @@ Example response: [ce-5347]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5347 +## Download a single artifact file + +> Introduced in GitLab 10.0 + +Download a single artifact file from within the job's artifacts archive. + +Only a single file is going to be extracted from the archive and streamed to a client. + +``` +GET /projects/:id/jobs/:job_id/artifacts/*artifact_path +``` + +Parameters + +| Attribute | Type | Required | Description | +|-----------------|---------|----------|-------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `job_id ` | integer | yes | The unique job identifier | +| `artifact_path` | string | yes | Path to a file inside the artifacts archive | + +Example request: + +``` +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/5/artifacts/some/release/file.pdf" +``` + +Example response: + +| Status | Description | +|-----------|--------------------------------------| +| 200 | Sends a single artifact file | +| 400 | Invalid path provided | +| 404 | Build not found or no file/artifacts | + ## Get a trace file Get a trace of a specific job of a project diff --git a/lib/api/api.rb b/lib/api/api.rb index 94df543853b..1405a5d0f0e 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -108,6 +108,7 @@ module API mount ::API::Internal mount ::API::Issues mount ::API::Jobs + mount ::API::JobArtifacts mount ::API::Keys mount ::API::Labels mount ::API::Lint diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3d377fdb9eb..e646c63467a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -128,6 +128,10 @@ module API merge_request end + def find_build!(id) + user_project.builds.find(id.to_i) + end + def authenticate! unauthorized! unless current_user && can?(initial_current_user, :access_api) end @@ -160,6 +164,14 @@ module API authorize! :admin_project, user_project end + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + def require_gitlab_workhorse! unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') @@ -210,7 +222,7 @@ module API def bad_request!(attribute) message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" + message << "\"" + attribute.to_s + "\" not given" if attribute render_api_error!(message.join(' '), 400) end @@ -432,6 +444,10 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end + def send_artifacts_entry(build, entry) + header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) + end + # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb new file mode 100644 index 00000000000..2a8fa7659bf --- /dev/null +++ b/lib/api/job_artifacts.rb @@ -0,0 +1,80 @@ +module API + class JobArtifacts < Grape::API + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + end + get ':id/jobs/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/artifacts' do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = find_build!(params[:job_id]) + not_found! unless build.artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + bad_request! unless path.valid? + + send_artifacts_entry(build, path) + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/artifacts/keep' do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Job + end + end + end +end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 5bab96398fd..3c1c412ba42 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -66,42 +66,11 @@ module API get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) present build, with: Entities::Job end - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:job_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - end - get ':id/jobs/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_read_builds! - - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) - - present_artifacts!(latest_build.artifacts_file) - end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. @@ -112,7 +81,7 @@ module API get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -131,7 +100,7 @@ module API post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) build.cancel @@ -148,7 +117,7 @@ module API post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? @@ -166,7 +135,7 @@ module API post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? @@ -174,25 +143,6 @@ module API present build, with: Entities::Job end - desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/artifacts/keep' do - authorize_update_builds! - - build = get_build!(params[:job_id]) - authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: Entities::Job - end - desc 'Trigger a manual job' do success Entities::Job detail 'This feature was added in GitLab 8.11' @@ -203,7 +153,7 @@ module API post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? @@ -216,14 +166,6 @@ module API end helpers do - def find_build(id) - user_project.builds.find_by(id: id.to_i) - end - - def get_build!(id) - find_build(id) || not_found! - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -234,14 +176,6 @@ module API builds.where(status: available_statuses && scope) end - - def authorize_read_builds! - authorize! :read_build, user_project - end - - def authorize_update_builds! - authorize! :update_build, user_project - end end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2e073334abc..22941d48edf 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,128 +1,128 @@ module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents an entry (path and metadata) to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Entry - attr_reader :path, :entries - attr_accessor :name + module Ci + module Build + module Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :entries + attr_accessor :name - def initialize(path, entries) - @path = path.dup.force_encoding('UTF-8') - @entries = entries + def initialize(path, entries) + @entries = entries + @path = Artifacts::Path.new(path) + end - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' + delegate :empty?, to: :children + + def directory? + blank_node? || @path.directory? + end + + def file? + !directory? + end + + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.name + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path.to_s] || {} + end + + def nodes + @path.nodes + (file? ? 1 : 0) + end + + def blank_node? + @path.to_s.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path.to_s) + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def path + @path.to_s + end + + def to_s + @path.to_s + end + + def ==(other) + path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{self}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end end - - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' - end - end - - delegate :empty?, to: :children - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def blob - return unless file? - - @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - self.class.new(@path.chomp(basename), @entries) - end - - def basename - (directory? && !blank_node?) ? name + '/' : name - end - - def name - @name || @path.split('/').last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |path| path =~ child_pattern } - end - - def directories(opts = {}) - return [] unless directory? - dirs = children.select(&:directory?) - return dirs unless has_parent? && opts[:parent] - - dotted_parent = parent - dotted_parent.name = '..' - dirs.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @entries[@path] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def exists? - blank_node? || @entries.include?(@path) - end - - def total_size - descendant_pattern = %r{^#{Regexp.escape(@path)}} - entries.sum do |path, entry| - (entry[:size] if path =~ descendant_pattern).to_i - end - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @entries == other.entries - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def select_entries - selected = @entries.select { |path, _metadata| yield path } - selected.map { |path, _metadata| self.class.new(path, @entries) } end end end diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb new file mode 100644 index 00000000000..9cd9b36c5f8 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -0,0 +1,51 @@ +module Gitlab + module Ci + module Build + module Artifacts + class Path + def initialize(path) + @path = path.dup.force_encoding('UTF-8') + end + + def valid? + nonzero? && utf8? + end + + def directory? + @path.end_with?('/') + end + + def name + @path.split('/').last.to_s + end + + def nodes + @path.count('/') + end + + def to_s + @path.tap do |path| + unless nonzero? + raise ArgumentError, 'Path contains zero byte character!' + end + + unless utf8? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + end + + private + + def nonzero? + @path.exclude?("\0") + end + + def utf8? + @path.valid_encoding? + end + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e5ad9b5a40c..7a94af2f8f1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -121,10 +121,10 @@ module Gitlab ] end - def send_artifacts_entry(build, entry) + def send_artifacts_entry(build, path) params = { 'Archive' => build.artifacts_file.path, - 'Entry' => Base64.encode64(entry.path) + 'Entry' => Base64.encode64(path.to_s) } [ diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb index d2c613a2423..caa63e7bd22 100644 --- a/spec/controllers/projects/artifacts_controller_spec.rb +++ b/spec/controllers/projects/artifacts_controller_spec.rb @@ -81,14 +81,6 @@ describe Projects::ArtifactsController do expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt')) end end - - context 'when the file does not exist' do - it 'responds Not Found' do - get :raw, namespace_id: project.namespace, project_id: project, job_id: job, path: 'unknown' - - expect(response).to be_not_found - end - end end describe 'GET latest_succeeded' do diff --git a/spec/lib/gitlab/ci/build/artifacts/path_spec.rb b/spec/lib/gitlab/ci/build/artifacts/path_spec.rb new file mode 100644 index 00000000000..7bd6a2ead25 --- /dev/null +++ b/spec/lib/gitlab/ci/build/artifacts/path_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe Gitlab::Ci::Build::Artifacts::Path do + describe '#valid?' do + context 'when path contains a zero character' do + it 'is not valid' do + expect(described_class.new("something/\255")).not_to be_valid + end + end + + context 'when path is not utf8 string' do + it 'is not valid' do + expect(described_class.new("something/\0")).not_to be_valid + end + end + + context 'when path is valid' do + it 'is valid' do + expect(described_class.new("some/file/path")).to be_valid + end + end + end + + describe '#directory?' do + context 'when path ends with a directory indicator' do + it 'is a directory' do + expect(described_class.new("some/file/dir/")).to be_directory + end + end + + context 'when path does not end with a directory indicator' do + it 'is not a directory' do + expect(described_class.new("some/file")).not_to be_directory + end + end + end + + describe '#name' do + it 'returns a base name' do + expect(described_class.new("some/file").name).to eq 'file' + end + end + + describe '#nodes' do + it 'returns number of path nodes' do + expect(described_class.new("some/dir/file").nodes).to eq 2 + end + end + + describe '#to_s' do + context 'when path is valid' do + it 'returns a string representation of a path' do + expect(described_class.new('some/path').to_s).to eq 'some/path' + end + end + + context 'when path is invalid' do + it 'raises an error' do + expect { described_class.new("invalid/\0").to_s } + .to raise_error ArgumentError + end + end + end +end diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb index f56baf9663d..2d7cc1a1798 100644 --- a/spec/requests/api/jobs_spec.rb +++ b/spec/requests/api/jobs_spec.rb @@ -1,11 +1,11 @@ require 'spec_helper' describe API::Jobs do - let!(:project) do + set(:project) do create(:project, :repository, public_builds: false) end - let!(:pipeline) do + set(:pipeline) do create(:ci_empty_pipeline, project: project, sha: project.commit.id, ref: project.default_branch) @@ -188,6 +188,84 @@ describe API::Jobs do end end + describe 'GET /projects/:id/jobs/:job_id/artifacts/:artifact_path' do + context 'when job has artifacts' do + let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) } + + let(:artifact) do + 'other_artifacts_0.1.2/another-subdirectory/banana_sample.gif' + end + + context 'when user is anonymous' do + let(:api_user) { nil } + + context 'when project is public' do + it 'allows to access artifacts' do + project.update_column(:visibility_level, + Gitlab::VisibilityLevel::PUBLIC) + project.update_column(:public_builds, true) + + get_artifact_file(artifact) + + expect(response).to have_http_status(200) + end + end + + context 'when project is public with builds access disabled' do + it 'rejects access to artifacts' do + project.update_column(:visibility_level, + Gitlab::VisibilityLevel::PUBLIC) + project.update_column(:public_builds, false) + + get_artifact_file(artifact) + + expect(response).to have_http_status(403) + end + end + + context 'when project is private' do + it 'rejects access and hides existence of artifacts' do + project.update_column(:visibility_level, + Gitlab::VisibilityLevel::PRIVATE) + project.update_column(:public_builds, true) + + get_artifact_file(artifact) + + expect(response).to have_http_status(404) + end + end + end + + context 'when user is authorized' do + it 'returns a specific artifact file for a valid path' do + expect(Gitlab::Workhorse) + .to receive(:send_artifacts_entry) + .and_call_original + + get_artifact_file(artifact) + + expect(response).to have_http_status(200) + expect(response.headers) + .to include('Content-Type' => 'application/json', + 'Gitlab-Workhorse-Send-Data' => /artifacts-entry/) + end + end + end + + context 'when job does not have artifacts' do + it 'does not return job artifact file' do + get_artifact_file('some/artifact') + + expect(response).to have_http_status(404) + end + end + + def get_artifact_file(artifact_path) + get api("/projects/#{project.id}/jobs/#{job.id}/" \ + "artifacts/#{artifact_path}", api_user) + end + end + describe 'GET /projects/:id/jobs/:job_id/artifacts' do before do get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) @@ -209,11 +287,12 @@ describe API::Jobs do end end - context 'unauthorized user' do + context 'when anonymous user is accessing private artifacts' do let(:api_user) { nil } - it 'does not return specific job artifacts' do - expect(response).to have_http_status(401) + it 'hides artifacts and rejects request' do + expect(project).to be_private + expect(response).to have_http_status(404) end end end @@ -242,8 +321,9 @@ describe API::Jobs do get_for_ref end - it 'gives 401' do - expect(response).to have_http_status(401) + it 'does not find a resource in a private project' do + expect(project).to be_private + expect(response).to have_http_status(404) end end