Merge branch 'artifacts' into 'master'

Implement Build Artifacts

This implements #3028 

1. It stores artifacts in shared/artifacts,
1. It adds `artifacts` to `.gitlab-ci.yml`,
1. We use GitLab Workhorse to offload artifacts uploading,
1. To download artifacts it uses GitLab Workhorse X-Sendfile extension,
1. There's one "artifact" per-build. The new upload removes previous one and creates a new one,
1. Default max artifact size is set to 100MB - this can be changed in settings.

Missing things:
1. Support for `.gitlab-ci.yml`: `artifacts: true or git-ls-files` which will upload all non tracked files,
1. Artifacts passing between builds.

GitLab Workhorse changes: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/5
GitLab Runner changes: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/merge_requests/46

Syntax:
```
artifacts:
  untracked: true # default: false
  paths: # default: empty
  - bin/files
```


See merge request !1584

.gitignore

@@ -37,6 +37,7 @@ nohup.out
 public/assets/
 public/uploads.*
 public/uploads/
+shared/artifacts/
 rails_best_practices_output.html
 /tags
 tmp/

CHANGELOG

@@ -102,6 +102,7 @@ v 8.1.0
   - Show CI status on Your projects page and Starred projects page
   - Remove "Continuous Integration" page from dashboard
   - Add notes and SSL verification entries to hook APIs (Ben Boeckel)
+  - Added build artifacts
   - Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
   - Move CI runners page to project settings area
   - Move CI variables page to project settings area

Gemfile

@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
 gem "github-linguist", "~> 4.7.0", require: "linguist"
 
 # API
-gem 'grape',        '~> 0.6.1'
+gem 'grape',        '~> 0.13.0'
 gem 'grape-entity', '~> 0.4.2'
 gem 'rack-cors',    '~> 0.4.0', require: 'rack/cors'
 

Gemfile.lock

@@ -306,10 +306,10 @@ GEM
     gon (5.0.4)
       actionpack (>= 2.3.0)
       json
-    grape (0.6.1)
+    grape (0.13.0)
       activesupport
       builder
-      hashie (>= 1.2.0)
+      hashie (>= 2.1.0)
       multi_json (>= 1.3.2)
       multi_xml (>= 0.5.2)
       rack (>= 1.3.0)
@@ -835,7 +835,7 @@ DEPENDENCIES
   gitlab_omniauth-ldap (~> 1.2.1)
   gollum-lib (~> 4.0.2)
   gon (~> 5.0.0)
-  grape (~> 0.6.1)
+  grape (~> 0.13.0)
   grape-entity (~> 0.4.2)
   haml-rails (~> 0.9.0)
   hipchat (~> 1.5.0)

app/controllers/admin/application_settings_controller.rb

@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :admin_notification_email,
       :user_oauth_applications,
       :shared_runners_enabled,
+      :max_artifacts_size,
       restricted_visibility_levels: [],
       import_sources: []
     )

app/controllers/projects/builds_controller.rb

@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
   before_action :build, except: [:index, :cancel_all]
 
   before_action :authorize_manage_builds!, except: [:index, :show, :status]
+  before_action :authorize_download_build_artifacts!, only: [:download]
 
   layout "project"
 
@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
     redirect_to build_path(build)
   end
 
+  def download
+    unless artifacts_file.file_storage?
+      return redirect_to artifacts_file.url
+    end
+
+    unless artifacts_file.exists?
+      return not_found!
+    end
+
+    send_file artifacts_file.path, disposition: 'attachment'
+  end
+
   def status
     render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
   end
@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
     @build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
   end
 
+  def artifacts_file
+    build.artifacts_file
+  end
+
   def build_path(build)
     namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
   end
@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
       return page_404
     end
   end
+
+  def authorize_download_build_artifacts!
+    unless can?(current_user, :download_build_artifacts, @project)
+      if current_user.nil?
+        return authenticate_user!
+      else
+        return render_404
+      end
+    end
+  end
 end

app/models/ability.rb

@@ -154,6 +154,7 @@ class Ability
         :create_merge_request,
         :create_wiki,
         :manage_builds,
+        :download_build_artifacts,
         :push_code
       ]
     end

app/models/application_setting.rb

@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
       restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
       import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+      max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
     )
   end
 

app/models/ci/build.rb

@@ -39,6 +39,8 @@ module Ci
     scope :ignore_failures, ->() { where(allow_failure: false) }
     scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
 
+    mount_uploader :artifacts_file, ArtifactUploader
+
     acts_as_taggable
 
     # To prevent db load megabytes of data from trace
@@ -217,6 +219,14 @@ module Ci
       "#{dir_to_trace}/#{id}.log"
     end
 
+    def token
+      project.token
+    end
+
+    def valid_token? token
+      project.valid_token? token
+    end
+
     def target_url
       Gitlab::Application.routes.url_helpers.
         namespace_project_build_url(gl_project.namespace, gl_project, self)
@@ -248,6 +258,13 @@ module Ci
       pending? && !any_runners_online?
     end
 
+    def download_url
+      if artifacts_file.exists?
+        Gitlab::Application.routes.url_helpers.
+          download_namespace_project_build_path(gl_project.namespace, gl_project, self)
+      end
+    end
+
     private
 
     def yaml_variables

app/models/commit_status.rb

@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
   def show_warning?
     false
   end
+
+  def download_url
+    nil
+  end
 end

app/uploaders/artifact_uploader.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+class ArtifactUploader < CarrierWave::Uploader::Base
+  storage :file
+
+  attr_accessor :build, :field
+
+  def self.artifacts_path
+    File.expand_path('shared/artifacts/', Rails.root)
+  end
+
+  def self.artifacts_upload_path
+    File.expand_path('shared/artifacts/tmp/uploads/', Rails.root)
+  end
+
+  def self.artifacts_cache_path
+    File.expand_path('shared/artifacts/tmp/cache/', Rails.root)
+  end
+
+  def initialize(build, field)
+    @build, @field = build, field
+  end
+
+  def artifacts_path
+    File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
+  end
+
+  def store_dir
+    File.join(ArtifactUploader.artifacts_path, artifacts_path)
+  end
+
+  def cache_dir
+    File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
+  end
+
+  def file_storage?
+    self.class.storage == CarrierWave::Storage::File
+  end
+
+  def exists?
+    file.try(:exists?)
+  end
+
+  def move_to_cache
+    true
+  end
+
+  def move_to_store
+    true
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -139,5 +139,10 @@
             = f.check_box :shared_runners_enabled
             Enable shared runners for a new projects
 
+    .form-group
+      = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_artifacts_size, class: 'form-control'
+
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'

app/views/projects/builds/show.html.haml

@@ -87,6 +87,9 @@
             Test coverage
           %h1 #{@build.coverage}%
 
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
+        .build-widget.center
+          = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
 
       .build-widget
         %h4.title

app/views/projects/commit_statuses/_commit_status.html.haml

@@ -61,6 +61,9 @@
 
   %td
     .pull-right
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
+        = link_to commit_status.download_url, title: 'Download artifacts' do
+          %i.fa.fa-download
       - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
         - if commit_status.active?
           - if commit_status.cancel_url

config/initializers/1_settings.rb

@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds']     = true if Settings.gitlab_ci['all_br
 Settings.gitlab_ci['add_pusher']            = false if Settings.gitlab_ci['add_pusher'].nil?
 Settings.gitlab_ci['url']                   ||= Settings.send(:build_gitlab_ci_url)
 Settings.gitlab_ci['builds_path']           = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
+Settings.gitlab_ci['max_artifacts_size']    ||= 100 # in megabytes
 
 #
 # Reply by email

config/routes.rb

@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
           member do
             get :status
             post :cancel
+            get :download
             post :retry
           end
         end

db/migrate/20151013092124_add_artifacts_file_to_builds.rb

@@ -0,0 +1,5 @@
+class AddArtifactsFileToBuilds < ActiveRecord::Migration
+  def change
+    add_column :ci_builds, :artifacts_file, :text
+  end
+end

db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb

@@ -0,0 +1,5 @@
+class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20151105094515) do
+ActiveRecord::Schema.define(version: 20151109100728) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
     t.text     "help_page_text"
     t.string   "admin_notification_email"
     t.boolean  "shared_runners_enabled",       default: true,  null: false
+    t.integer  "max_artifacts_size",           default: 100,   null: false
   end
 
   create_table "audit_events", force: true do |t|
@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
     t.string   "type"
     t.string   "target_url"
     t.string   "description"
+    t.text     "artifacts_file"
   end
 
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree

doc/ci/yaml/README.md

@@ -141,6 +141,7 @@ job_name:
 | tags          | optional | Defines a list of tags which are used to select runner |
 | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
 | when          | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
+| artifacts     | optional | Define list build artifacts |
 
 ### script
 `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
@@ -258,6 +259,35 @@ The above script will:
 1. Execute `cleanup_build` only when the `build` failed,
 2. Always execute `cleanup` as the last step in pipeline.
 
+### artifacts
+`artifacts` is used to specify list of files and directories which should be attached to build after success.
+
+1. Send all files in `binaries` and `.config`:
+```
+artifacts:
+  paths:
+  - binaries/
+  - .config
+```
+
+2. Send all git untracked files:
+```
+artifacts:
+  untracked: true
+```
+
+3. Send all git untracked files and files in `binaries`:
+```
+artifacts:
+  untracked: true
+  paths:
+  - binaries/
+```
+
+The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
+
+This feature requires GitLab Runner v0.7.0 or higher.
+
 ## Validate the .gitlab-ci.yml
 Each instance of GitLab CI has an embedded debug tool called Lint.
 You can find the link to the Lint in the project's settings page or use short url `/lint`.

doc/install/installation.md

@@ -246,6 +246,9 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
     # Change the permissions of the directory where CI build traces are stored
     sudo chmod -R u+rwX builds/
 
+    # Change the permissions of the directory where CI artifacts are stored
+    sudo chmod -R u+rwX shared/artifacts/
+
     # Copy the example Unicorn config
     sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
 

doc/raketasks/backup_restore.md

@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
 ```
 
 Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
-uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time.
+uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
+Use a comma to specify several options at the same time.
 
 ```
 sudo gitlab-rake gitlab:backup:create SKIP=db,uploads

lib/api/helpers.rb

@@ -133,6 +133,12 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def require_gitlab_workhorse!
+      unless env['HTTP_GITLAB_WORKHORSE'].present?
+        forbidden!('Request should be executed via GitLab Workhorse')
+      end
+    end
+
     def can?(object, action, subject)
       abilities.allowed?(object, action, subject)
     end
@@ -234,6 +240,10 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
 
+    def file_to_large!
+      render_api_error!('413 Request Entity Too Large', 413)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,44 @@ module API
       end
     end
 
+    # file helpers
+
+    def uploaded_file!(field, uploads_path)
+      if params[field]
+        bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
+        return params[field]
+      end
+
+      # sanitize file paths
+      # this requires all paths to exist
+      required_attributes! %W(#{field}.path)
+      uploads_path = File.realpath(uploads_path)
+      file_path = File.realpath(params["#{field}.path"])
+      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+      UploadedFile.new(
+        file_path,
+        params["#{field}.name"],
+        params["#{field}.type"] || 'application/octet-stream',
+      )
+    end
+
+    def present_file!(path, filename, content_type = 'application/octet-stream')
+      filename ||= File.basename(path)
+      header['Content-Disposition'] = "attachment; filename=#{filename}"
+      header['Content-Transfer-Encoding'] = 'binary'
+      content_type content_type
+
+      # Support download acceleration
+      case headers['X-Sendfile-Type']
+      when 'X-Sendfile'
+        header['X-Sendfile'] = path
+        body
+      else
+        file FileStreamer.new(path)
+      end
+    end
+
     private
 
     def add_pagination_headers(paginated, per_page)

lib/backup/artifacts.rb

@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Artifacts < Files
+    def initialize
+      super('artifacts', ArtifactUploader.artifacts_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end

lib/backup/manager.rb

@@ -150,7 +150,7 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup

lib/ci/api/api.rb

@@ -27,6 +27,7 @@ module Ci
 
       helpers Helpers
       helpers ::API::Helpers
+      helpers Gitlab::CurrentSettings
 
       mount Builds
       mount Commits

lib/ci/api/builds.rb

@@ -47,6 +47,106 @@ module Ci
             build.drop
           end
         end
+
+        # Authorize artifacts uploading for build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   filesize (optional) - the size of uploaded file
+        # Example Request:
+        #   POST /builds/:id/artifacts/authorize
+        post ":id/artifacts/authorize" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          if params[:filesize]
+            file_size = params[:filesize].to_i
+            file_to_large! unless file_size < max_artifacts_size
+          end
+
+          status 200
+          { TempPath: ArtifactUploader.artifacts_upload_path }
+        end
+
+        # Upload artifacts to build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   file (required) - The uploaded file
+        # Parameters (accelerated by GitLab Workhorse):
+        #   file.path - path to locally stored body (generated by Workhorse)
+        #   file.name - real filename as send in Content-Disposition
+        #   file.type - real content type as send in Content-Type
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Body:
+        #   The file content
+        #
+        # Example Request:
+        #   POST /builds/:id/artifacts
+        post ":id/artifacts" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
+          file_to_large! unless file.size < max_artifacts_size
+
+          if build.update_attributes(artifacts_file: file)
+            present build, with: Entities::Build
+          else
+            render_validation_error!(build)
+          end
+        end
+
+        # Download the artifacts file from build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   GET /builds/:id/artifacts
+        get ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          artifacts_file = build.artifacts_file
+
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
+          unless artifacts_file.exists?
+            not_found!
+          end
+
+          present_file!(artifacts_file.path, artifacts_file.filename)
+        end
+
+        # Remove the artifacts file from build
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   DELETE /builds/:id/artifacts
+        delete ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          build.remove_artifacts_file!
+        end
       end
     end
   end

lib/ci/api/entities.rb

@@ -11,10 +11,16 @@ module Ci
         expose :builds
       end
 
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
       class Build < Grape::Entity
         expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
           :before_sha, :allow_git_fetch, :project_name
 
+        expose :name, :token, :stage
+
         expose :options do |model|
           model.options
         end
@@ -24,6 +30,7 @@ module Ci
         end
 
         expose :variables
+        expose :artifacts_file, using: ArtifactFile
       end
 
       class Runner < Grape::Entity

lib/ci/api/helpers.rb

@@ -1,6 +1,8 @@
 module Ci
   module API
     module Helpers
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
         forbidden! unless project.valid_token?(params[:project_token])
       end
 
+      def authenticate_build_token!(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+        forbidden! unless token && build.valid_token?(token)
+      end
+
       def update_runner_last_contact
         # Use a random threshold to prevent beating DB updates
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
         info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
         current_runner.update(info)
       end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end

lib/ci/gitlab_ci_yaml_processor.rb

@@ -5,7 +5,7 @@ module Ci
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
 
     attr_reader :before_script, :image, :services, :variables, :path
 
@@ -77,7 +77,8 @@ module Ci
         when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
-          services: job[:services] || @services
+          services: job[:services] || @services,
+          artifacts: job[:artifacts]
         }.compact
       }
     end
@@ -159,7 +160,17 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      if job[:artifacts]
+        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+        end
+
+        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+        end
+      end
+
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
@@ -182,6 +193,10 @@ module Ci
       value.is_a?(String) || value.is_a?(Symbol)
     end
 
+    def validate_boolean(value)
+      value.in?([true, false])
+    end
+
     def process?(only_params, except_params, ref, tag)
       if only_params.present?
         return false unless matching?(only_params, ref, tag)

lib/file_streamer.rb

@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+  attr_reader :to_path
+
+  def initialize(path)
+    @to_path = path
+  end
+
+  # Stream the file's contents if Rack::Sendfile isn't present.
+  def each
+    File.open(to_path, 'rb') do |file|
+      while chunk = file.read(16384)
+        yield chunk
+      end
+    end
+  end
+end

lib/gitlab/current_settings.rb

@@ -25,6 +25,7 @@ module Gitlab
         session_expire_delay: Settings.gitlab['session_expire_delay'],
         import_sources: Settings.gitlab['import_sources'],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+        max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
       )
     end
 

lib/support/nginx/gitlab

@@ -131,6 +131,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.

lib/support/nginx/gitlab-ssl

@@ -178,6 +178,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.

lib/tasks/gitlab/backup.rake

@@ -12,6 +12,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
+      Rake::Task["gitlab:backup:artifacts:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :artifacts do
+      task create: :environment do
+        $progress.puts "Dumping artifacts ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Artifacts.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring artifacts ... ".blue
+        Backup::Artifacts.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a

lib/uploaded_file.rb

@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+  # The filename, *not* including the path, of the "uploaded" file
+  attr_reader :original_filename
+
+  # The tempfile
+  attr_reader :tempfile
+
+  # The content type of the "uploaded" file
+  attr_accessor :content_type
+
+  def initialize(path, filename, content_type = "text/plain")
+    raise "#{path} file does not exist" unless ::File.exist?(path)
+
+    @content_type = content_type
+    @original_filename = filename || ::File.basename(path)
+    @tempfile = File.new(path, 'rb')
+  end
+
+  def path
+    @tempfile.path
+  end
+
+  alias_method :local_path, :path
+
+  def method_missing(method_name, *args, &block) #:nodoc:
+    @tempfile.__send__(method_name, *args, &block)
+  end
+
+  def respond_to?(method_name, include_private = false) #:nodoc:
+    @tempfile.respond_to?(method_name, include_private) || super
+  end
+end

spec/features/builds_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe "Builds" do
+  let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+
   before do
     login_as(:user)
     @commit = FactoryGirl.create :ci_commit
@@ -66,6 +68,15 @@ describe "Builds" do
     it { expect(page).to have_content @commit.sha[0..7] }
     it { expect(page).to have_content @commit.git_commit_message }
     it { expect(page).to have_content @commit.git_author_name }
+
+    context "Download artifacts" do
+      before do
+        @build.update_attributes(artifacts_file: artifacts_file)
+        visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
+      end
+
+      it { expect(page).to have_content 'Download artifacts' }
+    end
   end
 
   describe "POST /:project/builds/:id/cancel" do
@@ -90,4 +101,14 @@ describe "Builds" do
     it { expect(page).to have_content 'pending' }
     it { expect(page).to have_content 'Cancel' }
   end
+
+  describe "GET /:project/builds/:id/download" do
+    before do
+      @build.update_attributes(artifacts_file: artifacts_file)
+      visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
+      click_link 'Download artifacts'
+    end
+
+    it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) }
+  end
 end

spec/features/commits_spec.rb

@@ -19,7 +19,7 @@ describe "Commits" do
       stub_ci_commit_to_return_yaml_file
     end
 
-    describe "GET /:project/commits/:sha" do
+    describe "GET /:project/commits/:sha/ci" do
       before do
         visit ci_status_path(@commit)
       end
@@ -29,6 +29,20 @@ describe "Commits" do
       it { expect(page).to have_content @commit.git_author_name }
     end
 
+    context "Download artifacts" do
+      let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+
+      before do
+        @build.update_attributes(artifacts_file: artifacts_file)
+      end
+
+      it do
+        visit ci_status_path(@commit)
+        click_on "Download artifacts"
+        expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type)
+      end
+    end
+
     describe "Cancel all builds" do
       it "cancels commit" do
         visit ci_status_path(@commit)

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -333,6 +333,43 @@ module Ci
       end
     end
 
+    describe "Artifacts" do
+      it "returns artifacts when defined" do
+        config = YAML.dump({
+                             image:         "ruby:2.1",
+                             services:      ["mysql"],
+                             before_script: ["pwd"],
+                             rspec:         {
+                               artifacts: { paths: ["logs/", "binaries/"], untracked: true },
+                               script: "rspec"
+                             }
+                           })
+
+        config_processor = GitlabCiYamlProcessor.new(config)
+
+        expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
+        expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
+          except: nil,
+          stage: "test",
+          stage_idx: 1,
+          name: :rspec,
+          only: nil,
+          commands: "pwd\nrspec",
+          tag_list: [],
+          options: {
+            image: "ruby:2.1",
+            services: ["mysql"],
+            artifacts: {
+              paths: ["logs/", "binaries/"],
+              untracked: true
+            }
+          },
+          when: "on_success",
+          allow_failure: false
+        })
+      end
+    end
+
     describe "Error handling" do
       it "indicates that object is invalid" do
         expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError)
@@ -491,6 +528,20 @@ module Ci
           GitlabCiYamlProcessor.new(config, path)
         end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always")
       end
+
+      it "returns errors if job artifacts:untracked is not an array of strings" do
+        config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:untracked parameter should be an boolean")
+      end
+
+      it "returns errors if job artifacts:paths is not an array of strings" do
+        config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { paths: "string" } } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:paths parameter should be an array of strings")
+      end
     end
   end
 end

spec/models/build_spec.rb

@@ -400,4 +400,19 @@ describe Ci::Build do
       end
     end
   end
+
+  describe :download_url do
+    subject { build.download_url }
+
+    it "should be nil if artifact doesn't exist" do
+      build.update_attributes(artifacts_file: nil)
+      is_expected.to be_nil
+    end
+
+    it 'should be nil if artifact exist' do
+      gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
+      build.update_attributes(artifacts_file: gif)
+      is_expected.to_not be_nil
+    end
+  end
 end

spec/requests/api/users_spec.rb

@@ -343,8 +343,9 @@ describe API::API, api: true  do
       end.to change{ user.keys.count }.by(1)
     end
 
-    it "should raise error for invalid ID" do
-      expect{post api("/users/ASDF/keys", admin) }.to raise_error(ActionController::RoutingError)
+    it "should return 405 for invalid ID" do
+      post api("/users/ASDF/keys", admin)
+      expect(response.status).to eq(405)
     end
   end
 
@@ -374,9 +375,9 @@ describe API::API, api: true  do
         expect(json_response.first['title']).to eq(key.title)
       end
 
-      it "should return 404 for invalid ID" do
+      it "should return 405 for invalid ID" do
         get api("/users/ASDF/keys", admin)
-        expect(response.status).to eq(404)
+        expect(response.status).to eq(405)
       end
     end
   end
@@ -434,7 +435,8 @@ describe API::API, api: true  do
     end
 
     it "should raise error for invalid ID" do
-      expect{post api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
+      post api("/users/ASDF/emails", admin)
+      expect(response.status).to eq(405)
     end
   end
 
@@ -465,7 +467,8 @@ describe API::API, api: true  do
       end
 
       it "should raise error for invalid ID" do
-        expect{put api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
+        put api("/users/ASDF/emails", admin)
+        expect(response.status).to eq(405)
       end
     end
   end

spec/requests/ci/api/builds_spec.rb

@@ -41,7 +41,7 @@ describe Ci::API::API do
 
       it "should return 404 error if no builds for specific runner" do
         commit = FactoryGirl.create(:ci_commit, gl_project: shared_gl_project)
-        FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
+        FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
 
         post ci_api("/builds/register"), token: runner.token
 
@@ -50,7 +50,7 @@ describe Ci::API::API do
 
       it "should return 404 error if no builds for shared runner" do
         commit = FactoryGirl.create(:ci_commit, gl_project: gl_project)
-        FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
+        FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
 
         post ci_api("/builds/register"), token: shared_runner.token
 
@@ -79,7 +79,7 @@ describe Ci::API::API do
           { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
           { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
           { "key" => "DB_NAME", "value" => "postgres", "public" => true },
-          { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
+          { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
         ])
       end
 
@@ -122,5 +122,194 @@ describe Ci::API::API do
         expect(build.reload.trace).to eq 'hello_world'
       end
     end
+
+    context "Artifacts" do
+      let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+      let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
+      let(:commit) { FactoryGirl.create(:ci_commit, gl_project: gl_project) }
+      let(:build) { FactoryGirl.create(:ci_build, commit: commit, runner_id: runner.id) }
+      let(:authorize_url) { ci_api("/builds/#{build.id}/artifacts/authorize") }
+      let(:post_url) { ci_api("/builds/#{build.id}/artifacts") }
+      let(:delete_url) { ci_api("/builds/#{build.id}/artifacts") }
+      let(:get_url) { ci_api("/builds/#{build.id}/artifacts") }
+      let(:headers) { { "GitLab-Workhorse" => "1.0" } }
+      let(:headers_with_token) { headers.merge(Ci::API::Helpers::BUILD_TOKEN_HEADER => build.project.token) }
+
+      describe "POST /builds/:id/artifacts/authorize" do
+        context "should authorize posting artifact to running build" do
+          before do
+            build.run!
+          end
+
+          it "using token as parameter" do
+            post authorize_url, { token: build.project.token }, headers
+            expect(response.status).to eq(200)
+            expect(json_response["TempPath"]).to_not be_nil
+          end
+
+          it "using token as header" do
+            post authorize_url, {}, headers_with_token
+            expect(response.status).to eq(200)
+            expect(json_response["TempPath"]).to_not be_nil
+          end
+        end
+
+        context "should fail to post too large artifact" do
+          before do
+            build.run!
+          end
+
+          it "using token as parameter" do
+            settings = Gitlab::CurrentSettings::current_application_settings
+            settings.update_attributes(max_artifacts_size: 0)
+            post authorize_url, { token: build.project.token, filesize: 100 }, headers
+            expect(response.status).to eq(413)
+          end
+
+          it "using token as header" do
+            settings = Gitlab::CurrentSettings::current_application_settings
+            settings.update_attributes(max_artifacts_size: 0)
+            post authorize_url, { filesize: 100 }, headers_with_token
+            expect(response.status).to eq(413)
+          end
+        end
+
+        context "should get denied" do
+          it do
+            post authorize_url, { token: 'invalid', filesize: 100 }
+            expect(response.status).to eq(403)
+          end
+        end
+      end
+
+      describe "POST /builds/:id/artifacts" do
+        context "Disable sanitizer" do
+          before do
+            # by configuring this path we allow to pass temp file from any path
+            allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
+          end
+
+          context "should post artifact to running build" do
+            before do
+              build.run!
+            end
+
+            it "uses regual file post" do
+              upload_artifacts(file_upload, headers_with_token, false)
+              expect(response.status).to eq(201)
+              expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
+            end
+
+            it "uses accelerated file post" do
+              upload_artifacts(file_upload, headers_with_token, true)
+              expect(response.status).to eq(201)
+              expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
+            end
+
+            it "updates artifact" do
+              upload_artifacts(file_upload, headers_with_token)
+              upload_artifacts(file_upload2, headers_with_token)
+              expect(response.status).to eq(201)
+              expect(json_response["artifacts_file"]["filename"]).to eq(file_upload2.original_filename)
+            end
+          end
+
+          context "should fail to post too large artifact" do
+            before do
+              build.run!
+            end
+
+            it do
+              settings = Gitlab::CurrentSettings::current_application_settings
+              settings.update_attributes(max_artifacts_size: 0)
+              upload_artifacts(file_upload, headers_with_token)
+              expect(response.status).to eq(413)
+            end
+          end
+
+          context "should fail to post artifacts without file" do
+            before do
+              build.run!
+            end
+
+            it do
+              post post_url, {}, headers_with_token
+              expect(response.status).to eq(400)
+            end
+          end
+
+          context "should fail to post artifacts without GitLab-Workhorse" do
+            before do
+              build.run!
+            end
+
+            it do
+              post post_url, { token: build.project.token }, {}
+              expect(response.status).to eq(403)
+            end
+          end
+        end
+
+        context "should fail to post artifacts for outside of tmp path" do
+          before do
+            # by configuring this path we allow to pass file from @tmpdir only
+            # but all temporary files are stored in system tmp directory
+            @tmpdir = Dir.mktmpdir
+            allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
+            build.run!
+          end
+
+          after do
+            FileUtils.remove_entry @tmpdir
+          end
+
+          it do
+            upload_artifacts(file_upload, headers_with_token)
+            expect(response.status).to eq(400)
+          end
+        end
+
+        def upload_artifacts(file, headers = {}, accelerated = true)
+          if accelerated
+            post post_url, {
+              'file.path' => file.path,
+              'file.name' => file.original_filename
+            }, headers
+          else
+            post post_url, { file: file }, headers
+          end
+        end
+      end
+
+      describe "DELETE /builds/:id/artifacts" do
+        before do
+          build.run!
+          post delete_url, token: build.project.token, file: file_upload
+        end
+
+        it "should delete artifact build" do
+          build.success
+          delete delete_url, token: build.project.token
+          expect(response.status).to eq(200)
+        end
+      end
+
+      describe "GET /builds/:id/artifacts" do
+        before do
+          build.run!
+        end
+
+        it "should download artifact" do
+          build.update_attributes(artifacts_file: file_upload)
+          get get_url, token: build.project.token
+          expect(response.status).to eq(200)
+        end
+
+        it "should fail to download if no artifact uploaded" do
+          get get_url, token: build.project.token
+          expect(response.status).to eq(404)
+        end
+      end
+    end
   end
 end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
   end
 
   def reenable_backup_sub_tasks
-    %w{db repo uploads builds}.each do |subtask|
+    %w{db repo uploads builds artifacts}.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -56,6 +56,7 @@ describe 'gitlab:app namespace rake task' do
         expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
+        expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
         expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
       end
@@ -113,19 +114,20 @@ describe 'gitlab:app namespace rake task' do
 
     it 'should set correct permissions on the tar contents' do
       tar_contents, exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
       )
       expect(exit_status).to eq(0)
       expect(tar_contents).to match('db/')
       expect(tar_contents).to match('uploads.tar.gz')
       expect(tar_contents).to match('repositories/')
       expect(tar_contents).to match('builds.tar.gz')
-      expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/)
+      expect(tar_contents).to match('artifacts.tar.gz')
+      expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
     end
 
     it 'should delete temp directories' do
       temp_dirs = Dir.glob(
-        File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds}')
+        File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}')
       )
 
       expect(temp_dirs).to be_empty
@@ -161,12 +163,13 @@ describe 'gitlab:app namespace rake task' do
 
     it "does not contain skipped item" do
       tar_contents, _exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
       )
 
       expect(tar_contents).to match('db/')
       expect(tar_contents).to match('uploads.tar.gz')
       expect(tar_contents).to match('builds.tar.gz')
+      expect(tar_contents).to match('artifacts.tar.gz')
       expect(tar_contents).not_to match('repositories/')
     end
 
@@ -178,6 +181,7 @@ describe 'gitlab:app namespace rake task' do
       expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
       expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
+      expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
     end