Add jobs requesting API

lib/api/entities.rb

@@ -697,5 +697,43 @@ module API
       expose :id, :message, :starts_at, :ends_at, :color, :font
       expose :active?, as: :active
     end
+
+    class ArtifactFile < Grape::Entity
+      expose :filename, :size
+    end
+
+    class JobCredentials < Grape::Entity
+      expose :type, :url, :username, :password
+    end
+
+    class JobResponse < Grape::Entity
+      expose :id, :ref, :tag, :sha, :status
+      expose :name, :token, :stage
+      expose :project_id
+      expose :project_name
+      expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
+    end
+
+    class RequestJobResponse < JobResponse
+      expose :commands
+      expose :repo_url
+      expose :before_sha
+      expose :allow_git_fetch
+      expose :token
+      expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
+
+      expose :options do |model|
+        model.options
+      end
+
+      expose :timeout do |model|
+        model.timeout
+      end
+
+      expose :variables
+      expose :depends_on_builds, using: JobResponse
+
+      expose :credentials, using: JobCredentials
+    end
   end
 end

lib/api/helpers/runner.rb

@@ -1,6 +1,8 @@
 module API
   module Helpers
     module Runner
+      UPDATE_RUNNER_EVERY = 10 * 60
+
       def runner_registration_token_valid?
         ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
                                                                   current_application_settings.runners_registration_token)
@@ -18,6 +20,32 @@ module API
       def current_runner
         @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
       end
+
+      def update_runner_info
+        return unless update_runner?
+
+        current_runner.contacted_at = Time.now
+        current_runner.assign_attributes(get_runner_version_from_params)
+        current_runner.save if current_runner.changed?
+      end
+
+      def update_runner?
+        # Use a random threshold to prevent beating DB updates.
+        # It generates a distribution between [40m, 80m].
+        #
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+
+        current_runner.contacted_at.nil? ||
+            (Time.now - current_runner.contacted_at) >= contacted_at_max_age
+      end
+
+      def build_not_found!
+        if headers['User-Agent'].to_s.match(/gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /)
+          no_content!
+        else
+          not_found!
+        end
+      end
     end
   end
 end

lib/api/runner.rb

@@ -48,5 +48,44 @@ module API
         Ci::Runner.find_by_token(params[:token]).destroy
       end
     end
+
+    resource :jobs do
+      desc 'Request a job' do
+        success Entities::RequestJobResponse
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+      end
+      post '/request' do
+        authenticate_runner!
+        not_found! unless current_runner.active?
+        update_runner_info
+
+        if current_runner.is_runner_queue_value_latest?(params[:last_update])
+          header 'X-GitLab-Last-Update', params[:last_update]
+          Gitlab::Metrics.add_event(:build_not_found_cached)
+          return build_not_found!
+        end
+
+        new_update = current_runner.ensure_runner_queue_value
+        result = ::Ci::RegisterBuildService.new(current_runner).execute
+
+        if result.valid?
+          if result.build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: result.build.project.path_with_namespace)
+            present result.build, with: Entities::RequestJobResponse
+          else
+            Gitlab::Metrics.add_event(:build_not_found)
+            header 'X-GitLab-Last-Update', new_update
+            build_not_found!
+          end
+        else
+          # We received build that is invalid due to concurrency conflict
+          Gitlab::Metrics.add_event(:build_invalid)
+          conflict!
+        end
+      end
+    end
   end
 end

spec/requests/api/runner_spec.rb

@@ -148,4 +148,261 @@ describe API::Runner do
       end
     end
   end
+
+  describe '/api/v4/jobs' do
+    let(:project) { create(:empty_project, shared_runners_enabled: false) }
+    let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') }
+    let!(:job) { create(:ci_build, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }
+    let(:runner) { create(:ci_runner) }
+
+    before { project.runners << runner }
+
+    describe 'POST /api/v4/jobs/request' do
+      let!(:last_update) {}
+      let!(:new_update) { }
+      let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' }
+
+      before { stub_container_registry_config(enabled: false) }
+
+      shared_examples 'no jobs available' do
+        before { request_job }
+
+        context 'when runner sends version in User-Agent' do
+          context 'for stable version' do
+            it 'gives 204 and set X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header).to have_key('X-GitLab-Last-Update')
+            end
+          end
+
+          context 'when last_update is up-to-date' do
+            let(:last_update) { runner.ensure_runner_queue_value }
+
+            it 'gives 204 and set the same X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header['X-GitLab-Last-Update']).to eq(last_update)
+            end
+          end
+
+          context 'when last_update is outdated' do
+            let(:last_update) { runner.ensure_runner_queue_value }
+            let(:new_update) { runner.tick_runner_queue }
+
+            it 'gives 204 and set a new X-GitLab-Last-Update' do
+              expect(response).to have_http_status(204)
+              expect(response.header['X-GitLab-Last-Update']).to eq(new_update)
+            end
+          end
+
+          context 'for beta version' do
+            let(:user_agent) { 'gitlab-runner 9.0.0~beta.167.g2b2bacc (master; go1.7.4; linux/amd64)' }
+            it { expect(response).to have_http_status(204) }
+          end
+
+          context 'for pre-9-0 version' do
+            let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0 (1-6-stable; go1.6.3; linux/amd64)' }
+            it { expect(response).to have_http_status(204) }
+          end
+
+          context 'for pre-9-0 beta version' do
+            let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0~beta.167.g2b2bacc (master; go1.6.3; linux/amd64)' }
+            it { expect(response).to have_http_status(204) }
+          end
+        end
+
+        context %q(when runner doesn't send version in User-Agent) do
+          let(:user_agent) { 'Go-http-client/1.1' }
+          it { expect(response).to have_http_status(404) }
+        end
+
+        context %q(when runner doesn't have a User-Agent) do
+          let(:user_agent) { nil }
+          it { expect(response).to have_http_status(404) }
+        end
+      end
+
+      context 'when no token is provided' do
+        it 'returns 400 error' do
+          post api('/jobs/request')
+          expect(response).to have_http_status 400
+        end
+      end
+
+      context 'when invalid token is provided' do
+        it 'returns 403 error' do
+          post api('/jobs/request'), token: 'invalid'
+          expect(response).to have_http_status 403
+        end
+      end
+
+      context 'when valid token is provided' do
+
+        context 'when Runner is not active' do
+          let(:runner) { create(:ci_runner, :inactive) }
+
+          it 'returns 404 error' do
+            request_job
+            expect(response).to have_http_status 404
+          end
+        end
+
+        context 'when jobs are finished' do
+          before { job.success }
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when other projects have pending jobs' do
+          before do
+            job.success
+            create(:ci_build, :pending)
+          end
+
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when shared runner requests job for project without shared_runners_enabled' do
+          let(:runner) { create(:ci_runner, :shared) }
+          it_behaves_like 'no jobs available'
+        end
+
+        context 'when there is a pending job' do
+          it 'starts a job' do
+            request_job info: {platform: :darwin}
+
+            expect(response).to have_http_status(201)
+            expect(response.headers).not_to have_key('X-GitLab-Last-Update')
+            expect(json_response['sha']).to eq(job.sha)
+            expect(json_response['options']).to eq({'image' => 'ruby:2.1', 'services' => ['postgres']})
+            expect(json_response['variables']).to include(
+                                                      {'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true},
+                                                      {'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true},
+                                                      {'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true}
+                                                  )
+            expect(runner.reload.platform).to eq('darwin')
+          end
+
+          it 'updates runner info' do
+            expect { request_job }.to change { runner.reload.contacted_at }
+          end
+
+          %w(name version revision platform architecture).each do |param|
+            context "when info parameter '#{param}' is present" do
+              let(:value) { "#{param}_value" }
+
+              it %q(updates provided Runner's parameter) do
+                request_job info: {param => value}
+
+                expect(response).to have_http_status(201)
+                runner.reload
+                expect(runner.read_attribute(param.to_sym)).to eq(value)
+              end
+            end
+          end
+
+          context 'when concurrently updating a job' do
+            before do
+              expect_any_instance_of(Ci::Build).to receive(:run!).
+                  and_raise(ActiveRecord::StaleObjectError.new(nil, nil))
+            end
+
+            it 'returns a conflict' do
+              request_job
+              expect(response).to have_http_status(409)
+              expect(response.headers).not_to have_key('X-GitLab-Last-Update')
+            end
+          end
+
+          context 'when project and pipeline have multiple jobs' do
+            let!(:test_job) { create(:ci_build, pipeline: pipeline, name: 'deploy', stage: 'deploy', stage_idx: 1) }
+
+            before { job.success }
+
+            it 'returns dependent jobs' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['id']).to eq(test_job.id)
+              expect(json_response['depends_on_builds'].count).to eq(1)
+              expect(json_response['depends_on_builds'][0]).to include('id' => job.id, 'name' => 'spinach')
+            end
+          end
+
+          context 'when job has no tags' do
+            before { job.update(tags: []) }
+
+            context 'when runner is allowed to pick untagged jobs' do
+              before { runner.update_column(:run_untagged, true) }
+
+              it 'picks job' do
+                request_job
+                expect(response).to have_http_status 201
+              end
+            end
+
+            context 'when runner is not allowed to pick untagged jobs' do
+              before { runner.update_column(:run_untagged, false) }
+              it_behaves_like 'no jobs available'
+            end
+          end
+
+          context 'when triggered job is available' do
+            before do
+              trigger = create(:ci_trigger, project: project)
+              create(:ci_trigger_request_with_variables, pipeline: pipeline, builds: [job], trigger: trigger)
+              project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
+            end
+
+            it 'returns variables for triggers' do
+              request_job
+
+              expect(response).to have_http_status(201)
+              expect(json_response['variables']).to include(
+                                                        {'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true},
+                                                        {'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true},
+                                                        {'key' => 'CI_BUILD_TRIGGERED', 'value' => 'true', 'public' => true},
+                                                        {'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true},
+                                                        {'key' => 'SECRET_KEY', 'value' => 'secret_value', 'public' => false},
+                                                        {'key' => 'TRIGGER_KEY_1', 'value' => 'TRIGGER_VALUE_1', 'public' => false},
+                                                    )
+            end
+          end
+
+          describe 'registry credentials support' do
+            let(:registry_url) { 'registry.example.com:5005' }
+            let(:registry_credentials) do
+              {'type' => 'registry',
+               'url' => registry_url,
+               'username' => 'gitlab-ci-token',
+               'password' => job.token}
+            end
+
+            context 'when registry is enabled' do
+              before { stub_container_registry_config(enabled: true, host_port: registry_url) }
+
+              it 'sends registry credentials key' do
+                request_job
+                expect(json_response).to have_key('credentials')
+                expect(json_response['credentials']).to include(registry_credentials)
+              end
+            end
+
+            context 'when registry is disabled' do
+              before { stub_container_registry_config(enabled: false, host_port: registry_url) }
+
+              it 'does not send registry credentials' do
+                request_job
+                expect(json_response).to have_key('credentials')
+                expect(json_response['credentials']).not_to include(registry_credentials)
+              end
+            end
+          end
+        end
+
+        def request_job(token = runner.token, **params)
+          new_params = params.merge(token: token, last_update: last_update)
+          post api('/jobs/request'), new_params, {'User-Agent' => user_agent}
+        end
+      end
+    end
+  end
 end