Add jobs requesting API
This commit is contained in:
parent
b8ca9bc43a
commit
3d26a8d0b6
|
@ -697,5 +697,43 @@ module API
|
||||||
expose :id, :message, :starts_at, :ends_at, :color, :font
|
expose :id, :message, :starts_at, :ends_at, :color, :font
|
||||||
expose :active?, as: :active
|
expose :active?, as: :active
|
||||||
end
|
end
|
||||||
|
|
||||||
|
class ArtifactFile < Grape::Entity
|
||||||
|
expose :filename, :size
|
||||||
|
end
|
||||||
|
|
||||||
|
class JobCredentials < Grape::Entity
|
||||||
|
expose :type, :url, :username, :password
|
||||||
|
end
|
||||||
|
|
||||||
|
class JobResponse < Grape::Entity
|
||||||
|
expose :id, :ref, :tag, :sha, :status
|
||||||
|
expose :name, :token, :stage
|
||||||
|
expose :project_id
|
||||||
|
expose :project_name
|
||||||
|
expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
|
||||||
|
end
|
||||||
|
|
||||||
|
class RequestJobResponse < JobResponse
|
||||||
|
expose :commands
|
||||||
|
expose :repo_url
|
||||||
|
expose :before_sha
|
||||||
|
expose :allow_git_fetch
|
||||||
|
expose :token
|
||||||
|
expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
|
||||||
|
|
||||||
|
expose :options do |model|
|
||||||
|
model.options
|
||||||
|
end
|
||||||
|
|
||||||
|
expose :timeout do |model|
|
||||||
|
model.timeout
|
||||||
|
end
|
||||||
|
|
||||||
|
expose :variables
|
||||||
|
expose :depends_on_builds, using: JobResponse
|
||||||
|
|
||||||
|
expose :credentials, using: JobCredentials
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
module API
|
module API
|
||||||
module Helpers
|
module Helpers
|
||||||
module Runner
|
module Runner
|
||||||
|
UPDATE_RUNNER_EVERY = 10 * 60
|
||||||
|
|
||||||
def runner_registration_token_valid?
|
def runner_registration_token_valid?
|
||||||
ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
|
ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
|
||||||
current_application_settings.runners_registration_token)
|
current_application_settings.runners_registration_token)
|
||||||
|
@ -18,6 +20,32 @@ module API
|
||||||
def current_runner
|
def current_runner
|
||||||
@runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
|
@runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def update_runner_info
|
||||||
|
return unless update_runner?
|
||||||
|
|
||||||
|
current_runner.contacted_at = Time.now
|
||||||
|
current_runner.assign_attributes(get_runner_version_from_params)
|
||||||
|
current_runner.save if current_runner.changed?
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_runner?
|
||||||
|
# Use a random threshold to prevent beating DB updates.
|
||||||
|
# It generates a distribution between [40m, 80m].
|
||||||
|
#
|
||||||
|
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
|
||||||
|
|
||||||
|
current_runner.contacted_at.nil? ||
|
||||||
|
(Time.now - current_runner.contacted_at) >= contacted_at_max_age
|
||||||
|
end
|
||||||
|
|
||||||
|
def build_not_found!
|
||||||
|
if headers['User-Agent'].to_s.match(/gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /)
|
||||||
|
no_content!
|
||||||
|
else
|
||||||
|
not_found!
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -48,5 +48,44 @@ module API
|
||||||
Ci::Runner.find_by_token(params[:token]).destroy
|
Ci::Runner.find_by_token(params[:token]).destroy
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
resource :jobs do
|
||||||
|
desc 'Request a job' do
|
||||||
|
success Entities::RequestJobResponse
|
||||||
|
end
|
||||||
|
params do
|
||||||
|
requires :token, type: String, desc: %q(Runner's authentication token)
|
||||||
|
end
|
||||||
|
post '/request' do
|
||||||
|
authenticate_runner!
|
||||||
|
not_found! unless current_runner.active?
|
||||||
|
update_runner_info
|
||||||
|
|
||||||
|
if current_runner.is_runner_queue_value_latest?(params[:last_update])
|
||||||
|
header 'X-GitLab-Last-Update', params[:last_update]
|
||||||
|
Gitlab::Metrics.add_event(:build_not_found_cached)
|
||||||
|
return build_not_found!
|
||||||
|
end
|
||||||
|
|
||||||
|
new_update = current_runner.ensure_runner_queue_value
|
||||||
|
result = ::Ci::RegisterBuildService.new(current_runner).execute
|
||||||
|
|
||||||
|
if result.valid?
|
||||||
|
if result.build
|
||||||
|
Gitlab::Metrics.add_event(:build_found,
|
||||||
|
project: result.build.project.path_with_namespace)
|
||||||
|
present result.build, with: Entities::RequestJobResponse
|
||||||
|
else
|
||||||
|
Gitlab::Metrics.add_event(:build_not_found)
|
||||||
|
header 'X-GitLab-Last-Update', new_update
|
||||||
|
build_not_found!
|
||||||
|
end
|
||||||
|
else
|
||||||
|
# We received build that is invalid due to concurrency conflict
|
||||||
|
Gitlab::Metrics.add_event(:build_invalid)
|
||||||
|
conflict!
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -148,4 +148,261 @@ describe API::Runner do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe '/api/v4/jobs' do
|
||||||
|
let(:project) { create(:empty_project, shared_runners_enabled: false) }
|
||||||
|
let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') }
|
||||||
|
let!(:job) { create(:ci_build, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }
|
||||||
|
let(:runner) { create(:ci_runner) }
|
||||||
|
|
||||||
|
before { project.runners << runner }
|
||||||
|
|
||||||
|
describe 'POST /api/v4/jobs/request' do
|
||||||
|
let!(:last_update) {}
|
||||||
|
let!(:new_update) { }
|
||||||
|
let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' }
|
||||||
|
|
||||||
|
before { stub_container_registry_config(enabled: false) }
|
||||||
|
|
||||||
|
shared_examples 'no jobs available' do
|
||||||
|
before { request_job }
|
||||||
|
|
||||||
|
context 'when runner sends version in User-Agent' do
|
||||||
|
context 'for stable version' do
|
||||||
|
it 'gives 204 and set X-GitLab-Last-Update' do
|
||||||
|
expect(response).to have_http_status(204)
|
||||||
|
expect(response.header).to have_key('X-GitLab-Last-Update')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when last_update is up-to-date' do
|
||||||
|
let(:last_update) { runner.ensure_runner_queue_value }
|
||||||
|
|
||||||
|
it 'gives 204 and set the same X-GitLab-Last-Update' do
|
||||||
|
expect(response).to have_http_status(204)
|
||||||
|
expect(response.header['X-GitLab-Last-Update']).to eq(last_update)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when last_update is outdated' do
|
||||||
|
let(:last_update) { runner.ensure_runner_queue_value }
|
||||||
|
let(:new_update) { runner.tick_runner_queue }
|
||||||
|
|
||||||
|
it 'gives 204 and set a new X-GitLab-Last-Update' do
|
||||||
|
expect(response).to have_http_status(204)
|
||||||
|
expect(response.header['X-GitLab-Last-Update']).to eq(new_update)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'for beta version' do
|
||||||
|
let(:user_agent) { 'gitlab-runner 9.0.0~beta.167.g2b2bacc (master; go1.7.4; linux/amd64)' }
|
||||||
|
it { expect(response).to have_http_status(204) }
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'for pre-9-0 version' do
|
||||||
|
let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0 (1-6-stable; go1.6.3; linux/amd64)' }
|
||||||
|
it { expect(response).to have_http_status(204) }
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'for pre-9-0 beta version' do
|
||||||
|
let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0~beta.167.g2b2bacc (master; go1.6.3; linux/amd64)' }
|
||||||
|
it { expect(response).to have_http_status(204) }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context %q(when runner doesn't send version in User-Agent) do
|
||||||
|
let(:user_agent) { 'Go-http-client/1.1' }
|
||||||
|
it { expect(response).to have_http_status(404) }
|
||||||
|
end
|
||||||
|
|
||||||
|
context %q(when runner doesn't have a User-Agent) do
|
||||||
|
let(:user_agent) { nil }
|
||||||
|
it { expect(response).to have_http_status(404) }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when no token is provided' do
|
||||||
|
it 'returns 400 error' do
|
||||||
|
post api('/jobs/request')
|
||||||
|
expect(response).to have_http_status 400
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when invalid token is provided' do
|
||||||
|
it 'returns 403 error' do
|
||||||
|
post api('/jobs/request'), token: 'invalid'
|
||||||
|
expect(response).to have_http_status 403
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when valid token is provided' do
|
||||||
|
|
||||||
|
context 'when Runner is not active' do
|
||||||
|
let(:runner) { create(:ci_runner, :inactive) }
|
||||||
|
|
||||||
|
it 'returns 404 error' do
|
||||||
|
request_job
|
||||||
|
expect(response).to have_http_status 404
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when jobs are finished' do
|
||||||
|
before { job.success }
|
||||||
|
it_behaves_like 'no jobs available'
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when other projects have pending jobs' do
|
||||||
|
before do
|
||||||
|
job.success
|
||||||
|
create(:ci_build, :pending)
|
||||||
|
end
|
||||||
|
|
||||||
|
it_behaves_like 'no jobs available'
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when shared runner requests job for project without shared_runners_enabled' do
|
||||||
|
let(:runner) { create(:ci_runner, :shared) }
|
||||||
|
it_behaves_like 'no jobs available'
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when there is a pending job' do
|
||||||
|
it 'starts a job' do
|
||||||
|
request_job info: {platform: :darwin}
|
||||||
|
|
||||||
|
expect(response).to have_http_status(201)
|
||||||
|
expect(response.headers).not_to have_key('X-GitLab-Last-Update')
|
||||||
|
expect(json_response['sha']).to eq(job.sha)
|
||||||
|
expect(json_response['options']).to eq({'image' => 'ruby:2.1', 'services' => ['postgres']})
|
||||||
|
expect(json_response['variables']).to include(
|
||||||
|
{'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true},
|
||||||
|
{'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true},
|
||||||
|
{'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true}
|
||||||
|
)
|
||||||
|
expect(runner.reload.platform).to eq('darwin')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'updates runner info' do
|
||||||
|
expect { request_job }.to change { runner.reload.contacted_at }
|
||||||
|
end
|
||||||
|
|
||||||
|
%w(name version revision platform architecture).each do |param|
|
||||||
|
context "when info parameter '#{param}' is present" do
|
||||||
|
let(:value) { "#{param}_value" }
|
||||||
|
|
||||||
|
it %q(updates provided Runner's parameter) do
|
||||||
|
request_job info: {param => value}
|
||||||
|
|
||||||
|
expect(response).to have_http_status(201)
|
||||||
|
runner.reload
|
||||||
|
expect(runner.read_attribute(param.to_sym)).to eq(value)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when concurrently updating a job' do
|
||||||
|
before do
|
||||||
|
expect_any_instance_of(Ci::Build).to receive(:run!).
|
||||||
|
and_raise(ActiveRecord::StaleObjectError.new(nil, nil))
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'returns a conflict' do
|
||||||
|
request_job
|
||||||
|
expect(response).to have_http_status(409)
|
||||||
|
expect(response.headers).not_to have_key('X-GitLab-Last-Update')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when project and pipeline have multiple jobs' do
|
||||||
|
let!(:test_job) { create(:ci_build, pipeline: pipeline, name: 'deploy', stage: 'deploy', stage_idx: 1) }
|
||||||
|
|
||||||
|
before { job.success }
|
||||||
|
|
||||||
|
it 'returns dependent jobs' do
|
||||||
|
request_job
|
||||||
|
|
||||||
|
expect(response).to have_http_status(201)
|
||||||
|
expect(json_response['id']).to eq(test_job.id)
|
||||||
|
expect(json_response['depends_on_builds'].count).to eq(1)
|
||||||
|
expect(json_response['depends_on_builds'][0]).to include('id' => job.id, 'name' => 'spinach')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when job has no tags' do
|
||||||
|
before { job.update(tags: []) }
|
||||||
|
|
||||||
|
context 'when runner is allowed to pick untagged jobs' do
|
||||||
|
before { runner.update_column(:run_untagged, true) }
|
||||||
|
|
||||||
|
it 'picks job' do
|
||||||
|
request_job
|
||||||
|
expect(response).to have_http_status 201
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when runner is not allowed to pick untagged jobs' do
|
||||||
|
before { runner.update_column(:run_untagged, false) }
|
||||||
|
it_behaves_like 'no jobs available'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when triggered job is available' do
|
||||||
|
before do
|
||||||
|
trigger = create(:ci_trigger, project: project)
|
||||||
|
create(:ci_trigger_request_with_variables, pipeline: pipeline, builds: [job], trigger: trigger)
|
||||||
|
project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'returns variables for triggers' do
|
||||||
|
request_job
|
||||||
|
|
||||||
|
expect(response).to have_http_status(201)
|
||||||
|
expect(json_response['variables']).to include(
|
||||||
|
{'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true},
|
||||||
|
{'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true},
|
||||||
|
{'key' => 'CI_BUILD_TRIGGERED', 'value' => 'true', 'public' => true},
|
||||||
|
{'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true},
|
||||||
|
{'key' => 'SECRET_KEY', 'value' => 'secret_value', 'public' => false},
|
||||||
|
{'key' => 'TRIGGER_KEY_1', 'value' => 'TRIGGER_VALUE_1', 'public' => false},
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'registry credentials support' do
|
||||||
|
let(:registry_url) { 'registry.example.com:5005' }
|
||||||
|
let(:registry_credentials) do
|
||||||
|
{'type' => 'registry',
|
||||||
|
'url' => registry_url,
|
||||||
|
'username' => 'gitlab-ci-token',
|
||||||
|
'password' => job.token}
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when registry is enabled' do
|
||||||
|
before { stub_container_registry_config(enabled: true, host_port: registry_url) }
|
||||||
|
|
||||||
|
it 'sends registry credentials key' do
|
||||||
|
request_job
|
||||||
|
expect(json_response).to have_key('credentials')
|
||||||
|
expect(json_response['credentials']).to include(registry_credentials)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when registry is disabled' do
|
||||||
|
before { stub_container_registry_config(enabled: false, host_port: registry_url) }
|
||||||
|
|
||||||
|
it 'does not send registry credentials' do
|
||||||
|
request_job
|
||||||
|
expect(json_response).to have_key('credentials')
|
||||||
|
expect(json_response['credentials']).not_to include(registry_credentials)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def request_job(token = runner.token, **params)
|
||||||
|
new_params = params.merge(token: token, last_update: last_update)
|
||||||
|
post api('/jobs/request'), new_params, {'User-Agent' => user_agent}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue