Merge branch 'lock-pipeline-schedule-worker' into 'master'
Prevent concurrent execution of PipelineScheduleWorker Closes gitlab-com/gl-infra/production#805 See merge request gitlab-org/gitlab-ce!27781
This commit is contained in:
commit
ca8e5aded8
|
@ -3,20 +3,26 @@
|
|||
class PipelineScheduleWorker
|
||||
include ApplicationWorker
|
||||
include CronjobQueue
|
||||
include ::Gitlab::ExclusiveLeaseHelpers
|
||||
|
||||
EXCLUSIVE_LOCK_KEY = 'pipeline_schedules:run:lock'
|
||||
LOCK_TIMEOUT = 50.minutes
|
||||
|
||||
# rubocop: disable CodeReuse/ActiveRecord
|
||||
def perform
|
||||
Ci::PipelineSchedule.active.where("next_run_at < ?", Time.now)
|
||||
.preload(:owner, :project).find_each do |schedule|
|
||||
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
|
||||
Ci::PipelineSchedule.active.where("next_run_at < ?", Time.now)
|
||||
.preload(:owner, :project).find_each do |schedule|
|
||||
|
||||
Ci::CreatePipelineService.new(schedule.project,
|
||||
schedule.owner,
|
||||
ref: schedule.ref)
|
||||
.execute!(:schedule, ignore_skip_ci: true, save_on_errors: true, schedule: schedule)
|
||||
rescue => e
|
||||
error(schedule, e)
|
||||
ensure
|
||||
schedule.schedule_next_run!
|
||||
schedule.schedule_next_run!
|
||||
|
||||
Ci::CreatePipelineService.new(schedule.project,
|
||||
schedule.owner,
|
||||
ref: schedule.ref)
|
||||
.execute!(:schedule, ignore_skip_ci: true, save_on_errors: true, schedule: schedule)
|
||||
rescue => e
|
||||
error(schedule, e)
|
||||
end
|
||||
end
|
||||
end
|
||||
# rubocop: enable CodeReuse/ActiveRecord
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
title: Prevent concurrent execution of PipelineScheduleWorker
|
||||
merge_request: 27781
|
||||
author:
|
||||
type: performance
|
|
@ -3,6 +3,8 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe PipelineScheduleWorker do
|
||||
include ExclusiveLeaseHelpers
|
||||
|
||||
subject { described_class.new.perform }
|
||||
|
||||
set(:project) { create(:project, :repository) }
|
||||
|
@ -39,6 +41,16 @@ describe PipelineScheduleWorker do
|
|||
|
||||
it_behaves_like 'successful scheduling'
|
||||
|
||||
context 'when exclusive lease has already been taken by the other instance' do
|
||||
before do
|
||||
stub_exclusive_lease_taken(described_class::EXCLUSIVE_LOCK_KEY, timeout: described_class::LOCK_TIMEOUT)
|
||||
end
|
||||
|
||||
it 'raises an error and does not start creating pipelines' do
|
||||
expect { subject }.to raise_error(Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the latest commit contains [ci skip]' do
|
||||
before do
|
||||
allow_any_instance_of(Ci::Pipeline)
|
||||
|
|
Loading…
Reference in New Issue