2017-02-07 13:00:46 +00:00
|
|
|
require 'spec_helper'
|
2015-10-21 09:29:47 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
describe StuckCiJobsWorker do
|
2017-02-07 13:00:46 +00:00
|
|
|
let!(:runner) { create :ci_runner }
|
2017-03-02 23:43:39 +00:00
|
|
|
let!(:job) { create :ci_build, runner: runner }
|
2016-06-12 13:15:58 +00:00
|
|
|
let(:worker) { described_class.new }
|
2017-02-09 02:29:38 +00:00
|
|
|
let(:exclusive_lease_uuid) { SecureRandom.uuid }
|
2015-10-21 09:29:47 +00:00
|
|
|
|
|
|
|
subject do
|
2017-03-02 23:43:39 +00:00
|
|
|
job.reload
|
|
|
|
job.status
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2017-02-07 22:06:16 +00:00
|
|
|
before do
|
2017-03-02 23:43:39 +00:00
|
|
|
job.update!(status: status, updated_at: updated_at)
|
2017-02-09 02:29:38 +00:00
|
|
|
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid)
|
2017-02-07 22:06:16 +00:00
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
shared_examples 'job is dropped' do
|
2017-02-07 13:00:46 +00:00
|
|
|
it 'changes status' do
|
|
|
|
worker.perform
|
|
|
|
is_expected.to eq('failed')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
shared_examples 'job is unchanged' do
|
2017-02-07 13:00:46 +00:00
|
|
|
it "doesn't change status" do
|
|
|
|
worker.perform
|
|
|
|
is_expected.to eq(status)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is pending' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:status) { 'pending' }
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is not stuck' do
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false)
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was not updated for more than 1 day ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 2.days.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is dropped'
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was updated in less than 1 day ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 6.hours.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 2.hours.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is stuck' do
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true)
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 2.hours.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is dropped'
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was updated in less than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2016-06-12 13:15:58 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is running' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:status) { 'running' }
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 2.hours.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is dropped'
|
2016-06-12 13:15:58 +00:00
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job was updated in less than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
%w(success skipped failed canceled).each do |status|
|
2017-03-02 23:43:39 +00:00
|
|
|
context "when job is #{status}" do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:status) { status }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2017-03-02 23:43:39 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'for deleted project' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
|
|
|
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
|
|
|
job.project.update(pending_delete: true)
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
it 'does not drop job' do
|
2016-06-12 13:15:58 +00:00
|
|
|
expect_any_instance_of(Ci::Build).not_to receive(:drop)
|
|
|
|
worker.perform
|
|
|
|
end
|
|
|
|
end
|
2017-02-07 22:06:16 +00:00
|
|
|
|
|
|
|
describe 'exclusive lease' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2017-02-09 02:29:38 +00:00
|
|
|
let(:worker2) { described_class.new }
|
2017-02-07 22:06:16 +00:00
|
|
|
|
2017-02-09 02:29:38 +00:00
|
|
|
it 'is guard by exclusive lease when executed concurrently' do
|
|
|
|
expect(worker).to receive(:drop).at_least(:once)
|
|
|
|
expect(worker2).not_to receive(:drop)
|
2017-02-07 22:06:16 +00:00
|
|
|
worker.perform
|
|
|
|
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
|
2017-02-09 02:29:38 +00:00
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'can be executed in sequence' do
|
|
|
|
expect(worker).to receive(:drop).at_least(:once)
|
|
|
|
expect(worker2).to receive(:drop).at_least(:once)
|
|
|
|
worker.perform
|
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'cancels exclusive lease after worker perform' do
|
|
|
|
expect(Gitlab::ExclusiveLease).to receive(:cancel).with(described_class::EXCLUSIVE_LEASE_KEY, exclusive_lease_uuid)
|
2017-02-07 22:06:16 +00:00
|
|
|
worker.perform
|
|
|
|
end
|
|
|
|
end
|
2017-03-01 12:56:54 +00:00
|
|
|
end
|