2019-03-30 07:15:48 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-02-07 13:00:46 +00:00
|
|
|
require 'spec_helper'
|
2015-10-21 09:29:47 +00:00
|
|
|
|
2020-06-26 09:08:59 +00:00
|
|
|
RSpec.describe StuckCiJobsWorker do
|
2018-06-28 18:51:36 +00:00
|
|
|
include ExclusiveLeaseHelpers
|
|
|
|
|
2017-02-07 13:00:46 +00:00
|
|
|
let!(:runner) { create :ci_runner }
|
2017-03-02 23:43:39 +00:00
|
|
|
let!(:job) { create :ci_build, runner: runner }
|
2018-06-28 18:51:36 +00:00
|
|
|
let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY }
|
|
|
|
let(:worker_lease_uuid) { SecureRandom.uuid }
|
2021-06-04 15:10:25 +00:00
|
|
|
let(:created_at) { }
|
|
|
|
let(:updated_at) { }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
|
|
|
subject(:worker) { described_class.new }
|
2015-10-21 09:29:47 +00:00
|
|
|
|
2017-02-07 22:06:16 +00:00
|
|
|
before do
|
2018-06-28 18:51:36 +00:00
|
|
|
stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
|
2021-06-04 15:10:25 +00:00
|
|
|
job_attributes = { status: status }
|
|
|
|
job_attributes[:created_at] = created_at if created_at
|
|
|
|
job_attributes[:updated_at] = updated_at if updated_at
|
|
|
|
job.update!(job_attributes)
|
2017-02-07 22:06:16 +00:00
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
shared_examples 'job is dropped' do
|
2019-11-01 00:06:02 +00:00
|
|
|
it "changes status" do
|
2017-02-07 13:00:46 +00:00
|
|
|
worker.perform
|
2017-09-02 08:06:25 +00:00
|
|
|
job.reload
|
|
|
|
|
|
|
|
expect(job).to be_failed
|
2017-08-31 13:03:41 +00:00
|
|
|
expect(job).to be_stuck_or_timeout_failure
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
2019-11-01 00:06:02 +00:00
|
|
|
|
|
|
|
context 'when job have data integrity problem' do
|
|
|
|
it "does drop the job and logs the reason" do
|
|
|
|
job.update_columns(yaml_variables: '[{"key" => "value"}]')
|
|
|
|
|
2019-12-16 12:07:43 +00:00
|
|
|
expect(Gitlab::ErrorTracking).to receive(:track_exception)
|
2019-12-13 12:07:41 +00:00
|
|
|
.with(anything, a_hash_including(build_id: job.id))
|
2019-11-01 00:06:02 +00:00
|
|
|
.once
|
|
|
|
.and_call_original
|
|
|
|
|
|
|
|
worker.perform
|
|
|
|
job.reload
|
|
|
|
|
|
|
|
expect(job).to be_failed
|
|
|
|
expect(job).to be_data_integrity_failure
|
|
|
|
end
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
shared_examples 'job is unchanged' do
|
2017-09-02 08:06:25 +00:00
|
|
|
before do
|
2017-02-07 13:00:46 +00:00
|
|
|
worker.perform
|
2017-09-02 08:06:25 +00:00
|
|
|
job.reload
|
|
|
|
end
|
|
|
|
|
|
|
|
it "doesn't change status" do
|
|
|
|
expect(job.status).to eq(status)
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is pending' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:status) { 'pending' }
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is not stuck' do
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false)
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when job was updated_at more than 1 day ago' do
|
|
|
|
let(:updated_at) { 1.5.days.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 1.5.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 3.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when job was updated less than 1 day ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 6.hours.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 1.5.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 3.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when job was updated more than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 2.hours.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 2.hours.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 3.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-03-02 23:43:39 +00:00
|
|
|
context 'when job is stuck' do
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true)
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when job was updated_at more than 1 hour ago' do
|
|
|
|
let(:updated_at) { 1.5.hours.ago }
|
|
|
|
|
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 1.5.hours.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 3.days.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when job was updated in less than 1 hour ago' do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 30.minutes.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 2.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2015-10-21 09:29:47 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2016-06-12 13:15:58 +00:00
|
|
|
|
2021-08-28 15:11:11 +00:00
|
|
|
context 'when job is running' do
|
|
|
|
let(:status) { 'running' }
|
2021-08-28 06:10:17 +00:00
|
|
|
|
2021-08-28 15:11:11 +00:00
|
|
|
context 'when job was updated_at more than an hour ago' do
|
|
|
|
let(:updated_at) { 2.hours.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-08-28 15:11:11 +00:00
|
|
|
it_behaves_like 'job is dropped'
|
2016-06-12 13:15:58 +00:00
|
|
|
end
|
|
|
|
|
2021-08-28 15:11:11 +00:00
|
|
|
context 'when job was updated in less than 1 hour ago' do
|
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-08-28 15:11:11 +00:00
|
|
|
it_behaves_like 'job is unchanged'
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
%w(success skipped failed canceled).each do |status|
|
2017-03-02 23:43:39 +00:00
|
|
|
context "when job is #{status}" do
|
2017-02-07 13:00:46 +00:00
|
|
|
let(:status) { status }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2021-06-04 15:10:25 +00:00
|
|
|
context 'when created_at is the same as updated_at' do
|
|
|
|
let(:created_at) { 2.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is before updated_at' do
|
|
|
|
let(:created_at) { 3.days.ago }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when created_at is outside lookback window' do
|
|
|
|
let(:created_at) { described_class::BUILD_LOOKBACK - 1.day }
|
|
|
|
|
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'for deleted project' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
|
|
|
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
2020-08-03 18:10:05 +00:00
|
|
|
job.project.update!(pending_delete: true)
|
2017-06-14 18:18:56 +00:00
|
|
|
end
|
2017-02-07 13:00:46 +00:00
|
|
|
|
2017-11-24 15:30:08 +00:00
|
|
|
it 'does drop job' do
|
2017-11-27 10:56:06 +00:00
|
|
|
expect_any_instance_of(Ci::Build).to receive(:drop).and_call_original
|
2016-06-12 13:15:58 +00:00
|
|
|
worker.perform
|
|
|
|
end
|
|
|
|
end
|
2017-02-07 22:06:16 +00:00
|
|
|
|
2018-10-03 09:52:18 +00:00
|
|
|
describe 'drop stale scheduled builds' do
|
2018-09-27 09:17:43 +00:00
|
|
|
let(:status) { 'scheduled' }
|
|
|
|
let(:updated_at) { }
|
|
|
|
|
|
|
|
context 'when scheduled at 2 hours ago but it is not executed yet' do
|
|
|
|
let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) }
|
|
|
|
|
|
|
|
it 'drops the stale scheduled build' do
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
|
|
|
|
worker.perform
|
|
|
|
job.reload
|
|
|
|
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(0)
|
|
|
|
expect(job).to be_failed
|
2018-10-02 09:10:58 +00:00
|
|
|
expect(job).to be_stale_schedule
|
2018-09-27 09:17:43 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when scheduled at 30 minutes ago but it is not executed yet' do
|
|
|
|
let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) }
|
|
|
|
|
|
|
|
it 'does not drop the stale scheduled build yet' do
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
|
|
|
|
worker.perform
|
|
|
|
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when there are no stale scheduled builds' do
|
|
|
|
it 'does not drop the stale scheduled build yet' do
|
|
|
|
expect { worker.perform }.not_to raise_error
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-02-07 22:06:16 +00:00
|
|
|
describe 'exclusive lease' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2017-02-09 02:29:38 +00:00
|
|
|
let(:worker2) { described_class.new }
|
2017-02-07 22:06:16 +00:00
|
|
|
|
2017-02-09 02:29:38 +00:00
|
|
|
it 'is guard by exclusive lease when executed concurrently' do
|
2017-11-27 10:56:06 +00:00
|
|
|
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
2017-02-09 02:29:38 +00:00
|
|
|
expect(worker2).not_to receive(:drop)
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2017-02-07 22:06:16 +00:00
|
|
|
worker.perform
|
2018-06-28 18:51:36 +00:00
|
|
|
|
|
|
|
stub_exclusive_lease_taken(worker_lease_key)
|
|
|
|
|
2017-02-09 02:29:38 +00:00
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'can be executed in sequence' do
|
2017-11-27 10:56:06 +00:00
|
|
|
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
|
|
|
expect(worker2).to receive(:drop).at_least(:once).and_call_original
|
2018-06-28 18:51:36 +00:00
|
|
|
|
2017-02-09 02:29:38 +00:00
|
|
|
worker.perform
|
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
2018-06-28 18:51:36 +00:00
|
|
|
it 'cancels exclusive leases after worker perform' do
|
|
|
|
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
|
2018-06-01 07:15:41 +00:00
|
|
|
|
2018-06-28 18:51:36 +00:00
|
|
|
worker.perform
|
2017-02-07 22:06:16 +00:00
|
|
|
end
|
|
|
|
end
|
2017-03-01 12:56:54 +00:00
|
|
|
end
|