Merge branch 'optimise-stuck-ci-jobs-worker' into 'master'
Optimise StuckCiJobsWorker Closes gitlab-com/infrastructure#3260 See merge request gitlab-org/gitlab-ce!15527
This commit is contained in:
commit
420cdc6a52
|
@ -104,6 +104,7 @@ module Ci
|
|||
end
|
||||
|
||||
before_transition any => [:failed] do |build|
|
||||
next unless build.project
|
||||
next if build.retries_max.zero?
|
||||
|
||||
if build.retries_count < build.retries_max
|
||||
|
|
|
@ -17,6 +17,7 @@ class CommitStatus < ActiveRecord::Base
|
|||
validates :name, presence: true, unless: :importing?
|
||||
|
||||
alias_attribute :author, :user
|
||||
alias_attribute :pipeline_id, :commit_id
|
||||
|
||||
scope :failed_but_allowed, -> do
|
||||
where(allow_failure: true, status: [:failed, :canceled])
|
||||
|
@ -103,26 +104,29 @@ class CommitStatus < ActiveRecord::Base
|
|||
end
|
||||
|
||||
after_transition do |commit_status, transition|
|
||||
next unless commit_status.project
|
||||
next if transition.loopback?
|
||||
|
||||
commit_status.run_after_commit do
|
||||
if pipeline
|
||||
if pipeline_id
|
||||
if complete? || manual?
|
||||
PipelineProcessWorker.perform_async(pipeline.id)
|
||||
PipelineProcessWorker.perform_async(pipeline_id)
|
||||
else
|
||||
PipelineUpdateWorker.perform_async(pipeline.id)
|
||||
PipelineUpdateWorker.perform_async(pipeline_id)
|
||||
end
|
||||
end
|
||||
|
||||
StageUpdateWorker.perform_async(commit_status.stage_id)
|
||||
ExpireJobCacheWorker.perform_async(commit_status.id)
|
||||
StageUpdateWorker.perform_async(stage_id)
|
||||
ExpireJobCacheWorker.perform_async(id)
|
||||
end
|
||||
end
|
||||
|
||||
after_transition any => :failed do |commit_status|
|
||||
next unless commit_status.project
|
||||
|
||||
commit_status.run_after_commit do
|
||||
MergeRequests::AddTodoWhenBuildFailsService
|
||||
.new(pipeline.project, nil).execute(self)
|
||||
.new(project, nil).execute(self)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -45,9 +45,17 @@ class StuckCiJobsWorker
|
|||
end
|
||||
|
||||
def search(status, timeout)
|
||||
builds = Ci::Build.where(status: status).where('ci_builds.updated_at < ?', timeout.ago)
|
||||
builds.joins(:project).merge(Project.without_deleted).includes(:tags, :runner, project: :namespace).find_each(batch_size: 50).each do |build|
|
||||
yield(build)
|
||||
loop do
|
||||
jobs = Ci::Build.where(status: status)
|
||||
.where('ci_builds.updated_at < ?', timeout.ago)
|
||||
.includes(:tags, :runner, project: :namespace)
|
||||
.limit(100)
|
||||
.to_a
|
||||
break if jobs.empty?
|
||||
|
||||
jobs.each do |job|
|
||||
yield(job)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
title: Optimise StuckCiJobsWorker using cheap SQL query outside, and expensive inside
|
||||
merge_request:
|
||||
author:
|
||||
type: performance
|
|
@ -105,8 +105,8 @@ describe StuckCiJobsWorker do
|
|||
job.project.update(pending_delete: true)
|
||||
end
|
||||
|
||||
it 'does not drop job' do
|
||||
expect_any_instance_of(Ci::Build).not_to receive(:drop)
|
||||
it 'does drop job' do
|
||||
expect_any_instance_of(Ci::Build).to receive(:drop).and_call_original
|
||||
worker.perform
|
||||
end
|
||||
end
|
||||
|
@ -117,7 +117,7 @@ describe StuckCiJobsWorker do
|
|||
let(:worker2) { described_class.new }
|
||||
|
||||
it 'is guard by exclusive lease when executed concurrently' do
|
||||
expect(worker).to receive(:drop).at_least(:once)
|
||||
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
||||
expect(worker2).not_to receive(:drop)
|
||||
worker.perform
|
||||
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
|
||||
|
@ -125,8 +125,8 @@ describe StuckCiJobsWorker do
|
|||
end
|
||||
|
||||
it 'can be executed in sequence' do
|
||||
expect(worker).to receive(:drop).at_least(:once)
|
||||
expect(worker2).to receive(:drop).at_least(:once)
|
||||
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
||||
expect(worker2).to receive(:drop).at_least(:once).and_call_original
|
||||
worker.perform
|
||||
worker2.perform
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue