kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Simplify StuckCiJobsWorker

Browse source
This commit is contained in:
Shinya Maeda 2018-09-26 14:28:22 +09:00 committed by Alessio Caiazza
parent b1d24c0d14
commit c514636a83
2 changed files with 26 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/services/ci/process_pipeline_service.rb
View file

@ -56,7 +56,7 @@ module Ci
when 'manual' when 'manual'
%w[success skipped] %w[success skipped]
when 'delayed' when 'delayed'
%w[success skipped] # This might be `success` only %w[success skipped]
else else
[] []
end end

55
app/workers/stuck_ci_jobs_worker.rb
View file

@ -16,10 +16,10 @@ class StuckCiJobsWorker
Rails.logger.info "#{self.class}: Cleaning stuck builds" Rails.logger.info "#{self.class}: Cleaning stuck builds"
drop :running, condition_for_outdated_running, :stuck_or_timeout_failure drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT
drop :pending, condition_for_outdated_pending, :stuck_or_timeout_failure drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT
drop :scheduled, condition_for_outdated_scheduled, :schedule_expired drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
drop_stuck :pending, condition_for_outdated_pending_stuck, :stuck_or_timeout_failure drop_stale_scheduled_builds
remove_lease remove_lease
end end
@ -34,41 +34,25 @@ class StuckCiJobsWorker
Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
end end
def drop(status, condition, reason) def drop(status, timeout)
search(status, condition) do |build| search(status, timeout) do |build|
drop_build :outdated, build, status, reason drop_build :outdated, build, status, timeout, :stuck_or_timeout_failure
end end
end end
def drop_stuck(status, condition, reason) def drop_stuck(status, timeout)
search(status, condition) do |build| search(status, timeout) do |build|
break unless build.stuck? break unless build.stuck?
drop_build :stuck, build, status, reason drop_build :stuck, build, status, timeout, :stuck_or_timeout_failure
end end
end end
def condition_for_outdated_running
["updated_at < ?", BUILD_RUNNING_OUTDATED_TIMEOUT.ago]
end
def condition_for_outdated_pending
["updated_at < ?", BUILD_PENDING_OUTDATED_TIMEOUT.ago]
end
def condition_for_outdated_scheduled
["scheduled_at <> '' && scheduled_at < ?", BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago]
end
def condition_for_outdated_pending_stuck
["updated_at < ?", BUILD_PENDING_STUCK_TIMEOUT.ago]
end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def search(status, condition) def search(status, timeout)
loop do loop do
jobs = Ci::Build.where(status: status) jobs = Ci::Build.where(status: status)
.where(*condition) .where('ci_builds.updated_at < ?', timeout.ago)
.includes(:tags, :runner, project: :namespace) .includes(:tags, :runner, project: :namespace)
.limit(100) .limit(100)
.to_a .to_a
@ -81,10 +65,21 @@ class StuckCiJobsWorker
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def drop_build(type, build, status, reason) def drop_build(type, build, status, timeout, reason)
Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status})" Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})"
Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| Gitlab::OptimisticLocking.retry_lock(build, 3) do |b|
b.drop(reason) b.drop(reason)
end end
end end
def drop_stale_scheduled_builds
# `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition.
# Therefore this query's first step uses Index Search, and the following expensive
# filter `scheduled_at < ?` will only perform on a small subset (max: 100 rows)
Ci::Build.include(EachBach).where('scheduled_at <> NULL').each_batch(of: 100) do |relation|
relation.where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago).find_each do |build|
drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired)
end
end
end
end end