2018-06-27 07:23:28 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-03-24 11:08:34 +00:00
|
|
|
class StuckImportJobsWorker
|
2017-11-28 16:08:30 +00:00
|
|
|
include ApplicationWorker
|
2017-03-24 11:08:34 +00:00
|
|
|
include CronjobQueue
|
|
|
|
|
2017-08-17 09:00:31 +00:00
|
|
|
IMPORT_JOBS_EXPIRATION = 15.hours.to_i
|
2017-03-24 11:08:34 +00:00
|
|
|
|
|
|
|
def perform
|
2018-11-07 14:32:15 +00:00
|
|
|
import_state_without_jid_count = mark_import_states_without_jid_as_failed!
|
|
|
|
import_state_with_jid_count = mark_import_states_with_jid_as_failed!
|
2017-08-17 09:00:31 +00:00
|
|
|
|
|
|
|
Gitlab::Metrics.add_event(:stuck_import_jobs,
|
2018-11-07 14:32:15 +00:00
|
|
|
projects_without_jid_count: import_state_without_jid_count,
|
|
|
|
projects_with_jid_count: import_state_with_jid_count)
|
2017-08-17 09:00:31 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
def mark_import_states_without_jid_as_failed!
|
|
|
|
enqueued_import_states_without_jid.each do |import_state|
|
|
|
|
import_state.mark_as_failed(error_message)
|
2017-08-17 09:00:31 +00:00
|
|
|
end.count
|
|
|
|
end
|
|
|
|
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
2018-11-07 14:32:15 +00:00
|
|
|
def mark_import_states_with_jid_as_failed!
|
|
|
|
jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h
|
2017-08-17 09:00:31 +00:00
|
|
|
|
2018-02-23 15:25:48 +00:00
|
|
|
# Find the jobs that aren't currently running or that exceeded the threshold.
|
|
|
|
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
|
|
|
|
return unless completed_jids.any?
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
completed_import_state_ids = jids_and_ids.values_at(*completed_jids)
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
# We select the import states again, because they may have transitioned from
|
2018-02-23 15:25:48 +00:00
|
|
|
# scheduled/started to finished/failed while we were looking up their Sidekiq status.
|
2018-11-07 14:32:15 +00:00
|
|
|
completed_import_states = enqueued_import_states_with_jid.where(id: completed_import_state_ids)
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ')
|
2019-07-10 19:26:47 +00:00
|
|
|
Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}") # rubocop:disable Gitlab/RailsLogger
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
completed_import_states.each do |import_state|
|
|
|
|
import_state.mark_as_failed(error_message)
|
2018-02-23 15:25:48 +00:00
|
|
|
end.count
|
2017-08-17 09:00:31 +00:00
|
|
|
end
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-11-07 14:32:15 +00:00
|
|
|
def enqueued_import_states
|
|
|
|
ProjectImportState.with_status([:scheduled, :started])
|
2017-03-24 11:08:34 +00:00
|
|
|
end
|
|
|
|
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
2018-11-07 14:32:15 +00:00
|
|
|
def enqueued_import_states_with_jid
|
|
|
|
enqueued_import_states.where.not(jid: nil)
|
2017-08-17 09:00:31 +00:00
|
|
|
end
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
2017-03-24 11:08:34 +00:00
|
|
|
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
2018-11-07 14:32:15 +00:00
|
|
|
def enqueued_import_states_without_jid
|
|
|
|
enqueued_import_states.where(jid: nil)
|
2017-03-24 11:08:34 +00:00
|
|
|
end
|
2018-08-27 15:31:01 +00:00
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
2017-03-24 11:08:34 +00:00
|
|
|
|
|
|
|
def error_message
|
2018-11-27 09:41:27 +00:00
|
|
|
_("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION }
|
2017-03-24 11:08:34 +00:00
|
|
|
end
|
|
|
|
end
|