2017-03-24 11:08:34 +00:00
class StuckImportJobsWorker
2017-11-28 16:08:30 +00:00
include ApplicationWorker
2017-03-24 11:08:34 +00:00
include CronjobQueue
2017-08-17 09:00:31 +00:00
IMPORT_JOBS_EXPIRATION = 15 . hours . to_i
2017-03-24 11:08:34 +00:00
def perform
2017-08-17 09:00:31 +00:00
projects_without_jid_count = mark_projects_without_jid_as_failed!
projects_with_jid_count = mark_projects_with_jid_as_failed!
Gitlab :: Metrics . add_event ( :stuck_import_jobs ,
projects_without_jid_count : projects_without_jid_count ,
projects_with_jid_count : projects_with_jid_count )
end
private
def mark_projects_without_jid_as_failed!
2017-10-03 14:39:04 +00:00
enqueued_projects_without_jid . each do | project |
2017-08-17 09:00:31 +00:00
project . mark_import_as_failed ( error_message )
end . count
end
def mark_projects_with_jid_as_failed!
2018-05-02 13:35:04 +00:00
# TODO: Rollback this change to use SQL through #pluck
jids_and_ids = enqueued_projects_with_jid . map { | project | [ project . import_jid , project . id ] } . to_h
2017-08-17 09:00:31 +00:00
2018-02-23 15:25:48 +00:00
# Find the jobs that aren't currently running or that exceeded the threshold.
completed_jids = Gitlab :: SidekiqStatus . completed_jids ( jids_and_ids . keys )
return unless completed_jids . any?
2017-03-24 11:08:34 +00:00
2018-02-23 15:25:48 +00:00
completed_project_ids = jids_and_ids . values_at ( * completed_jids )
2017-03-24 11:08:34 +00:00
2018-02-23 15:25:48 +00:00
# We select the projects again, because they may have transitioned from
# scheduled/started to finished/failed while we were looking up their Sidekiq status.
completed_projects = enqueued_projects_with_jid . where ( id : completed_project_ids )
2017-03-24 11:08:34 +00:00
2018-02-23 15:25:48 +00:00
Rails . logger . info ( " Marked stuck import jobs as failed. JIDs: #{ completed_projects . map ( & :import_jid ) . join ( ', ' ) } " )
2017-03-24 11:08:34 +00:00
2018-02-23 15:25:48 +00:00
completed_projects . each do | project |
project . mark_import_as_failed ( error_message )
end . count
2017-08-17 09:00:31 +00:00
end
2017-03-24 11:08:34 +00:00
2017-10-03 14:39:04 +00:00
def enqueued_projects
2018-05-02 13:35:04 +00:00
Project . joins_import_state . where ( " (import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started') " )
2017-03-24 11:08:34 +00:00
end
2017-10-03 14:39:04 +00:00
def enqueued_projects_with_jid
2018-05-02 13:35:04 +00:00
enqueued_projects . where . not ( " import_state.jid IS NULL AND projects.import_jid IS NULL " )
2017-08-17 09:00:31 +00:00
end
2017-03-24 11:08:34 +00:00
2017-10-03 14:39:04 +00:00
def enqueued_projects_without_jid
2018-05-02 13:35:04 +00:00
enqueued_projects . where ( " import_state.jid IS NULL AND projects.import_jid IS NULL " )
2017-03-24 11:08:34 +00:00
end
def error_message
2017-08-17 09:00:31 +00:00
" Import timed out. Import took longer than #{ IMPORT_JOBS_EXPIRATION } seconds "
2017-03-24 11:08:34 +00:00
end
end