2018-11-29 00:34:14 -05:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
module Ci
|
|
|
|
class ArchiveTraceService
|
2019-07-18 05:22:46 -04:00
|
|
|
def execute(job, worker_name:)
|
|
|
|
# TODO: Remove this logging once we confirmed new live trace architecture is functional.
|
|
|
|
# See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
|
|
|
|
unless job.has_live_trace?
|
|
|
|
Sidekiq.logger.warn(class: worker_name,
|
|
|
|
message: 'The job does not have live trace but going to be archived.',
|
|
|
|
job_id: job.id)
|
|
|
|
return
|
|
|
|
end
|
|
|
|
|
2018-11-29 00:34:14 -05:00
|
|
|
job.trace.archive!
|
2020-09-15 14:09:43 -04:00
|
|
|
job.remove_pending_state!
|
2019-07-18 05:22:46 -04:00
|
|
|
|
|
|
|
# TODO: Remove this logging once we confirmed new live trace architecture is functional.
|
|
|
|
# See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
|
|
|
|
unless job.has_archived_trace?
|
|
|
|
Sidekiq.logger.warn(class: worker_name,
|
|
|
|
message: 'The job does not have archived trace after archiving.',
|
|
|
|
job_id: job.id)
|
|
|
|
end
|
2018-11-29 00:34:14 -05:00
|
|
|
rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
|
|
|
|
# It's already archived, thus we can safely ignore this exception.
|
2021-04-26 08:09:44 -04:00
|
|
|
rescue StandardError => e
|
2018-11-29 01:13:36 -05:00
|
|
|
# Tracks this error with application logs, Sentry, and Prometheus.
|
|
|
|
# If `archive!` keeps failing for over a week, that could incur data loss.
|
2020-10-05 05:08:17 -04:00
|
|
|
# (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
|
2018-11-29 01:13:36 -05:00
|
|
|
# In order to avoid interrupting the system, we do not raise an exception here.
|
2019-07-18 05:22:46 -04:00
|
|
|
archive_error(e, job, worker_name)
|
2018-11-29 00:34:14 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
def failed_archive_counter
|
2018-11-29 01:13:36 -05:00
|
|
|
@failed_archive_counter ||=
|
|
|
|
Gitlab::Metrics.counter(:job_trace_archive_failed_total,
|
|
|
|
"Counter of failed attempts of trace archiving")
|
2018-11-29 00:34:14 -05:00
|
|
|
end
|
|
|
|
|
2019-07-18 05:22:46 -04:00
|
|
|
def archive_error(error, job, worker_name)
|
2018-11-29 00:34:14 -05:00
|
|
|
failed_archive_counter.increment
|
2019-07-18 05:22:46 -04:00
|
|
|
|
|
|
|
Sidekiq.logger.warn(class: worker_name,
|
|
|
|
message: "Failed to archive trace. message: #{error.message}.",
|
|
|
|
job_id: job.id)
|
2018-11-29 01:13:36 -05:00
|
|
|
|
2019-12-16 07:07:43 -05:00
|
|
|
Gitlab::ErrorTracking
|
2019-12-13 07:07:41 -05:00
|
|
|
.track_and_raise_for_dev_exception(error,
|
2019-09-18 10:02:45 -04:00
|
|
|
issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/51502',
|
2019-12-13 07:07:41 -05:00
|
|
|
job_id: job.id )
|
2018-11-29 00:34:14 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|