diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index 8504ceb2327..a1dd00721b5 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -7,19 +7,29 @@ module Ci rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue => e + # Tracks this error with application logs, Sentry, and Prometheus. + # If `archive!` keeps failing for over a week, that could incur data loss. + # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture) + # In order to avoid interrupting the system, we do not raise an exception here. archive_error(e, job) end private def failed_archive_counter - @failed_archive_counter ||= Gitlab::Metrics.counter(:job_trace_archive_failed_total, "Counter of failed attempts of trace archiving") + @failed_archive_counter ||= + Gitlab::Metrics.counter(:job_trace_archive_failed_total, + "Counter of failed attempts of trace archiving") end def archive_error(error, job) failed_archive_counter.increment - Gitlab::Sentry.track_exception(error, issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', extra: { job_id: job.id }) Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" + + Gitlab::Sentry + .track_exception(error, + issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', + extra: { job_id: job.id }) end end end diff --git a/spec/workers/ci/archive_traces_cron_worker_spec.rb b/spec/workers/ci/archive_traces_cron_worker_spec.rb index 4ba3d436e94..478fb7d2c0f 100644 --- a/spec/workers/ci/archive_traces_cron_worker_spec.rb +++ b/spec/workers/ci/archive_traces_cron_worker_spec.rb @@ -48,6 +48,21 @@ describe Ci::ArchiveTracesCronWorker do expect(build2.job_artifacts_trace).to be_exist end end + + context 'when an unexpected exception happened during archiving' do + let!(:build) { create(:ci_build, :success, :trace_live) } + + before do + allow(Gitlab::Sentry).to receive(:track_exception) + allow_any_instance_of(Gitlab::Ci::Trace).to receive(:archive!).and_raise('Unexpected error') + end + + it 'puts a log' do + expect(Rails.logger).to receive(:error).with("Failed to archive trace. id: #{build.id} message: Unexpected error") + + subject + end + end end context 'when a job was cancelled' do