From 50989bca2e21cb716e28bf3b7b552499a47abf4c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 6 Jun 2018 18:03:09 +0900 Subject: [PATCH] Add metric for failed archiving with stale live trace --- app/workers/ci/rescue_stale_live_trace_worker.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/workers/ci/rescue_stale_live_trace_worker.rb b/app/workers/ci/rescue_stale_live_trace_worker.rb index 156ed61c2c5..3f111d9d862 100644 --- a/app/workers/ci/rescue_stale_live_trace_worker.rb +++ b/app/workers/ci/rescue_stale_live_trace_worker.rb @@ -7,10 +7,13 @@ module Ci # Archive stale live traces which still resides in redis or database # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL # More details in https://gitlab.com/gitlab-org/gitlab-ce/issues/36791 + failed_archive_counter = Gitlab::Metrics.counter(:job_stale_live_trace_failed_archive_total, "Counter of failed archiving with stale live trace") + Ci::Build.finished.with_live_trace.find_each(batch_size: 100) do |build| begin build.trace.archive! rescue => e + failed_archive_counter.increment Rails.logger.error "Failed to archive stale live trace. id: #{build.id} message: #{e.message}" end end