Rescue stale live traces

app/workers/all_queues.yml

@@ -17,6 +17,7 @@
 - cronjob:stuck_ci_jobs
 - cronjob:stuck_import_jobs
 - cronjob:stuck_merge_jobs
+- cronjob:fource_archive_stale_live_trace
 - cronjob:trending_projects
 - cronjob:issue_due_scheduler
 

app/workers/fource_archive_stale_live_trace_worker.rb

@@ -0,0 +1,23 @@
+class FourceArchiveStaleLiveTraceWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    # Find jobs with the following condition
+    # - Finished 4 hours ago (Jobs finished 4 hours ago should have an archived trace)
+    # - Not archived yet (Because ArchiveTraceWorker failed by some reason)
+    Ci::Build.finished
+      .where('finished_at < ?', 4.hours.ago)
+      .where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      .find_each(batch_size: 1000) do |job|
+        begin
+          job.trace.archive!
+        rescue => e
+          Rails.logger.error "#{job.id}: Failed to archive stable live trace: #{e.message}"
+        end
+
+        Rails.logger.warning "#{job.id}: Live trace was force archived because it was considered as stale"
+      end
+    end
+  end
+end

config/initializers/1_settings.rb

@@ -289,6 +289,9 @@ Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'Repository
 Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
+Settings.cron_jobs['fource_archive_stale_live_trace_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['fource_archive_stale_live_trace_worker']['cron'] ||= '*/4 * * * *'
+Settings.cron_jobs['fource_archive_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
 Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'