From 25810d03e2e6ff4039f4d45112fe7f23f0679b57 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Wed, 2 May 2018 23:14:06 +0900
Subject: [PATCH] Added flush-to-db process for the cron worker. Rename to
 RescueStaleLiveTraceWorker.

---
 app/workers/all_queues.yml                    |  2 +-
 .../fource_archive_stale_live_trace_worker.rb | 23 -----------
 app/workers/rescue_stale_live_trace_worker.rb | 39 +++++++++++++++++++
 config/initializers/1_settings.rb             |  6 +--
 4 files changed, 43 insertions(+), 27 deletions(-)
 delete mode 100644 app/workers/fource_archive_stale_live_trace_worker.rb
 create mode 100644 app/workers/rescue_stale_live_trace_worker.rb

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 262da7598dc..85d9dcbc3c9 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -17,7 +17,7 @@
 - cronjob:stuck_ci_jobs
 - cronjob:stuck_import_jobs
 - cronjob:stuck_merge_jobs
-- cronjob:fource_archive_stale_live_trace
+- cronjob:rescue_stale_live_trace
 - cronjob:trending_projects
 - cronjob:issue_due_scheduler
 
diff --git a/app/workers/fource_archive_stale_live_trace_worker.rb b/app/workers/fource_archive_stale_live_trace_worker.rb
deleted file mode 100644
index 7398d5542ba..00000000000
--- a/app/workers/fource_archive_stale_live_trace_worker.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-class FourceArchiveStaleLiveTraceWorker
-  include ApplicationWorker
-  include CronjobQueue
-
-  def perform
-    # Find jobs with the following condition
-    # - Finished 4 hours ago (Jobs finished 4 hours ago should have an archived trace)
-    # - Not archived yet (Because ArchiveTraceWorker failed by some reason)
-    Ci::Build.finished
-      .where('finished_at < ?', 4.hours.ago)
-      .where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
-      .find_each(batch_size: 1000) do |job|
-        begin
-          job.trace.archive!
-        rescue => e
-          Rails.logger.error "#{job.id}: Failed to archive stable live trace: #{e.message}"
-        end
-
-        Rails.logger.warning "#{job.id}: Live trace was force archived because it was considered as stale"
-      end
-    end
-  end
-end
diff --git a/app/workers/rescue_stale_live_trace_worker.rb b/app/workers/rescue_stale_live_trace_worker.rb
new file mode 100644
index 00000000000..ba0b7c9a90a
--- /dev/null
+++ b/app/workers/rescue_stale_live_trace_worker.rb
@@ -0,0 +1,39 @@
+class RescueStaleLiveTraceWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    # Reschedule to archive live traces
+    #
+    # The target jobs are with the following conditions
+    # - Finished 4 hours ago, but it's not archived yet
+    #   Jobs finished 4 hours ago should have an archived trace. Probably ArchiveTraceWorker failed by Sidekiq's inconsistancy
+    Ci::Build.finished
+      .where('finished_at BETWEEN ? AND ?', 1.week.ago, 4.hours.ago)
+      .where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      .find_in_batch(batch_size: 1000) do |jobs|
+        job_ids = jobs.map { |job| [job.id] }
+
+        ArchiveTraceWorker.bulk_perform_async(job_ids)
+
+        Rails.logger.warning "Scheduled to archive stale live traces from #{job_ids.min} to #{job_ids.max}"
+      end
+
+    # Schedule to flush redis-chunk to database
+    #
+    # The target build_trace_chunks are with the following conditions
+    # - The last patching of the trace was 1 hour ago
+    # - The job is still running
+    Ci::BuildTraceChunk.redis
+      .joins(:build)
+      .where('ci_builds.update_at < ?', 1.hour.ago)
+      .where('ci_builds.status = ?', 'running')
+      .find_in_batch(batch_size: 1000) do |build_trace_chunks|
+        build_trace_chunk_ids = build_trace_chunks.map { |build_trace_chunk| [build_trace_chunk.id] }
+
+        BuildTraceChunkFlushToDBWorker.bulk_perform_async(build_trace_chunk_ids)
+
+        Rails.logger.warning "Scheduled to flush stale live traces to database from #{build_trace_chunk_ids.min} to #{build_trace_chunk_ids.max}"
+      end
+  end
+end
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 095adce82ee..3c1e67f59f1 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -289,9 +289,9 @@ Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'Repository
 Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
-Settings.cron_jobs['fource_archive_stale_live_trace_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['fource_archive_stale_live_trace_worker']['cron'] ||= '*/4 * * * *'
-Settings.cron_jobs['fource_archive_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
+Settings.cron_jobs['rescue_stale_live_trace_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['rescue_stale_live_trace_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['rescue_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
 Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'