Merge branch 'dm-application-worker' into 'master'

Add ApplicationWorker and make every worker include it

See merge request gitlab-org/gitlab-ce!15632

app/models/group.rb

@@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord'
 
 class Group < Namespace
   include Gitlab::ConfigHelper
+  include AfterCommitQueue
   include AccessRequestable
   include Avatarable
   include Referable

app/models/key.rb

@@ -2,6 +2,7 @@ require 'digest/md5'
 
 class Key < ActiveRecord::Base
   include Gitlab::CurrentSettings
+  include AfterCommitQueue
   include Sortable
 
   belongs_to :user

app/models/member.rb

@@ -1,4 +1,5 @@
 class Member < ActiveRecord::Base
+  include AfterCommitQueue
   include Sortable
   include Importable
   include Expirable

app/models/service.rb

@@ -211,7 +211,7 @@ class Service < ActiveRecord::Base
   def async_execute(data)
     return unless supported_events.include?(data[:object_kind])
 
-    Sidekiq::Client.enqueue(ProjectServiceWorker, id, data)
+    ProjectServiceWorker.perform_async(id, data)
   end
 
   def issue_tracker?

app/models/user.rb

@@ -7,6 +7,7 @@ class User < ActiveRecord::Base
   include Gitlab::ConfigHelper
   include Gitlab::CurrentSettings
   include Gitlab::SQL::Pattern
+  include AfterCommitQueue
   include Avatarable
   include Referable
   include Sortable
@@ -903,6 +904,7 @@ class User < ActiveRecord::Base
 
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
+
     system_hook_service.execute_hooks_for(self, :destroy)
   end
 

app/services/system_hooks_service.rb

@@ -1,6 +1,10 @@
 class SystemHooksService
   def execute_hooks_for(model, event)
-    execute_hooks(build_event_data(model, event))
+    data = build_event_data(model, event)
+
+    model.run_after_commit_or_now do
+      SystemHooksService.new.execute_hooks(data)
+    end
   end
 
   def execute_hooks(data, hooks_scope = :all)

app/services/web_hook_service.rb

@@ -63,7 +63,7 @@ class WebHookService
   end
 
   def async_execute
-    Sidekiq::Client.enqueue(WebHookWorker, hook.id, data, hook_name)
+    WebHookWorker.perform_async(hook.id, data, hook_name)
   end
 
   private

app/workers/admin_email_worker.rb

@@ -1,5 +1,5 @@
 class AdminEmailWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/authorized_projects_worker.rb

@@ -1,6 +1,5 @@
 class AuthorizedProjectsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # Schedules multiple jobs and waits for them to be completed.
   def self.bulk_perform_and_wait(args_list)
@@ -17,11 +16,6 @@ class AuthorizedProjectsWorker
     waiter.wait
   end
 
-  # Schedules multiple jobs to run in sidekiq without waiting for completion
-  def self.bulk_perform_async(args_list)
-    Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
-  end
-
   # Performs multiple jobs directly. Failed jobs will be put into sidekiq so
   # they can benefit from retries
   def self.bulk_perform_inline(args_list)

app/workers/background_migration_worker.rb

@@ -1,34 +1,5 @@
 class BackgroundMigrationWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-
-  # Enqueues a number of jobs in bulk.
-  #
-  # The `jobs` argument should be an Array of Arrays, each sub-array must be in
-  # the form:
-  #
-  #     [migration-class, [arg1, arg2, ...]]
-  def self.perform_bulk(jobs)
-    Sidekiq::Client.push_bulk('class' => self,
-                              'queue' => sidekiq_options['queue'],
-                              'args' => jobs)
-  end
-
-  # Schedules multiple jobs in bulk, with a delay.
-  #
-  def self.perform_bulk_in(delay, jobs)
-    now = Time.now.to_i
-    schedule = now + delay.to_i
-
-    if schedule <= now
-      raise ArgumentError, 'The schedule time must be in the future!'
-    end
-
-    Sidekiq::Client.push_bulk('class' => self,
-                              'queue' => sidekiq_options['queue'],
-                              'args' => jobs,
-                              'at' => schedule)
-  end
+  include ApplicationWorker
 
   # Performs the background migration.
   #

app/workers/build_coverage_worker.rb

@@ -1,5 +1,5 @@
 class BuildCoverageWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(build_id)

app/workers/build_finished_worker.rb

@@ -1,5 +1,5 @@
 class BuildFinishedWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_hooks_worker.rb

@@ -1,5 +1,5 @@
 class BuildHooksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :hooks

app/workers/build_queue_worker.rb

@@ -1,5 +1,5 @@
 class BuildQueueWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_success_worker.rb

@@ -1,5 +1,5 @@
 class BuildSuccessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_trace_sections_worker.rb

@@ -1,5 +1,5 @@
 class BuildTraceSectionsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(build_id)

app/workers/cluster_install_app_worker.rb

@@ -1,5 +1,5 @@
 class ClusterInstallAppWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
   include ClusterApplications
 

app/workers/cluster_provision_worker.rb

@@ -1,5 +1,5 @@
 class ClusterProvisionWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
 
   def perform(cluster_id)

app/workers/cluster_wait_for_app_installation_worker.rb

@@ -1,5 +1,5 @@
 class ClusterWaitForAppInstallationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
   include ClusterApplications
 

app/workers/concerns/application_worker.rb

@@ -0,0 +1,40 @@
+Sidekiq::Worker.extend ActiveSupport::Concern
+
+module ApplicationWorker
+  extend ActiveSupport::Concern
+
+  include Sidekiq::Worker
+
+  included do
+    sidekiq_options queue: base_queue_name
+  end
+
+  module ClassMethods
+    def base_queue_name
+      name
+        .sub(/\AGitlab::/, '')
+        .sub(/Worker\z/, '')
+        .underscore
+        .tr('/', '_')
+    end
+
+    def queue
+      get_sidekiq_options['queue'].to_s
+    end
+
+    def bulk_perform_async(args_list)
+      Sidekiq::Client.push_bulk('class' => self, 'args' => args_list)
+    end
+
+    def bulk_perform_in(delay, args_list)
+      now = Time.now.to_i
+      schedule = now + delay.to_i
+
+      if schedule <= now
+        raise ArgumentError, 'The schedule time must be in the future!'
+      end
+
+      Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule)
+    end
+  end
+end

app/workers/concerns/dedicated_sidekiq_queue.rb

@@ -1,9 +0,0 @@
-# Concern that sets the queue of a Sidekiq worker based on the worker's class
-# name/namespace.
-module DedicatedSidekiqQueue
-  extend ActiveSupport::Concern
-
-  included do
-    sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_')
-  end
-end

app/workers/concerns/gitlab/github_import/object_importer.rb

@@ -8,7 +8,7 @@ module Gitlab
       extend ActiveSupport::Concern
 
       included do
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include ReschedulingMethods
         include NotifyUponDeath

app/workers/create_gpg_signature_worker.rb

@@ -1,6 +1,5 @@
 class CreateGpgSignatureWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(commit_sha, project_id)
     project = Project.find_by(id: project_id)

app/workers/create_pipeline_worker.rb

@@ -1,5 +1,5 @@
 class CreatePipelineWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :creation

app/workers/delete_merged_branches_worker.rb

@@ -1,6 +1,5 @@
 class DeleteMergedBranchesWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(project_id, user_id)
     begin

app/workers/delete_user_worker.rb

@@ -1,6 +1,5 @@
 class DeleteUserWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(current_user_id, delete_user_id, options = {})
     delete_user  = User.find(delete_user_id)

app/workers/email_receiver_worker.rb

@@ -1,6 +1,5 @@
 class EmailReceiverWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(raw)
     return unless Gitlab::IncomingEmail.enabled?

app/workers/emails_on_push_worker.rb

@@ -1,6 +1,5 @@
 class EmailsOnPushWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   attr_reader :email, :skip_premailer
 

app/workers/expire_build_artifacts_worker.rb

@@ -1,5 +1,5 @@
 class ExpireBuildArtifactsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform
@@ -8,6 +8,6 @@ class ExpireBuildArtifactsWorker
     build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
     build_ids = build_ids.map { |build_id| [build_id] }
 
-    Sidekiq::Client.push_bulk('class' => ExpireBuildInstanceArtifactsWorker, 'args' => build_ids )
+    ExpireBuildInstanceArtifactsWorker.bulk_perform_async(build_ids)
   end
 end

app/workers/expire_build_instance_artifacts_worker.rb

@@ -1,6 +1,5 @@
 class ExpireBuildInstanceArtifactsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(build_id)
     build = Ci::Build

app/workers/expire_job_cache_worker.rb

@@ -1,5 +1,5 @@
 class ExpireJobCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :cache

app/workers/expire_pipeline_cache_worker.rb

@@ -1,5 +1,5 @@
 class ExpirePipelineCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :cache

app/workers/git_garbage_collect_worker.rb

@@ -1,6 +1,5 @@
 class GitGarbageCollectWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include Gitlab::CurrentSettings
 
   sidekiq_options retry: false

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -7,7 +7,7 @@ module Gitlab
     # been completed this worker will advance the import process to the next
     # stage.
     class AdvanceStageWorker
-      include Sidekiq::Worker
+      include ApplicationWorker
 
       sidekiq_options queue: 'github_importer_advance_stage', dead: false
 

app/workers/gitlab/github_import/refresh_import_jid_worker.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module GithubImport
     class RefreshImportJidWorker
-      include Sidekiq::Worker
+      include ApplicationWorker
       include GithubImport::Queue
 
       # The interval to schedule new instances of this job at.

app/workers/gitlab/github_import/stage/finish_import_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class FinishImportWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_base_data_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportBaseDataWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportIssuesAndDiffNotesWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_notes_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportNotesWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportPullRequestsWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_repository_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportRepositoryWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab_shell_worker.rb

@@ -1,7 +1,6 @@
 class GitlabShellWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include Gitlab::ShellAdapter
-  include DedicatedSidekiqQueue
 
   def perform(action, *arg)
     gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend

app/workers/gitlab_usage_ping_worker.rb

@@ -1,7 +1,7 @@
 class GitlabUsagePingWorker
   LEASE_TIMEOUT = 86400
 
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/group_destroy_worker.rb

@@ -1,6 +1,5 @@
 class GroupDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   def perform(group_id, user_id)

app/workers/import_export_project_cleanup_worker.rb

@@ -1,5 +1,5 @@
 class ImportExportProjectCleanupWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/invalid_gpg_signature_update_worker.rb

@@ -1,6 +1,5 @@
 class InvalidGpgSignatureUpdateWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(gpg_key_id)
     gpg_key = GpgKey.find_by(id: gpg_key_id)

app/workers/irker_worker.rb

@@ -2,8 +2,7 @@ require 'json'
 require 'socket'
 
 class IrkerWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(project_id, chans, colors, push_data, settings)
     project = Project.find(project_id)

app/workers/merge_worker.rb

@@ -1,6 +1,5 @@
 class MergeWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(merge_request_id, current_user_id, params)
     params = params.with_indifferent_access

app/workers/namespaceless_project_destroy_worker.rb

@@ -5,14 +5,9 @@
 # The worker will reject doing anything for projects that *do* have a
 # namespace. For those use ProjectDestroyWorker instead.
 class NamespacelessProjectDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
-  def self.bulk_perform_async(args_list)
-    Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
-  end
-
   def perform(project_id)
     begin
       project = Project.unscoped.find(project_id)

app/workers/new_issue_worker.rb

@@ -1,6 +1,5 @@
 class NewIssueWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include NewIssuable
 
   def perform(issue_id, user_id)

app/workers/new_merge_request_worker.rb

@@ -1,6 +1,5 @@
 class NewMergeRequestWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include NewIssuable
 
   def perform(merge_request_id, user_id)

app/workers/new_note_worker.rb

@@ -1,6 +1,5 @@
 class NewNoteWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # Keep extra parameter to preserve backwards compatibility with
   # old `NewNoteWorker` jobs (can remove later)

app/workers/pages_worker.rb

@@ -1,5 +1,5 @@
 class PagesWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
 
   sidekiq_options queue: :pages, retry: false
 

app/workers/pipeline_hooks_worker.rb

@@ -1,5 +1,5 @@
 class PipelineHooksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :hooks

app/workers/pipeline_metrics_worker.rb

@@ -1,5 +1,5 @@
 class PipelineMetricsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(pipeline_id)

app/workers/pipeline_notification_worker.rb

@@ -1,5 +1,5 @@
 class PipelineNotificationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(pipeline_id, recipients = nil)

app/workers/pipeline_process_worker.rb

@@ -1,5 +1,5 @@
 class PipelineProcessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/pipeline_schedule_worker.rb

@@ -1,5 +1,5 @@
 class PipelineScheduleWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/pipeline_success_worker.rb

@@ -1,5 +1,5 @@
 class PipelineSuccessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/pipeline_update_worker.rb

@@ -1,5 +1,5 @@
 class PipelineUpdateWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/post_receive.rb

@@ -1,6 +1,5 @@
 class PostReceive
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(gl_repository, identifier, changes)
     project, is_wiki = Gitlab::GlRepository.parse(gl_repository)

app/workers/process_commit_worker.rb

@@ -5,8 +5,7 @@
 # Consider using an extra worker if you need to add any extra (and potentially
 # slow) processing of commits.
 class ProcessCommitWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # project_id - The ID of the project this commit belongs to.
   # user_id - The ID of the user that pushed the commit.

app/workers/project_cache_worker.rb

@@ -1,7 +1,6 @@
 # Worker for updating any project specific caches.
 class ProjectCacheWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 15.minutes.to_i
 

app/workers/project_destroy_worker.rb

@@ -1,6 +1,5 @@
 class ProjectDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   def perform(project_id, user_id, params)

app/workers/project_export_worker.rb

@@ -1,6 +1,5 @@
 class ProjectExportWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   sidekiq_options retry: 3

app/workers/project_migrate_hashed_storage_worker.rb

@@ -1,6 +1,5 @@
 class ProjectMigrateHashedStorageWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 30.seconds.to_i
 

app/workers/project_service_worker.rb

@@ -1,6 +1,5 @@
 class ProjectServiceWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   sidekiq_options dead: false
 

app/workers/propagate_service_template_worker.rb

@@ -1,7 +1,6 @@
 # Worker for updating any project specific caches.
 class PropagateServiceTemplateWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 4.hours.to_i
 

app/workers/prune_old_events_worker.rb

@@ -1,5 +1,5 @@
 class PruneOldEventsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/reactive_caching_worker.rb

@@ -1,6 +1,5 @@
 class ReactiveCachingWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(class_name, id, *args)
     klass = begin

app/workers/remove_expired_group_links_worker.rb

@@ -1,5 +1,5 @@
 class RemoveExpiredGroupLinksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/remove_expired_members_worker.rb

@@ -1,5 +1,5 @@
 class RemoveExpiredMembersWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/remove_old_web_hook_logs_worker.rb

@@ -1,5 +1,5 @@
 class RemoveOldWebHookLogsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   WEB_HOOK_LOG_LIFETIME = 2.days

app/workers/remove_unreferenced_lfs_objects_worker.rb

@@ -1,5 +1,5 @@
 class RemoveUnreferencedLfsObjectsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/repository_archive_cache_worker.rb

@@ -1,5 +1,5 @@
 class RepositoryArchiveCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/repository_check/batch_worker.rb

@@ -1,6 +1,6 @@
 module RepositoryCheck
   class BatchWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include CronjobQueue
 
     RUN_TIME = 3600

app/workers/repository_check/clear_worker.rb

@@ -1,6 +1,6 @@
 module RepositoryCheck
   class ClearWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include RepositoryCheckQueue
 
     def perform

app/workers/repository_check/single_repository_worker.rb

@@ -1,6 +1,6 @@
 module RepositoryCheck
   class SingleRepositoryWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include RepositoryCheckQueue
 
     def perform(project_id)

app/workers/repository_fork_worker.rb

@@ -1,9 +1,8 @@
 class RepositoryForkWorker
   ForkError = Class.new(StandardError)
 
-  include Sidekiq::Worker
+  include ApplicationWorker
   include Gitlab::ShellAdapter
-  include DedicatedSidekiqQueue
   include ProjectStartImport
 
   sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION

app/workers/repository_import_worker.rb

@@ -1,8 +1,7 @@
 class RepositoryImportWorker
   ImportError = Class.new(StandardError)
 
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
   include ProjectStartImport
 

app/workers/requests_profiles_worker.rb

@@ -1,5 +1,5 @@
 class RequestsProfilesWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/schedule_update_user_activity_worker.rb

@@ -1,5 +1,5 @@
 class ScheduleUpdateUserActivityWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform(batch_size = 500)

app/workers/stage_update_worker.rb

@@ -1,5 +1,5 @@
 class StageUpdateWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/storage_migrator_worker.rb

@@ -1,6 +1,5 @@
 class StorageMigratorWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   BATCH_SIZE = 100
 

app/workers/stuck_ci_jobs_worker.rb

@@ -1,5 +1,5 @@
 class StuckCiJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'.freeze

app/workers/stuck_import_jobs_worker.rb

@@ -1,5 +1,5 @@
 class StuckImportJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   IMPORT_JOBS_EXPIRATION = 15.hours.to_i

app/workers/stuck_merge_jobs_worker.rb

@@ -1,5 +1,5 @@
 class StuckMergeJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/system_hook_push_worker.rb

@@ -1,6 +1,5 @@
 class SystemHookPushWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(push_data, hook_id)
     SystemHooksService.new.execute_hooks(push_data, hook_id)

app/workers/trending_projects_worker.rb

@@ -1,5 +1,5 @@
 class TrendingProjectsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/update_merge_requests_worker.rb

@@ -1,6 +1,5 @@
 class UpdateMergeRequestsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LOG_TIME_THRESHOLD = 90 # seconds
 

app/workers/update_user_activity_worker.rb

@@ -1,6 +1,5 @@
 class UpdateUserActivityWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(pairs)
     pairs = cast_data(pairs)

app/workers/upload_checksum_worker.rb

@@ -1,6 +1,5 @@
 class UploadChecksumWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(upload_id)
     upload = Upload.find(upload_id)

app/workers/wait_for_cluster_creation_worker.rb

@@ -1,5 +1,5 @@
 class WaitForClusterCreationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
 
   def perform(cluster_id)

app/workers/web_hook_worker.rb

@@ -1,6 +1,5 @@
 class WebHookWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   sidekiq_options retry: 4, dead: false
 

config/initializers/forbid_sidekiq_in_transactions.rb

@@ -13,20 +13,19 @@ module Sidekiq
 
     module ClassMethods
       module NoSchedulingFromTransactions
-        NESTING = ::Rails.env.test? ? 1 : 0
-
         %i(perform_async perform_at perform_in).each do |name|
           define_method(name) do |*args|
-            return super(*args) if Sidekiq::Worker.skip_transaction_check
-            return super(*args) unless ActiveRecord::Base.connection.open_transactions > NESTING
+            if !Sidekiq::Worker.skip_transaction_check && AfterCommitQueue.inside_transaction?
+              raise <<-MSG.strip_heredoc
+                `#{self}.#{name}` cannot be called inside a transaction as this can lead to
+                race conditions when the worker runs before the transaction is committed and
+                tries to access a model that has not been saved yet.
 
-            raise <<-MSG.strip_heredoc
-              `#{self}.#{name}` cannot be called inside a transaction as this can lead to
-              race conditions when the worker runs before the transaction is committed and
-              tries to access a model that has not been saved yet.
+                Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
+              MSG
+            end
 
-              Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
-            MSG
+            super(*args)
           end
         end
       end

config/initializers/sidekiq.rb

@@ -64,13 +64,13 @@ end
 # The Sidekiq client API always adds the queue to the Sidekiq queue
 # list, but mail_room and gitlab-shell do not. This is only necessary
 # for monitoring.
-config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
-
 begin
+  queues = Gitlab::SidekiqConfig.worker_queues
+
   Sidekiq.redis do |conn|
     conn.pipelined do
-      config[:queues].each do |queue|
-        conn.sadd('queues', queue[0])
+      queues.each do |queue|
+        conn.sadd('queues', queue)
       end
     end
   end

db/post_migrate/20170627101016_schedule_event_migrations.rb

@@ -25,14 +25,14 @@ class ScheduleEventMigrations < ActiveRecord::Migration
         # We push multiple jobs at a time to reduce the time spent in
         # Sidekiq/Redis operations. We're using this buffer based approach so we
         # don't need to run additional queries for every range.
-        BackgroundMigrationWorker.perform_bulk(jobs)
+        BackgroundMigrationWorker.bulk_perform_async(jobs)
         jobs.clear
       end
 
       jobs << ['MigrateEventsToPushEventPayloads', [min, max]]
     end
 
-    BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+    BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
   end
 
   def down

db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb

@@ -19,7 +19,7 @@ class ScheduleCreateGpgKeySubkeysFromGpgKeys < ActiveRecord::Migration
         [MIGRATION, [id]]
       end
 
-      BackgroundMigrationWorker.perform_bulk(jobs)
+      BackgroundMigrationWorker.bulk_perform_async(jobs)
     end
   end
 

doc/development/background_migrations.md

@@ -68,10 +68,10 @@ BackgroundMigrationWorker.perform_async('BackgroundMigrationClassName', [arg1, a
 ```
 
 Usually it's better to enqueue jobs in bulk, for this you can use
-`BackgroundMigrationWorker.perform_bulk`:
+`BackgroundMigrationWorker.bulk_perform_async`:
 
 ```ruby
-BackgroundMigrationWorker.perform_bulk(
+BackgroundMigrationWorker.bulk_perform_async(
   [['BackgroundMigrationClassName', [1]],
    ['BackgroundMigrationClassName', [2]]]
 )
@@ -85,13 +85,13 @@ updates. Removals in turn can be handled by simply defining foreign keys with
 cascading deletes.
 
 If you would like to schedule jobs in bulk with a delay, you can use
-`BackgroundMigrationWorker.perform_bulk_in`:
+`BackgroundMigrationWorker.bulk_perform_in`:
 
 ```ruby
 jobs = [['BackgroundMigrationClassName', [1]],
         ['BackgroundMigrationClassName', [2]]]
 
-BackgroundMigrationWorker.perform_bulk_in(5.minutes, jobs)
+BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs)
 ```
 
 ## Cleaning Up
@@ -201,7 +201,7 @@ class ScheduleExtractServicesUrl < ActiveRecord::Migration
         ['ExtractServicesUrl', [id]]
       end
 
-      BackgroundMigrationWorker.perform_bulk(jobs)
+      BackgroundMigrationWorker.bulk_perform_async(jobs)
     end
   end
 

doc/development/sidekiq_style_guide.md

@@ -3,6 +3,12 @@
 This document outlines various guidelines that should be followed when adding or
 modifying Sidekiq workers.
 
+## ApplicationWorker
+
+All workers should include `ApplicationWorker` instead of `Sidekiq::Worker`,
+which adds some convenience methods and automatically sets the queue based on
+the worker's name.
+
 ## Default Queue
 
 Use of the "default" queue is not allowed. Every worker should use a queue that
@@ -13,19 +19,10 @@ A list of all available queues can be found in `config/sidekiq_queues.yml`.
 
 ## Dedicated Queues
 
-Most workers should use their own queue. To ease this process a worker can
-include the `DedicatedSidekiqQueue` concern as follows:
-
-```ruby
-class ProcessSomethingWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-end
-```
-
-This will set the queue name based on the class' name, minus the `Worker`
-suffix. In the above example this would lead to the queue being
-`process_something`.
+Most workers should use their own queue, which is automatically set based on the
+worker class name. For a worker named `ProcessSomethingWorker`, the queue name
+would be `process_something`. If you're not sure what a worker's queue name is,
+you can find it using `SomeWorker.queue`.
 
 In some cases multiple workers do use the same queue. For example, the various
 workers for updating CI pipelines all use the `pipeline` queue. Adding workers
@@ -39,7 +36,7 @@ tests should be placed in `spec/workers`.
 
 ## Removing or renaming queues
 
-Try to avoid renaming or removing queues in minor and patch releases. 
-During online update instance can have pending jobs and removing the queue can 
-lead to those jobs being stuck forever. If you can't write migration for those 
-Sidekiq jobs, please consider doing rename or remove queue in major release only. 
+Try to avoid renaming or removing queues in minor and patch releases.
+During online update instance can have pending jobs and removing the queue can
+lead to those jobs being stuck forever. If you can't write migration for those
+Sidekiq jobs, please consider doing rename or remove queue in major release only.