diff --git a/Gemfile b/Gemfile index d6e5899e3e7..eebd4c7cf4c 100644 --- a/Gemfile +++ b/Gemfile @@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3" gem "draper", "~> 0.18.0" # Background jobs -gem "resque", git: "https://github.com/gitlabhq/resque.git", ref: "9ef4700306dd946a3ac000612428967ce0c32213" -gem 'resque_mailer' +gem 'slim' +gem 'sinatra', :require => nil +gem 'sidekiq', '2.6.4' +gem 'sidekiq_mailer' # HTTP requests gem "httparty" @@ -157,7 +159,6 @@ group :test do gem "simplecov", require: false gem "shoulda-matchers", "1.3.0" gem 'email_spec' - gem 'resque_spec' gem "webmock" gem 'test_after_commit' end diff --git a/Gemfile.lock b/Gemfile.lock index 7d76bf9e75c..b9008340d99 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -54,17 +54,6 @@ GIT specs: raphael-rails (2.1.0) -GIT - remote: https://github.com/gitlabhq/resque.git - revision: 9ef4700306dd946a3ac000612428967ce0c32213 - ref: 9ef4700306dd946a3ac000612428967ce0c32213 - specs: - resque (2.0.0.pre.1) - json - redis-namespace (~> 1.0) - sinatra (>= 0.9.2) - vegas (~> 0.1.2) - GIT remote: https://github.com/jonleighton/poltergeist.git revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca @@ -132,6 +121,9 @@ GEM carrierwave (0.7.1) activemodel (>= 3.2.0) activesupport (>= 3.2.0) + celluloid (0.12.4) + facter (>= 1.6.12) + timers (>= 1.0.0) charlock_holmes (0.6.9) childprocess (0.3.6) ffi (~> 1.0, >= 1.0.6) @@ -150,6 +142,7 @@ GEM coffee-script-source (1.4.0) colored (1.2) colorize (0.5.8) + connection_pool (1.0.0) crack (0.3.1) daemons (1.1.9) devise (2.1.2) @@ -169,6 +162,7 @@ GEM eventmachine (1.0.0) execjs (1.4.0) multi_json (~> 1.0) + facter (1.6.17) factory_girl (4.1.0) activesupport (>= 3.0.0) factory_girl_rails (4.1.0) @@ -370,11 +364,6 @@ GEM redis (3.0.2) redis-namespace (1.2.1) redis (~> 3.0.0) - resque_mailer (2.1.0) - actionmailer (~> 3.0) - resque_spec (0.12.5) - resque (>= 1.19.0) - rspec (>= 2.5.0) rspec (2.12.0) rspec-core (~> 2.12.0) rspec-expectations (~> 2.12.0) @@ -412,6 +401,16 @@ GEM sexp_processor (4.1.3) shoulda-matchers (1.3.0) activesupport (>= 3.0.0) + sidekiq (2.6.4) + celluloid (~> 0.12.0) + connection_pool (~> 1.0) + multi_json (~> 1) + redis (~> 3) + redis-namespace + sidekiq_mailer (0.0.4) + actionmailer (~> 3.0) + activesupport (~> 3.0) + sidekiq (~> 2.3) simplecov (0.7.1) multi_json (~> 1.0) simplecov-html (~> 0.7.1) @@ -421,6 +420,9 @@ GEM rack-protection (~> 1.2) tilt (~> 1.3, >= 1.3.3) six (0.2.0) + slim (1.3.6) + temple (~> 0.5.5) + tilt (~> 1.3.3) slop (3.3.3) spinach (0.5.2) colorize @@ -435,6 +437,7 @@ GEM rack (~> 1.0) tilt (~> 1.1, != 1.3.0) stamp (0.3.0) + temple (0.5.5) test_after_commit (0.0.1) therubyracer (0.10.2) libv8 (~> 3.3.10) @@ -444,6 +447,7 @@ GEM rack (>= 1.0.0) thor (0.16.0) tilt (1.3.3) + timers (1.0.2) treetop (1.4.12) polyglot polyglot (>= 0.3.1) @@ -455,8 +459,6 @@ GEM kgio (~> 2.6) rack raindrops (~> 0.7) - vegas (0.1.11) - rack (>= 1.0.0) virtus (0.5.2) backports (~> 2.6.1) warden (1.2.1) @@ -534,17 +536,18 @@ DEPENDENCIES rb-fsevent rb-inotify redcarpet (~> 2.2.2) - resque! - resque_mailer - resque_spec rspec-rails sass-rails (~> 3.2.5) sdoc seed-fu settingslogic shoulda-matchers (= 1.3.0) + sidekiq (= 2.6.4) + sidekiq_mailer simplecov + sinatra six + slim spinach-rails stamp test_after_commit diff --git a/Procfile b/Procfile index 09ec8cc2745..08556c6493a 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: bundle exec rails s -p $PORT -worker: bundle exec rake environment resque:work QUEUE=* VVERBOSE=1 +worker: bundle exec sidekiq -q post_receive,mailer,system_hook,common diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index 827dd0cf7cd..f97c56b0b31 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController @projects = Project.order("created_at DESC").limit(10) @users = User.order("created_at DESC").limit(10) - @resque_accessible = true - @workers = Resque.workers - @pending_jobs = Resque.size(:post_receive) - rescue Redis::InheritedError @resque_accessible = false end diff --git a/app/mailers/notify.rb b/app/mailers/notify.rb index cec680bc356..87d3e4b204e 100644 --- a/app/mailers/notify.rb +++ b/app/mailers/notify.rb @@ -1,5 +1,5 @@ class Notify < ActionMailer::Base - include Resque::Mailer + include Sidekiq::Mailer add_template_helper ApplicationHelper add_template_helper GitlabMarkdownHelper diff --git a/app/models/system_hook.rb b/app/models/system_hook.rb index 2ae5b1314e9..5f1bd6477c4 100644 --- a/app/models/system_hook.rb +++ b/app/models/system_hook.rb @@ -19,6 +19,6 @@ class SystemHook < WebHook end def async_execute(data) - Resque.enqueue(SystemHookWorker, id, data) + Sidekiq::Client.enqueue(SystemHookWorker, id, data) end end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 672dad4377e..2e572aaba79 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -1,5 +1,5 @@ .admin_dash.row - .span3 + .span4 .ui-box %h5.title Projects .data.padded @@ -7,7 +7,7 @@ %h1= Project.count %hr = link_to 'New Project', new_project_path, class: "btn small" - .span3 + .span4 .ui-box %h5.title Groups .data.padded @@ -15,7 +15,7 @@ %h1= Group.count %hr = link_to 'New Group', new_admin_group_path, class: "btn small" - .span3 + .span4 .ui-box %h5.title Users .data.padded @@ -23,25 +23,6 @@ %h1= User.count %hr = link_to 'New User', new_admin_user_path, class: "btn small" - .span3 - .ui-box - %h5.title - Resque Workers - .data.padded - - if @resque_accessible - = link_to admin_resque_path do - %h1{class: @workers.present? ? "cgreen" : "cred"} - = @workers.count - %hr - %p - %strong{class: @pending_jobs > 0 ? "cred" : "cgreen"} - #{@pending_jobs} post receive jobs waiting - - else - = link_to admin_resque_path do - %h1.cdark ? - %hr - %p - %strong Resque status unknown .row .span6 diff --git a/app/views/admin/resque/show.html.haml b/app/views/admin/resque/show.html.haml index 41254a6b6c2..499738f9a06 100644 --- a/app/views/admin/resque/show.html.haml +++ b/app/views/admin/resque/show.html.haml @@ -1,4 +1,4 @@ -%h3.page_title Resque +%h3.page_title Background Jobs %br .ui-box - %iframe{src: resque_path, width: '100%', height: 600, style: "border: none"} + %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"} diff --git a/app/views/layouts/admin.html.haml b/app/views/layouts/admin.html.haml index 6b643ec8ccb..a60e7febe76 100644 --- a/app/views/layouts/admin.html.haml +++ b/app/views/layouts/admin.html.haml @@ -19,6 +19,6 @@ = nav_link(controller: :hooks) do = link_to "Hooks", admin_hooks_path = nav_link(controller: :resque) do - = link_to "Resque", admin_resque_path + = link_to "Background Jobs", admin_resque_path .content= yield diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 9e3f32f835d..e74379a65dd 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -1,7 +1,9 @@ class PostReceive - @queue = :post_receive + include Sidekiq::Worker - def self.perform(repo_path, oldrev, newrev, ref, identifier) + sidekiq_options queue: :post_receive + + def perform(repo_path, oldrev, newrev, ref, identifier) repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "") repo_path.gsub!(/.git$/, "") repo_path.gsub!(/^\//, "") diff --git a/app/workers/system_hook_worker.rb b/app/workers/system_hook_worker.rb index ca154136b97..3ebc62b7e7a 100644 --- a/app/workers/system_hook_worker.rb +++ b/app/workers/system_hook_worker.rb @@ -1,7 +1,9 @@ class SystemHookWorker - @queue = :system_hook + include Sidekiq::Worker - def self.perform(hook_id, data) + sidekiq_options queue: :system_hook + + def perform(hook_id, data) SystemHook.find(hook_id).execute data end end diff --git a/config/initializers/4_resque.rb b/config/initializers/4_resque.rb index ab34050cb50..6abe6e74fa0 100644 --- a/config/initializers/4_resque.rb +++ b/config/initializers/4_resque.rb @@ -1,29 +1,22 @@ # Custom Redis configuration config_file = Rails.root.join('config', 'resque.yml') -if File.exists?(config_file) - resque_config = YAML.load_file(config_file) - Resque.redis = resque_config[Rails.env] -end -Resque.redis.namespace = 'resque:gitlab' -Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } +resque_url = if File.exists?(config_file) + YAML.load_file(config_file)[Rails.env] + else + "localhost:6379" + end -# Authentication -require 'resque/server' -class ResqueAuthentication - def initialize(app) - @app = app - end - - def call(env) - account = env['warden'].authenticate!(:database_authenticatable, :rememberable, scope: :user) - raise "Access denied" if !account.admin? - @app.call(env) - end +Sidekiq.configure_server do |config| + config.redis = { + url: "redis://#{resque_url}", + namespace: 'resque:gitlab' + } end -Resque::Server.use ResqueAuthentication - -# Mailer -Resque::Mailer.excluded_environments = [] - +Sidekiq.configure_client do |config| + config.redis = { + url: "redis://#{resque_url}", + namespace: 'resque:gitlab' + } +end diff --git a/config/routes.rb b/config/routes.rb index 27188b3b89f..4c1f61e9a83 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,5 @@ +require 'sidekiq/web' + Gitlab::Application.routes.draw do # # Search @@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do require 'api' mount Gitlab::API => '/api' - # Optionally, enable Resque here - require 'resque/server' - mount Resque::Server => '/info/resque', as: 'resque' + constraint = lambda { |request| request.env["warden"].authenticate? and request.env['warden'].user.admin? } + constraints constraint do + mount Sidekiq::Web, at: "/admin/workers", as: :sidekiq + end # Enable Grack support mount Grack::Bundle.new({ diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake index 0c3b93c5bed..e469aef3f94 100644 --- a/lib/tasks/resque.rake +++ b/lib/tasks/resque.rake @@ -2,20 +2,20 @@ require 'resque/tasks' namespace :resque do task setup: :environment do - Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } + #Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } end desc "Resque | kill all workers (using -QUIT), god will take care of them" task :stop_workers => :environment do - pids = Array.new + #pids = Array.new - Resque.workers.each do |worker| - pids << worker.to_s.split(/:/).second - end + #Resque.workers.each do |worker| + #pids << worker.to_s.split(/:/).second + #end - if pids.size > 0 - system("kill -QUIT #{pids.join(' ')}") - end + #if pids.size > 0 + #system("kill -QUIT #{pids.join(' ')}") + #end end end