Merge in master

.travis.yml

@@ -1,13 +1,16 @@
 language: ruby
 sudo: false
+cache: bundler
 services:
   - redis-server
 rvm:
-  - jruby-19mode
-  - rbx-2
-  - 2.0.0
   - 2.1
+  - 2.0.0
   - 2.2
+  - jruby-19mode
+  - jruby
+  - rbx-2
 matrix:
   allow_failures:
     - rvm: rbx-2
+    - rvm: jruby

Changes.md

@@ -1,10 +1,12 @@
 HEAD
 -----------
-
+- Add random integer to process identity [#2113, michaeldiscala]
 - Log Sidekiq Pro's Batch ID if available [#2076]
 - Refactor Processor Redis usage to avoid redis/redis-rb#490 [#]
 - Move /dashboard/stats to /stats.  Add /stats/queues. [moserke, #2099]
-
+- Add better usage text for `sidekiqctl`.
+- `Sidekiq::Logging.with_context` is now a stack so you can set your
+  own job context for logging purposes [grosser, #2110]
 
 3.3.0
 -----------

Rakefile

@@ -1,7 +1,6 @@
 require 'bundler/gem_tasks'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'test'
   #SO MUCH NOISE
   #test.warning = true
   test.pattern = 'test/**/test_*.rb'

bin/sidekiqctl

@@ -3,23 +3,29 @@
 require 'fileutils'
 
 class Sidekiqctl
-  DEFAULT_TIMEOUT = 10
+  DEFAULT_KILL_TIMEOUT = 10
 
-  attr_reader :stage, :pidfile, :timeout
+  attr_reader :stage, :pidfile, :kill_timeout
 
   def self.print_usage
+    puts "#{File.basename($0)} - stop a Sidekiq process from the command line."
     puts
-    puts "Usage: #{File.basename($0)} <command> <pidfile> <timeout>"
-    puts " where <command> is either 'quiet', 'stop' or 'shutdown'"
+    puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
+    puts " where <command> is either 'quiet' or 'stop'"
     puts "       <pidfile> is path to a pidfile"
-    puts "       <timeout> is number of seconds to wait till Sidekiq exits (default: #{Sidekiqctl::DEFAULT_TIMEOUT})"
+    puts "       <kill_timeout> is number of seconds to wait until Sidekiq exits"
+    puts "       (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
+    puts
+    puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout.  If you want"
+    puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
+    puts " path_to_pidfile 61`"
     puts
   end
 
   def initialize(stage, pidfile, timeout)
     @stage = stage
     @pidfile = pidfile
-    @timeout = timeout
+    @kill_timeout = timeout
 
     done('No pidfile given', :error) if !pidfile
     done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
@@ -30,7 +36,7 @@ class Sidekiqctl
     begin
       send(stage)
     rescue NoMethodError
-      done 'Invalid control command', :error
+      done "Invalid command: #{stage}", :error
     end
   end
 
@@ -59,7 +65,7 @@ class Sidekiqctl
 
   def stop
     `kill -TERM #{pid}`
-    timeout.times do
+    kill_timeout.times do
       begin
         Process.getpgid(pid)
       rescue Errno::ESRCH
@@ -72,11 +78,7 @@ class Sidekiqctl
     FileUtils.rm_f pidfile
     done 'Sidekiq shut down forcefully.'
   end
-
-  def shutdown
-    quiet
-    stop
-  end
+  alias_method :shutdown, :stop
 end
 
 if ARGV.length < 2
@@ -85,7 +87,7 @@ else
   stage = ARGV[0]
   pidfile = ARGV[1]
   timeout = ARGV[2].to_i
-  timeout = Sidekiqctl::DEFAULT_TIMEOUT if timeout == 0
+  timeout = Sidekiqctl::DEFAULT_KILL_TIMEOUT if timeout == 0
 
   Sidekiqctl.new(stage, pidfile, timeout)
 end

lib/sidekiq/api.rb

@@ -616,6 +616,7 @@ module Sidekiq
   #   'queues' => ['default', 'low'],
   #   'busy' => 10,
   #   'beat' => <last heartbeat>,
+  #   'identity' => <unique string identifying the process>,
   # }
   class Process
     def initialize(hash)
@@ -655,7 +656,7 @@ module Sidekiq
     end
 
     def identity
-      @id ||= "#{self['hostname']}:#{self['pid']}"
+      self['identity']
     end
   end
 

lib/sidekiq/launcher.rb

@@ -75,6 +75,7 @@ module Sidekiq
         'concurrency' => @options[:concurrency],
         'queues' => @options[:queues].uniq,
         'labels' => Sidekiq.options[:labels],
+        'identity' => identity,
       }
       # this data doesn't change so dump it to a string
       # now so we don't need to dump it every heartbeat.

lib/sidekiq/logging.rb

@@ -5,6 +5,8 @@ module Sidekiq
   module Logging
 
     class Pretty < Logger::Formatter
+      SPACE = " "
+
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
         "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
@@ -12,17 +14,16 @@ module Sidekiq
 
       def context
         c = Thread.current[:sidekiq_context]
-        c ? " #{c}" : ''
+        " #{c.join(SPACE)}" if c && c.any?
       end
     end
 
     def self.with_context(msg)
-      begin
-        Thread.current[:sidekiq_context] = msg
-        yield
-      ensure
-        Thread.current[:sidekiq_context] = nil
-      end
+      Thread.current[:sidekiq_context] ||= []
+      Thread.current[:sidekiq_context] << msg
+      yield
+    ensure
+      Thread.current[:sidekiq_context].pop
     end
 
     def self.initialize_logger(log_target = STDOUT)

lib/sidekiq/manager.rb

@@ -48,7 +48,7 @@ module Sidekiq
 
         @done = true
 
-        logger.info { "Shutting down #{@ready.size} quiet workers" }
+        logger.info { "Terminating #{@ready.size} quiet workers" }
         @ready.each { |x| x.terminate if x.alive? }
         @ready.clear
 

lib/sidekiq/util.rb

@@ -1,4 +1,5 @@
 require 'socket'
+require 'securerandom'
 require 'sidekiq/exception_handler'
 require 'sidekiq/core_ext'
 
@@ -30,8 +31,12 @@ module Sidekiq
       ENV['DYNO'] || Socket.gethostname
     end
 
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
     def identity
-      @@identity ||= "#{hostname}:#{$$}"
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
     def fire_event(event)

lib/sidekiq/web.rb

@@ -45,8 +45,8 @@ module Sidekiq
     end
 
     post "/busy" do
-      if params['hostname']
-        p = Sidekiq::Process.new('hostname' => params["hostname"], 'pid' => params['pid'])
+      if params['identity']
+        p = Sidekiq::Process.new('identity' => params['identity'])
         p.quiet! if params[:quiet]
         p.stop! if params[:stop]
       else

test/test_api.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 
 class TestApi < Sidekiq::Test
 
@@ -374,7 +374,15 @@ class TestApi < Sidekiq::Test
     end
 
     it 'can enumerate processes' do
-      odata = { 'pid' => 123, 'hostname' => hostname, 'key' => "#{hostname}:123", 'started_at' => Time.now.to_f - 15 }
+      identity_string = "identity_string"
+      odata = {
+        'pid' => 123,
+        'hostname' => hostname,
+        'key' => identity_string,
+        'identity' => identity_string,
+        'started_at' => Time.now.to_f - 15,
+      }
+
       time = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.multi do
@@ -392,8 +400,9 @@ class TestApi < Sidekiq::Test
       assert_equal 123, data['pid']
       data.quiet!
       data.stop!
-      assert_equal "TERM", Sidekiq.redis{|c| c.lpop("#{hostname}:123-signals") }
-      assert_equal "USR1", Sidekiq.redis{|c| c.lpop("#{hostname}:123-signals") }
+      signals_string = "#{odata['key']}-signals"
+      assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
+      assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
     end
 
     it 'can enumerate workers' do

test/test_cli.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/cli'
 require 'tempfile'
 

test/test_client.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/client'
 require 'sidekiq/worker'
 

test/test_exception_handler.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/exception_handler'
 require 'stringio'
 require 'logger'

test/test_extensions.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq'
 require 'active_record'
 require 'action_mailer'

test/test_fetch.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/fetch'
 
 class TestFetcher < Sidekiq::Test

test/test_logging.rb

@@ -0,0 +1,34 @@
+require_relative 'helper'
+require 'sidekiq/logging'
+
+class TestFetcher < Sidekiq::Test
+  describe Sidekiq::Logging do
+    describe "#with_context" do
+      def context
+        Sidekiq::Logging.logger.formatter.context
+      end
+
+      it "has no context by default" do
+        context.must_equal nil
+      end
+
+      it "can add a context" do
+        Sidekiq::Logging.with_context "xx" do
+          context.must_equal " xx"
+        end
+        context.must_equal nil
+      end
+
+      it "can use multiple contexts" do
+        Sidekiq::Logging.with_context "xx" do
+          context.must_equal " xx"
+          Sidekiq::Logging.with_context "yy" do
+            context.must_equal " xx yy"
+          end
+          context.must_equal " xx"
+        end
+        context.must_equal nil
+      end
+    end
+  end
+end

test/test_manager.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/manager'
 
 class TestManager < Sidekiq::Test

test/test_middleware.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/middleware/chain'
 require 'sidekiq/processor'
 

test/test_processor.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/processor'
 
 class TestProcessor < Sidekiq::Test

test/test_redis_connection.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/redis_connection'
 
 class TestRedisConnection < Sidekiq::Test

test/test_retry.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/scheduled'
 require 'sidekiq/middleware/server/retry_jobs'
 

test/test_scheduled.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/scheduled'
 
 class TestScheduled < Sidekiq::Test

test/test_scheduling.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq/scheduled'
 
 class TestScheduling < Sidekiq::Test

test/test_sidekiq.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require 'helper'
+require_relative 'helper'
 
 class TestSidekiq < Sidekiq::Test
   describe 'json processing' do

test/test_testing.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq'
 require 'sidekiq/worker'
 require 'active_record'

test/test_testing_fake.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq'
 require 'sidekiq/worker'
 require 'active_record'

test/test_testing_inline.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq'
 require 'sidekiq/worker'
 require 'active_record'

test/test_web.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'sidekiq'
 require 'sidekiq/web'
 require 'rack/test'
@@ -50,17 +50,23 @@ class TestWeb < Sidekiq::Test
       end
 
       it 'can quiet a process' do
-        assert_nil Sidekiq.redis { |c| c.lpop "host:pid-signals" }
-        post '/busy', 'quiet' => '1', 'hostname' => 'host', 'pid' => 'pid'
+        identity = 'identity'
+        signals_key = "#{identity}-signals"
+
+        assert_nil Sidekiq.redis { |c| c.lpop signals_key }
+        post '/busy', 'quiet' => '1', 'identity' => identity
         assert_equal 302, last_response.status
-        assert_equal 'USR1', Sidekiq.redis { |c| c.lpop "host:pid-signals" }
+        assert_equal 'USR1', Sidekiq.redis { |c| c.lpop signals_key }
       end
 
       it 'can stop a process' do
-        assert_nil Sidekiq.redis { |c| c.lpop "host:pid-signals" }
-        post '/busy', 'stop' => '1', 'hostname' => 'host', 'pid' => 'pid'
+        identity = 'identity'
+        signals_key = "#{identity}-signals"
+
+        assert_nil Sidekiq.redis { |c| c.lpop signals_key }
+        post '/busy', 'stop' => '1', 'identity' => identity
         assert_equal 302, last_response.status
-        assert_equal 'TERM', Sidekiq.redis { |c| c.lpop "host:pid-signals" }
+        assert_equal 'TERM', Sidekiq.redis { |c| c.lpop signals_key }
       end
     end
 

test/test_worker_generator.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'rails/generators/test_case'
 require 'generators/sidekiq/worker_generator'
 

web/locales/ru.yml

@@ -66,3 +66,9 @@ ru:
   Thread: Поток
   Threads: Потоки
   Jobs: Задачи
+  Paused: Приостановлено
+  Stop: Остановить
+  Quiet: Отдыхать
+  StopAll: Остановить все
+  QuietAll: Отдыхать всем
+  PollingInterval: Интервал опроса

web/views/busy.erb

@@ -39,8 +39,7 @@
       <td>
         <div class="btn-group pull-right">
           <form method="POST">
-            <input type="hidden" name="hostname" value="<%= process['hostname'] %>"/>
-            <input type="hidden" name="pid" value="<%= process['pid'] %>"/>
+            <input type="hidden" name="identity" value="<%= process['identity'] %>"/>
             <button class="btn btn-warn" type="submit" name="quiet" value="1"><%= t('Quiet') %></button>
             <button class="btn btn-danger" type="submit" name="stop" value="1"><%= t('Stop') %></button>
           </form>