merge master

Changes.md

@@ -7,10 +7,23 @@
   Thank you to Sidekiq's newest committer, **badosu**, for writing the code
   and doing a lot of testing to ensure compatibility with many different
   3rd party plugins.  If your Web UI works with 4.1.4 but fails with
-  4.2.0, please open an issue.
+  4.2.0, please open an issue. [#3075]
 - Add support for development mode code reloading with Rails 5's new
   thread-safe Interlock API.  With Rails 5, you no longer need to
-  restart Sidekiq when making code changes locally!
+  restart Sidekiq when making code changes locally! [#2457]
+- Allow tuning of concurrency with the `RAILS_MAX_THREADS` env var. [#2985]
+  This is the same var used by Puma so you can tune all of your systems
+  the same way:
+```sh
+web: RAILS_MAX_THREADS=5 bundle exec puma ...
+worker: RAILS_MAX_THREADS=10 bundle exec sidekiq ...
+```
+Using `-c` or `config/sidekiq.yml` overrides this setting.  I recommend
+adjusting your `config/database.yml` to use it too so connections are
+auto-scaled:
+```yaml
+  pool: <%= ENV['RAILS_MAX_THREADS'] || 5 %>
+```
 
 4.1.4
 -----------

Pro-Changes.md

@@ -3,9 +3,10 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-HEAD
+3.3.2
 ---------
 
+- Minimize batch memory usage after success [#3083]
 - Extract batch's 24 hr linger expiry to a LINGER constant so it can be tuned. [#3011]
 
 3.3.1

lib/sidekiq.rb

@@ -28,6 +28,7 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
+      heartbeat: [],
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60 # 6 months

lib/sidekiq/cli.rb

@@ -208,6 +208,8 @@ module Sidekiq
       opts = parse_config(cfile).merge(opts) if cfile
 
       opts[:strict] = true if opts[:strict].nil?
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:identity] = identity
 
       options.merge!(opts)
     end

lib/sidekiq/launcher.rb

@@ -94,15 +94,19 @@ module Sidekiq
         end
         fails = procd = 0
 
-        _, _, _, msg = Sidekiq.redis do |conn|
+        _, exists, _, _, msg = Sidekiq.redis do |conn|
           conn.multi do
             conn.sadd('processes', key)
+            conn.exists(key)
             conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           end
         end
 
+        # first heartbeat or recovering from an outage and need to reestablish our heartbeat
+        fire_event(:heartbeat) if !exists
+
         return unless msg
 
         if JVM_RESERVED_SIGNALS.include?(msg)

lib/sidekiq/manager.rb

@@ -4,6 +4,7 @@ require 'sidekiq/util'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
 require 'thread'
+require 'set'
 
 module Sidekiq
 

test/test_cli.rb

@@ -42,6 +42,18 @@ class TestCli < Sidekiq::Test
       assert_equal 60, Sidekiq.options[:concurrency]
     end
 
+    it 'changes concurrency with ENV' do
+      begin
+        ENV['RAILS_MAX_THREADS'] = '9'
+        @cli.parse(['sidekiq', '-c', '60', '-r', './test/fake_env.rb'])
+        assert_equal 60, Sidekiq.options[:concurrency]
+        @cli.parse(['sidekiq', '-r', './test/fake_env.rb'])
+        assert_equal 9, Sidekiq.options[:concurrency]
+      ensure
+        ENV.delete('RAILS_MAX_THREADS')
+      end
+    end
+
     it 'changes queues' do
       @cli.parse(['sidekiq', '-q', 'foo', '-r', './test/fake_env.rb'])
       assert_equal ['foo'], Sidekiq.options[:queues]

test/test_launcher.rb

@@ -15,8 +15,6 @@ class TestLauncher < Sidekiq::Test
 
     describe 'heartbeat' do
       before do
-        uow = Object.new
-
         @mgr = new_manager(options)
         @launcher = Sidekiq::Launcher.new(options)
         @launcher.manager = @mgr
@@ -31,6 +29,18 @@ class TestLauncher < Sidekiq::Test
         $0 = @proctitle
       end
 
+      it 'fires new heartbeat events' do
+        i = 0
+        Sidekiq.on(:heartbeat) do
+          i += 1
+        end
+        assert_equal 0, i
+        @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
+        assert_equal 1, i
+        @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
+        assert_equal 1, i
+      end
+
       describe 'when manager is active' do
         before do
           Sidekiq::CLI::PROCTITLES << proc { "xyz" }

web/assets/stylesheets/application.css

@@ -752,3 +752,28 @@ div.interval-slider input {
 .redis-url {
   text-overflow: ellipsis;
 }
+
+.product-version {
+  color:white;
+}
+
+.warning-messages {
+  margin-top: 20px;
+  margin-bottom: 10px;
+}
+
+.toggle {
+  display: none;
+}
+
+.box {
+  width: 50%;
+}
+
+.checkbox-column {
+  width: 20px;
+}
+
+.delete-confirm {
+  width: 20%;
+}

web/locales/de.yml

@@ -48,7 +48,7 @@ de:
   NoScheduledFound: Keine geplanten Jobs gefunden
   When: Wann
   ScheduledJobs: Jobs in der Warteschlange
-  idle: inaktiv
+  idle: untätig
   active: aktiv
   Version: Version
   Connections: Verbindungen

web/views/_footer.erb

@@ -3,7 +3,7 @@
     <div class="container text-center">
         <ul class="nav">
           <li>
-            <p class="navbar-text" style="color:white;"><%= product_version %></p>
+            <p class="navbar-text product-version"><%= product_version %></p>
           </li>
           <li>
             <p class="navbar-text redis-url" title="<%= redis_connection_and_namespace %>"><%= redis_connection_and_namespace %></p>

web/views/busy.erb

@@ -3,7 +3,7 @@
     <h3><%= t('Processes') %></h3>
   </div>
   <div class="col-sm-4 pull-right">
-    <form method="POST" style="margin-top: 20px; margin-bottom: 10px;">
+    <form method="POST" class="warning-messages">
       <%= csrf_tag %>
       <div class="btn-group pull-right">
         <button class="btn btn-warn" type="submit" name="quiet" value="1" data-confirm="<%= t('AreYouSure') %>"><%= t('QuietAll') %></button>
@@ -24,7 +24,7 @@
     </thead>
     <% processes.each do |process| %>
       <tr>
-        <td width="50%">
+        <td class="box">
           <%= "#{process['hostname']}:#{process['pid']}" %>
           <span class="label label-success"><%= process.tag %></span>
           <% process.labels.each do |label| %>

web/views/morgue.erb

@@ -17,7 +17,7 @@
       <table class="table table-striped table-bordered table-white">
         <thead>
           <tr>
-            <th width="20px" class="table-checkbox">
+            <th class="table-checkbox checkbox-column">
               <label>
                 <input type="checkbox" class="check_all" />
               </label>

web/views/queue.erb

@@ -26,7 +26,7 @@
           <% if a.size > 100 %>
             <%= h(msg.display_args.inspect[0..100]) + "... " %>
             <button data-toggle="collapse" data-target="#worker_<%= index %>" class="btn btn-default btn-xs"><%= t('ShowAll') %></button>
-            <div class="toggle" id="worker_<%= index %>" style="display: none;"><%= h(msg.display_args) %></div>
+            <div class="toggle" id="worker_<%= index %>"><%= h(msg.display_args) %></div>
           <% else %>
             <%= h(msg.display_args) %>
           <% end %>

web/views/queues.erb

@@ -16,7 +16,7 @@
           <% end %>
         </td>
         <td><%= number_with_delimiter(queue.size) %> </td>
-        <td width="20%">
+        <td class="delete-confirm">
           <form action="<%=root_path %>queues/<%= queue.name %>" method="post">
             <%= csrf_tag %>
             <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSureDeleteQueue', :queue => h(queue.name)) %>" />

web/views/retries.erb

@@ -17,7 +17,7 @@
       <table class="table table-striped table-bordered table-white">
         <thead>
           <tr>
-            <th width="20px" class="table-checkbox">
+            <th class="table-checkbox checkbox-column">
               <label>
                 <input type="checkbox" class="check_all" />
               </label>

web/views/scheduled.erb

@@ -18,7 +18,7 @@
       <table class="table table-striped table-bordered table-white">
         <thead>
           <tr>
-            <th width="20px">
+            <th class="checkbox-column">
               <input type="checkbox" class="check_all" />
             </th>
             <th><%= t('When') %></th>