From 5bf2d06ded75f584adc4060d71b0773d0e687cd5 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Jul 2016 10:06:34 -0700 Subject: [PATCH 01/15] release 1.2.3 --- Ent-Changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Ent-Changes.md b/Ent-Changes.md index 725489e3..6ad779b7 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -3,7 +3,7 @@ Sidekiq Enterprise Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. -HEAD +1.2.3 ------------- - Periodic jobs could stop executing until process restart if Redis goes down [#3047] From f934127d409d2d03833e02f0a92f64a585ba80f3 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Thu, 7 Jul 2016 14:07:46 -0700 Subject: [PATCH 02/15] periodic off by one fix --- Ent-Changes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Ent-Changes.md b/Ent-Changes.md index 6ad779b7..41fe7976 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -3,6 +3,12 @@ Sidekiq Enterprise Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +1.2.4 +------------- + +- Fix issue causing some minutely jobs to execute every other minute. +- Log a warning if slow periodic processing causes us to miss a clock tick. + 1.2.3 ------------- From 0d6a0e9c74483e39c435a3bff82ffe828772d49a Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 18 Jul 2016 09:33:38 -0700 Subject: [PATCH 03/15] 1.3.0 --- Ent-Changes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Ent-Changes.md b/Ent-Changes.md index 41fe7976..a74b2398 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -3,6 +3,12 @@ Sidekiq Enterprise Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +1.3.0 +------------- + +- **BETA** [New encryption feature](https://github.com/mperham/sidekiq/wiki/Ent-Encryption) + which automatically encrypts the last argument of a Worker, aka the secret bag. + 1.2.4 ------------- From ab3ef75755df032d4302899abdfd17dc089166b7 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 20 Jul 2016 12:49:20 -0700 Subject: [PATCH 04/15] 131 --- Ent-Changes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Ent-Changes.md b/Ent-Changes.md index a74b2398..0472b4c2 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -3,6 +3,13 @@ Sidekiq Enterprise Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +1.3.1 +------------- + +- Fix multi-process memory monitoring on CentOS 6.x [#3063] +- Polish the new encryption feature a bit. + + 1.3.0 ------------- From c89ff2e03473e9346f91d798f5f78e74cf954905 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 25 Jul 2016 09:36:31 -0700 Subject: [PATCH 05/15] upgraded encryption --- Ent-Changes.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Ent-Changes.md b/Ent-Changes.md index 0472b4c2..34d4990c 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -3,13 +3,17 @@ Sidekiq Enterprise Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +1.3.2 +------------- + +- Upgrade encryption to use OpenSSL's more secure GCM mode. [#3060] + 1.3.1 ------------- - Fix multi-process memory monitoring on CentOS 6.x [#3063] - Polish the new encryption feature a bit. - 1.3.0 ------------- From 9e390bf37b3b121dc3880e6589243a48705a2c6e Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 26 Jul 2016 09:57:09 -0700 Subject: [PATCH 06/15] LINGER constant --- Pro-Changes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Pro-Changes.md b/Pro-Changes.md index afbfd7f5..2c04d2e3 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -3,6 +3,11 @@ Sidekiq Pro Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +HEAD +--------- + +- Extract batch's 24 hr linger expiry to a LINGER constant so it can be tuned. [#3011] + 3.3.1 --------- From 8f085052dc2d2d09fcc43460a0ee6f9ce8cfc964 Mon Sep 17 00:00:00 2001 From: Spoonflower Date: Fri, 29 Jul 2016 10:09:17 -0400 Subject: [PATCH 07/15] taking out inline styling and putting them in application.css --- web/assets/stylesheets/application.css | 25 +++++++++++++++++++++++++ web/views/_footer.erb | 2 +- web/views/busy.erb | 4 ++-- web/views/morgue.erb | 2 +- web/views/queue.erb | 2 +- web/views/queues.erb | 2 +- web/views/retries.erb | 2 +- web/views/scheduled.erb | 2 +- 8 files changed, 33 insertions(+), 8 deletions(-) diff --git a/web/assets/stylesheets/application.css b/web/assets/stylesheets/application.css index 64e8e60e..d3b1535b 100755 --- a/web/assets/stylesheets/application.css +++ b/web/assets/stylesheets/application.css @@ -752,3 +752,28 @@ div.interval-slider input { .redis-url { text-overflow: ellipsis; } + +#product_version { + color:white; +} + +#warningmessages { + margin-top: 20px; + margin-bottom: 10px; +} + +.toggle { + display: none; +} + +#box { + width: 50%; +} + +.width { + width: 20px; +} + +#deleteconfirm { + width: 20%; +} diff --git a/web/views/_footer.erb b/web/views/_footer.erb index e2296874..2b90c896 100644 --- a/web/views/_footer.erb +++ b/web/views/_footer.erb @@ -3,7 +3,7 @@
  • -

    <%= product_version %>

    +

    <%= product_version %>

  • <%= redis_connection_and_namespace %>

    diff --git a/web/views/busy.erb b/web/views/busy.erb index 82972196..4b7fbb36 100644 --- a/web/views/busy.erb +++ b/web/views/busy.erb @@ -3,7 +3,7 @@

    <%= t('Processes') %>

-
+ <%= csrf_tag %>
@@ -24,7 +24,7 @@ <% processes.each do |process| %> - + <%= "#{process['hostname']}:#{process['pid']}" %> <%= process.tag %> <% process.labels.each do |label| %> diff --git a/web/views/morgue.erb b/web/views/morgue.erb index 9ff0a052..68adb917 100644 --- a/web/views/morgue.erb +++ b/web/views/morgue.erb @@ -17,7 +17,7 @@ - -
+ diff --git a/web/views/queue.erb b/web/views/queue.erb index d9112c07..3e041ee3 100644 --- a/web/views/queue.erb +++ b/web/views/queue.erb @@ -26,7 +26,7 @@ <% if a.size > 100 %> <%= h(msg.display_args.inspect[0..100]) + "... " %> - +
<%= h(msg.display_args) %>
<% else %> <%= h(msg.display_args) %> <% end %> diff --git a/web/views/queues.erb b/web/views/queues.erb index ce9d4c37..1d2d8791 100644 --- a/web/views/queues.erb +++ b/web/views/queues.erb @@ -16,7 +16,7 @@ <% end %> 		<%= number_with_delimiter(queue.size) %> + <%= csrf_tag %> diff --git a/web/views/retries.erb b/web/views/retries.erb index e0ab51ba..5cccc210 100644 --- a/web/views/retries.erb +++ b/web/views/retries.erb @@ -17,7 +17,7 @@ -
+ diff --git a/web/views/scheduled.erb b/web/views/scheduled.erb index 17b41e57..9566e72f 100644 --- a/web/views/scheduled.erb +++ b/web/views/scheduled.erb @@ -18,7 +18,7 @@ - From d2c927a7b2767a6eefc4489963df48235d163aaa Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 1 Aug 2016 09:06:45 -0700 Subject: [PATCH 08/15] 3.3.2 --- Pro-Changes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pro-Changes.md b/Pro-Changes.md index 2c04d2e3..ee694d42 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -3,9 +3,10 @@ Sidekiq Pro Changelog Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. -HEAD +3.3.2 --------- +- Minimize batch memory usage after success [#3083] - Extract batch's 24 hr linger expiry to a LINGER constant so it can be tuned. [#3011] 3.3.1 From 52828e42123cc1c54b2a326bda5d00965ec05acb Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 1 Aug 2016 09:33:12 -0700 Subject: [PATCH 09/15] Tune concurrency via env, fixes #2985 --- Changes.md | 18 ++++++++++++++++++ lib/sidekiq/cli.rb | 1 + test/test_cli.rb | 12 ++++++++++++ 3 files changed, 31 insertions(+) diff --git a/Changes.md b/Changes.md index deb64c7f..bd008a50 100644 --- a/Changes.md +++ b/Changes.md @@ -1,5 +1,23 @@ # Sidekiq Changes +HEAD +----------- + +- Allow tuning of concurrency with the `RAILS_MAX_THREADS` env var. [#2985] + This is the same var used by Puma so you can tune all of your systems + the same way: +```sh +web: RAILS_MAX_THREADS=5 bundle exec puma ... +worker: RAILS_MAX_THREADS=10 bundle exec sidekiq ... +``` +Using `-c` or `config/sidekiq.yml` overrides this setting. I recommend +adjusting your `config/database.yml` to use it too so connections are +auto-scaled: +```yaml + pool: <%= ENV['RAILS_MAX_THREADS'] || 5 %> +``` + + 4.1.4 ----------- diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index c7ff6147..aabb981e 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -208,6 +208,7 @@ module Sidekiq opts = parse_config(cfile).merge(opts) if cfile opts[:strict] = true if opts[:strict].nil? + opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"] options.merge!(opts) end diff --git a/test/test_cli.rb b/test/test_cli.rb index b487abec..3388bae3 100644 --- a/test/test_cli.rb +++ b/test/test_cli.rb @@ -42,6 +42,18 @@ class TestCli < Sidekiq::Test assert_equal 60, Sidekiq.options[:concurrency] end + it 'changes concurrency with ENV' do + begin + ENV['RAILS_MAX_THREADS'] = '9' + @cli.parse(['sidekiq', '-c', '60', '-r', './test/fake_env.rb']) + assert_equal 60, Sidekiq.options[:concurrency] + @cli.parse(['sidekiq', '-r', './test/fake_env.rb']) + assert_equal 9, Sidekiq.options[:concurrency] + ensure + ENV.delete('RAILS_MAX_THREADS') + end + end + it 'changes queues' do @cli.parse(['sidekiq', '-q', 'foo', '-r', './test/fake_env.rb']) assert_equal ['foo'], Sidekiq.options[:queues] From e634177565a9b5194cfd8a854156b2792c0ef12a Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 3 Aug 2016 14:07:20 -0700 Subject: [PATCH 10/15] save process identity as global for all to share and enjoy --- lib/sidekiq/cli.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index aabb981e..ed251f69 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -209,6 +209,7 @@ module Sidekiq opts[:strict] = true if opts[:strict].nil? opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"] + opts[:identity] = identity options.merge!(opts) end From dee9c8bd86ea6a5974a78372ef3b5ed21e6cc186 Mon Sep 17 00:00:00 2001 From: Spoonflower Date: Thu, 4 Aug 2016 11:18:07 -0400 Subject: [PATCH 11/15] change inline style into classes --- web/assets/stylesheets/application.css | 10 +++++----- web/views/_footer.erb | 2 +- web/views/busy.erb | 4 ++-- web/views/morgue.erb | 2 +- web/views/queues.erb | 2 +- web/views/retries.erb | 2 +- web/views/scheduled.erb | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/web/assets/stylesheets/application.css b/web/assets/stylesheets/application.css index d3b1535b..fd24a1a2 100755 --- a/web/assets/stylesheets/application.css +++ b/web/assets/stylesheets/application.css @@ -753,11 +753,11 @@ div.interval-slider input { text-overflow: ellipsis; } -#product_version { +.product-version { color:white; } -#warningmessages { +.warning-messages { margin-top: 20px; margin-bottom: 10px; } @@ -766,14 +766,14 @@ div.interval-slider input { display: none; } -#box { +.box { width: 50%; } -.width { +.checkbox-column { width: 20px; } -#deleteconfirm { +.delete-confirm { width: 20%; } diff --git a/web/views/_footer.erb b/web/views/_footer.erb index 2b90c896..e482a316 100644 --- a/web/views/_footer.erb +++ b/web/views/_footer.erb @@ -3,7 +3,7 @@
  • -

    <%= product_version %>

    +

    <%= product_version %>

  • <%= redis_connection_and_namespace %>

    diff --git a/web/views/busy.erb b/web/views/busy.erb index 4b7fbb36..556f05a6 100644 --- a/web/views/busy.erb +++ b/web/views/busy.erb @@ -3,7 +3,7 @@

    <%= t('Processes') %>

- + <%= csrf_tag %>
@@ -24,7 +24,7 @@
<% processes.each do |process| %> -
+ <%= t('When') %>
+ <%= "#{process['hostname']}:#{process['pid']}" %> <%= process.tag %> <% process.labels.each do |label| %> diff --git a/web/views/morgue.erb b/web/views/morgue.erb index 68adb917..1f1cd482 100644 --- a/web/views/morgue.erb +++ b/web/views/morgue.erb @@ -17,7 +17,7 @@ - -
+ diff --git a/web/views/queues.erb b/web/views/queues.erb index 1d2d8791..2399fba2 100644 --- a/web/views/queues.erb +++ b/web/views/queues.erb @@ -16,7 +16,7 @@ <% end %> <%= number_with_delimiter(queue.size) %> + <%= csrf_tag %> diff --git a/web/views/retries.erb b/web/views/retries.erb index 5cccc210..3deac5a0 100644 --- a/web/views/retries.erb +++ b/web/views/retries.erb @@ -17,7 +17,7 @@ -
+ diff --git a/web/views/scheduled.erb b/web/views/scheduled.erb index 9566e72f..9d3f008e 100644 --- a/web/views/scheduled.erb +++ b/web/views/scheduled.erb @@ -18,7 +18,7 @@ - From b6de0bae6b86d5abae25852141768d8ecc8ddedf Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Sun, 7 Aug 2016 11:56:43 -0700 Subject: [PATCH 12/15] require set --- lib/sidekiq/manager.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sidekiq/manager.rb b/lib/sidekiq/manager.rb index a267d71c..b033e40d 100644 --- a/lib/sidekiq/manager.rb +++ b/lib/sidekiq/manager.rb @@ -4,6 +4,7 @@ require 'sidekiq/util' require 'sidekiq/processor' require 'sidekiq/fetch' require 'thread' +require 'set' module Sidekiq From 008f1feff3690933795506fe4ce9baa07bc134a2 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Fri, 12 Aug 2016 12:34:41 -0700 Subject: [PATCH 13/15] Add event when a process has a new heartbeat --- lib/sidekiq.rb | 1 + lib/sidekiq/launcher.rb | 6 +++++- test/test_launcher.rb | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq.rb b/lib/sidekiq.rb index 6764fd1d..ec971606 100644 --- a/lib/sidekiq.rb +++ b/lib/sidekiq.rb @@ -28,6 +28,7 @@ module Sidekiq startup: [], quiet: [], shutdown: [], + heartbeat: [], }, dead_max_jobs: 10_000, dead_timeout_in_seconds: 180 * 24 * 60 * 60 # 6 months diff --git a/lib/sidekiq/launcher.rb b/lib/sidekiq/launcher.rb index d30ffd47..46154a37 100644 --- a/lib/sidekiq/launcher.rb +++ b/lib/sidekiq/launcher.rb @@ -94,15 +94,19 @@ module Sidekiq end fails = procd = 0 - _, _, _, msg = Sidekiq.redis do |conn| + _, exists, _, _, msg = Sidekiq.redis do |conn| conn.multi do conn.sadd('processes', key) + conn.exists(key) conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done) conn.expire(key, 60) conn.rpop("#{key}-signals") end end + # first heartbeat or recovering from an outage and need to reestablish our heartbeat + fire_event(:heartbeat) if !exists + return unless msg if JVM_RESERVED_SIGNALS.include?(msg) diff --git a/test/test_launcher.rb b/test/test_launcher.rb index bd86bd21..72f3270a 100644 --- a/test/test_launcher.rb +++ b/test/test_launcher.rb @@ -15,8 +15,6 @@ class TestLauncher < Sidekiq::Test describe 'heartbeat' do before do - uow = Object.new - @mgr = new_manager(options) @launcher = Sidekiq::Launcher.new(options) @launcher.manager = @mgr @@ -31,6 +29,18 @@ class TestLauncher < Sidekiq::Test $0 = @proctitle end + it 'fires new heartbeat events' do + i = 0 + Sidekiq.on(:heartbeat) do + i += 1 + end + assert_equal 0, i + @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data)) + assert_equal 1, i + @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data)) + assert_equal 1, i + end + describe 'when manager is active' do before do Sidekiq::CLI::PROCTITLES << proc { "xyz" } From c90da62128c60d54ccf4bed4feda224609643840 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Fri, 12 Aug 2016 12:58:20 -0700 Subject: [PATCH 14/15] bump --- lib/sidekiq/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sidekiq/version.rb b/lib/sidekiq/version.rb index 27dcc796..d0c61a03 100644 --- a/lib/sidekiq/version.rb +++ b/lib/sidekiq/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true module Sidekiq - VERSION = "4.1.4" + VERSION = "4.1.5" end From 9214b87a4d3ddcaa0fafc5cd7bd186c6d93344fa Mon Sep 17 00:00:00 2001 From: JJ Buckley Date: Thu, 18 Aug 2016 18:03:26 +0200 Subject: [PATCH 15/15] Fixed the translation of "idle" for the de locale "Inaktiv" has a specific connotation in German, which I suspect is not what we're looking to convey here. --- web/locales/de.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/locales/de.yml b/web/locales/de.yml index 24cf4564..b94710c9 100644 --- a/web/locales/de.yml +++ b/web/locales/de.yml @@ -48,7 +48,7 @@ de: NoScheduledFound: Keine geplanten Jobs gefunden When: Wann ScheduledJobs: Jobs in der Warteschlange - idle: inaktiv + idle: untätig active: aktiv Version: Version Connections: Verbindungen
+ <%= t('When') %>