From 8269205a73bbbc5c71c925e000afac405266c448 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Fri, 10 Jun 2022 11:35:59 -0700 Subject: [PATCH 01/22] notes --- Ent-Changes.md | 5 +++++ Pro-Changes.md | 5 +++++ lib/sidekiq/middleware/modules.rb | 2 ++ 3 files changed, 12 insertions(+) diff --git a/Ent-Changes.md b/Ent-Changes.md index 4ee5f656..d63e6d3e 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -4,6 +4,11 @@ Please see [sidekiq.org](https://sidekiq.org) for more details and how to buy. +HEAD +------------- + +- Fix crash with empty periodic data [#5374] + 2.5.0 ------------- diff --git a/Pro-Changes.md b/Pro-Changes.md index 836c67eb..8e920f9b 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -4,6 +4,11 @@ Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy. +HEAD +--------- + +- Unbreak queue pausing [#5382] + 5.5.0 --------- diff --git a/lib/sidekiq/middleware/modules.rb b/lib/sidekiq/middleware/modules.rb index 6ed875d1..5320bff3 100644 --- a/lib/sidekiq/middleware/modules.rb +++ b/lib/sidekiq/middleware/modules.rb @@ -1,4 +1,6 @@ module Sidekiq + # Server-side middleware must import this Module in order + # to get access to server resources during `call`. module ServerMiddleware attr_accessor :config def redis_pool From 2b58b74344d8400a8d5fe890501105e10c3b7ab9 Mon Sep 17 00:00:00 2001 From: Jean byroot Boussier Date: Mon, 13 Jun 2022 15:27:57 +0200 Subject: [PATCH 02/22] redis-client: Fix `ZADD` compatibility (#5387) --- lib/sidekiq/client.rb | 2 +- test/test_client.rb | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/sidekiq/client.rb b/lib/sidekiq/client.rb index a0b68a84..7e75f85a 100644 --- a/lib/sidekiq/client.rb +++ b/lib/sidekiq/client.rb @@ -220,7 +220,7 @@ module Sidekiq def atomic_push(conn, payloads) if payloads.first.key?("at") - conn.zadd("schedule", *payloads.map { |hash| + conn.zadd("schedule", payloads.flat_map { |hash| at = hash.delete("at").to_s [at, Sidekiq.dump_json(hash)] }) diff --git a/test/test_client.rb b/test/test_client.rb index 903abe83..6c8a4187 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -271,13 +271,16 @@ describe Sidekiq::Client do assert_equal 1_000, jids.size end - it "can push jobs scheduled at different times" do - first_at = Time.new(2019, 1, 1) - second_at = Time.new(2019, 1, 2) - jids = Sidekiq::Client.push_bulk("class" => QueuedWorker, "args" => [[1], [2]], "at" => [first_at.to_f, second_at.to_f]) - (first_jid, second_jid) = jids - assert_equal first_at, Sidekiq::ScheduledSet.new.find_job(first_jid).at - assert_equal second_at, Sidekiq::ScheduledSet.new.find_job(second_jid).at + [1, 2, 3].each do |job_count| + it "can push #{job_count} jobs scheduled at different times" do + times = job_count.times.map { |i| Time.new(2019, 1, i + 1) } + args = job_count.times.map { |i| [i] } + + jids = Sidekiq::Client.push_bulk("class" => QueuedWorker, "args" => args, "at" => times.map(&:to_f)) + + assert_equal job_count, jids.size + assert_equal times, jids.map { |jid| Sidekiq::ScheduledSet.new.find_job(jid).at } + end end it "can push jobs scheduled using ActiveSupport::Duration" do From 9102c3873514225ddba6852f83e4cfce3d5644ce Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 13 Jun 2022 10:03:12 -0700 Subject: [PATCH 03/22] Better rdoc for public APIs --- lib/sidekiq/api.rb | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 36497607..29bd4771 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -202,9 +202,10 @@ module Sidekiq end ## - # Encapsulates a queue within Sidekiq. + # Represents a queue within Sidekiq. # Allows enumeration of all jobs within the queue - # and deletion of jobs. + # and deletion of jobs. NB: this queue data is real-time + # and is changing within Redis moment by moment. # # queue = Sidekiq::Queue.new("mailer") # queue.each do |job| @@ -296,6 +297,7 @@ module Sidekiq end # delete all jobs within this queue + # @return [Boolean] true def clear Sidekiq.redis do |conn| conn.multi do |transaction| @@ -303,6 +305,7 @@ module Sidekiq transaction.srem("queues", name) end end + true end alias_method :💣, :clear @@ -312,15 +315,18 @@ module Sidekiq end ## - # Encapsulates a pending job within a Sidekiq queue or - # sorted set. + # Represents a pending job within a Sidekiq queue. # # The job should be considered immutable but may be # removed from the queue via JobRecord#delete. # class JobRecord + + # the parsed Hash of job data attr_reader :item + # the underlying String in Redis attr_reader :value + # the queue associated with this job attr_reader :queue def initialize(item, queue_name = nil) # :nodoc: @@ -341,6 +347,8 @@ module Sidekiq {} end + # This is the job class which Sidekiq will execute. If using ActiveJob, + # this class will be the ActiveJob adapter class rather than a specific job. def klass self["class"] end @@ -480,7 +488,9 @@ module Sidekiq end # Represents a job within a Redis sorted set where the score - # represents a timestamp for the job. + # represents a timestamp associated with the job. This timestamp + # could be the scheduled time for it to run (e.g. scheduled set), + # or the expiration date after which the entry should be deleted (e.g. dead set). class SortedEntry < JobRecord attr_reader :score attr_reader :parent @@ -491,10 +501,12 @@ module Sidekiq @parent = parent end + # The timestamp associated with this entry def at Time.at(score).utc end + # remove this entry from the sorted set def delete if @value @parent.delete_by_value(@parent.name, @value) @@ -505,7 +517,7 @@ module Sidekiq # Change the scheduled time for this job. # - # @param [Time] the new timestamp when this job will be enqueued. + # @param [Time] the new timestamp for this job def reschedule(at) Sidekiq.redis do |conn| conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item)) @@ -579,20 +591,30 @@ module Sidekiq end end + # Base class for all sorted sets within Sidekiq, e.g. scheduled, retry and dead. + # Sidekiq Pro and Enterprise add additional sorted sets for Batches, etc. class SortedSet include Enumerable + # Redis key of the set attr_reader :name - def initialize(name) + def initialize(name) # :nodoc: @name = name @_size = size end + # real-time size of the set, will change def size Sidekiq.redis { |c| c.zcard(name) } end + # Scan through each element of the sorted set, yielding each to the supplied block. + # Please see Redis's SCAN documentation for implementation details. + # + # @param match [String] a snippet or regexp to filter matches + # @param count [Integer] number of elements to retrieve at a time, default 100 + # @yieldparam [SortedEntry] each entry def scan(match, count = 100) return to_enum(:scan, match, count) unless block_given? From 4508477bf5c3460352107495ffc915be7b91b6b7 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 13 Jun 2022 10:43:52 -0700 Subject: [PATCH 04/22] more doc --- lib/sidekiq/api.rb | 79 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 29bd4771..cd43a4dc 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -591,8 +591,7 @@ module Sidekiq end end - # Base class for all sorted sets within Sidekiq, e.g. scheduled, retry and dead. - # Sidekiq Pro and Enterprise add additional sorted sets for Batches, etc. + # Base class for all sorted sets within Sidekiq. class SortedSet include Enumerable @@ -612,9 +611,9 @@ module Sidekiq # Scan through each element of the sorted set, yielding each to the supplied block. # Please see Redis's SCAN documentation for implementation details. # - # @param match [String] a snippet or regexp to filter matches + # @param match [String] a snippet or regexp to filter matches. # @param count [Integer] number of elements to retrieve at a time, default 100 - # @yieldparam [SortedEntry] each entry + # @yieldparam [Sidekiq::SortedEntry] each entry def scan(match, count = 100) return to_enum(:scan, match, count) unless block_given? @@ -626,10 +625,12 @@ module Sidekiq end end + # @return [Boolean] always true def clear Sidekiq.redis do |conn| conn.unlink(name) end + true end alias_method :💣, :clear @@ -638,10 +639,17 @@ module Sidekiq end end + # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead. + # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data, + # e.g. Batches. class JobSet < SortedSet - def schedule(timestamp, message) + + # Add a job with the associated timestamp to this set. + # @param timestamp [Time] the score for the job + # @param job [Hash] the job data + def schedule(timestamp, job) Sidekiq.redis do |conn| - conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message)) + conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job)) end end @@ -669,6 +677,11 @@ module Sidekiq ## # Fetch jobs that match a given time or Range. Job ID is an # optional second argument. + # + # @param score [Time] a specific timestamp + # @param score [Range] a timestamp range + # @param jid [String, optional] find a specific JID within the score + # @return [Array] any results found, can be empty def fetch(score, jid = nil) begin_score, end_score = if score.is_a?(Range) @@ -690,7 +703,10 @@ module Sidekiq ## # Find the job with the given JID within this sorted set. - # This is a slower O(n) operation. Do not use for app logic. + # *This is a slow O(n) operation*. Do not use for app logic. + # + # @param jid [String] the job identifier + # @returns [SortedEntry] the record or nil def find_job(jid) Sidekiq.redis do |conn| conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score| @@ -702,7 +718,7 @@ module Sidekiq nil end - def delete_by_value(name, value) + def delete_by_value(name, value) # :nodoc: Sidekiq.redis do |conn| ret = conn.zrem(name, value) @_size -= 1 if ret @@ -710,7 +726,7 @@ module Sidekiq end end - def delete_by_jid(score, jid) + def delete_by_jid(score, jid) # :nodoc: Sidekiq.redis do |conn| elements = conn.zrangebyscore(name, score, score) elements.each do |element| @@ -730,10 +746,10 @@ module Sidekiq end ## - # Allows enumeration of scheduled jobs within Sidekiq. + # The set of scheduled jobs within Sidekiq. # Based on this, you can search/filter for jobs. Here's an - # example where I'm selecting all jobs of a certain type - # and deleting them from the schedule queue. + # example where I'm selecting jobs based on some complex logic + # and deleting them from the scheduled set. # # r = Sidekiq::ScheduledSet.new # r.select do |scheduled| @@ -748,7 +764,7 @@ module Sidekiq end ## - # Allows enumeration of retries within Sidekiq. + # The set of retries within Sidekiq. # Based on this, you can search/filter for jobs. Here's an # example where I'm selecting all jobs of a certain type # and deleting them from the retry queue. @@ -764,23 +780,29 @@ module Sidekiq super "retry" end + # Enqueues all jobs pending within the retry set. def retry_all each(&:retry) while size > 0 end + # Kills all jobs pending within the retry set. def kill_all each(&:kill) while size > 0 end end ## - # Allows enumeration of dead jobs within Sidekiq. + # The set of dead jobs within Sidekiq. Dead jobs have failed all of + # their retries and are helding in this set pending some sort of manual + # fix. They will be removed after 6 months (dead_timeout) if not. # class DeadSet < JobSet def initialize super "dead" end + # Add the given job to the Dead set. + # @param message [String] the job data as JSON def kill(message, opts = {}) now = Time.now.to_f Sidekiq.redis do |conn| @@ -802,14 +824,19 @@ module Sidekiq true end + # Enqueue all dead jobs def retry_all each(&:retry) while size > 0 end + # The maximum size of the Dead set. Older entries will be trimmed + # to stay within this limit. Default value is 10,000. def self.max_jobs Sidekiq[:dead_max_jobs] end + # The time limit for entries within the Dead set. Older entries will be thrown away. + # Default value is six months. def self.timeout Sidekiq[:dead_timeout_in_seconds] end @@ -820,18 +847,18 @@ module Sidekiq # right now. Each process sends a heartbeat to Redis every 5 seconds # so this set should be relatively accurate, barring network partitions. # - # Yields a Sidekiq::Process. + # @yieldparam [Sidekiq::Process] # class ProcessSet include Enumerable - def initialize(clean_plz = true) + def initialize(clean_plz = true) # :nodoc: cleanup if clean_plz end # Cleans up dead processes recorded in Redis. # Returns the number of processes cleaned. - def cleanup + def cleanup # :nodoc: count = 0 Sidekiq.redis do |conn| procs = conn.sscan_each("processes").to_a.sort @@ -885,6 +912,7 @@ module Sidekiq # based on current heartbeat. #each does that and ensures the set only # contains Sidekiq processes which have sent a heartbeat within the last # 60 seconds. + # @return [Integer] current number of registered Sidekiq processes def size Sidekiq.redis { |conn| conn.scard("processes") } end @@ -892,10 +920,12 @@ module Sidekiq # Total number of threads available to execute jobs. # For Sidekiq Enterprise customers this number (in production) must be # less than or equal to your licensed concurrency. + # @return [Integer] the sum of process concurrency def total_concurrency sum { |x| x["concurrency"].to_i } end + # @return [Integer] total amount of RSS memory consumed by Sidekiq processes def total_rss_in_kb sum { |x| x["rss"].to_i } end @@ -930,7 +960,7 @@ module Sidekiq # 'identity' => , # } class Process - def initialize(hash) + def initialize(hash) # :nodoc: @attribs = hash end @@ -954,18 +984,31 @@ module Sidekiq self["queues"] end + # Signal this process to stop processing new jobs. + # It will continue to execute jobs it has already fetched. + # This method is *asynchronous* and it can take 5-10 + # seconds for the process to quiet. def quiet! signal("TSTP") end + # Signal this process to shutdown. + # It will shutdown within its configured :timeout value, default 25 seconds. + # This method is *asynchronous* and it can take 5-10 + # seconds for the process to start shutting down. def stop! signal("TERM") end + # Signal this process to log backtraces for all threads. + # Useful if you have a frozen or deadlocked process which is + # still sending a heartbeat. + # This method is *asynchronous* and it can take 5-10 seconds. def dump_threads signal("TTIN") end + # @return [Boolean] true if this process is quiet or shutting down def stopping? self["quiet"] == "true" end From 16b51f396125b75e9b0ec40174971a6b8ca5efcb Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 13 Jun 2022 15:22:26 -0700 Subject: [PATCH 05/22] switch from rdoc to yard --- Rakefile | 13 ++++++++----- docs/menu.md | 29 +++++++++++++++++++++++++++++ lib/sidekiq/api.rb | 7 +++---- 3 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 docs/menu.md diff --git a/Rakefile b/Rakefile index ffd8e4db..abc6a3ff 100644 --- a/Rakefile +++ b/Rakefile @@ -1,15 +1,18 @@ require "bundler/gem_tasks" require "rake/testtask" require "standard/rake" -require "rdoc/task" +require "yard" +require "yard/rake/yardoc_task" -RDoc::Task.new do |rdoc| - rdoc.main = "docs/rdoc.rdoc" - rdoc.rdoc_files.include("docs/rdoc.rdoc", +YARD::Rake::YardocTask.new do |yard| + yard.files = [ "lib/sidekiq/api.rb", "lib/sidekiq/client.rb", "lib/sidekiq/worker.rb", - "lib/sidekiq/job.rb") + # "lib/sidekiq/job.rb", + "-", + "Changes.md", + "docs/menu.md"] end Rake::TestTask.new(:test) do |test| diff --git a/docs/menu.md b/docs/menu.md new file mode 100644 index 00000000..f168c8c6 --- /dev/null +++ b/docs/menu.md @@ -0,0 +1,29 @@ +# Sidekiq public API documentation + +Sidekiq provides a number of public APIs for various functionality. + +1. Middleware +2. Lifecycle Events +3. Data API +4. Components + +## Middleware + +Middleware run around the the client-side push and the server-side execution of jobs. This allows plugins which mutate job data or provide additional functionality during the executiong of specific jobs. + +## Lifecycle Events + +With lifecycle events, Sidekiq plugins can register a callback upon `startup`, `quiet` or `shutdown`. +This is useful for starting and stopping your own Threads or services within the Sidekiq process. + +## Data API + +The code in `sidekiq/api` provides a Ruby facade on top of Sidekiq's persistent data within Redis. +It contains many classes and methods for discovering, searching and iterating through the real-time job data within the queues and sets inside Redis. +This API powers the Sidekiq::Web UI. + +## Components (ALPHA) + +Coming in Sidekiq 7.0, Components are elements of code which run inside each Sidekiq process. +They are passed a handle to the Sidekiq container which gives them direct access +to resources like the Logger, Redis connection pool and other registered resources. \ No newline at end of file diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index cd43a4dc..bf0177fe 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -517,7 +517,7 @@ module Sidekiq # Change the scheduled time for this job. # - # @param [Time] the new timestamp for this job + # @param at [Time] the new timestamp for this job def reschedule(at) Sidekiq.redis do |conn| conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item)) @@ -678,8 +678,7 @@ module Sidekiq # Fetch jobs that match a given time or Range. Job ID is an # optional second argument. # - # @param score [Time] a specific timestamp - # @param score [Range] a timestamp range + # @param score [Time,Range] a specific timestamp or range # @param jid [String, optional] find a specific JID within the score # @return [Array] any results found, can be empty def fetch(score, jid = nil) @@ -706,7 +705,7 @@ module Sidekiq # *This is a slow O(n) operation*. Do not use for app logic. # # @param jid [String] the job identifier - # @returns [SortedEntry] the record or nil + # @return [SortedEntry] the record or nil def find_job(jid) Sidekiq.redis do |conn| conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score| From dd1cf0901e9241d747e50aaa24fc57e3fd8a2aaa Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Tue, 14 Jun 2022 09:11:20 -0700 Subject: [PATCH 06/22] Add yardopts file --- .yardopts | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .yardopts diff --git a/.yardopts b/.yardopts new file mode 100644 index 00000000..14bf432e --- /dev/null +++ b/.yardopts @@ -0,0 +1,2 @@ +--title "Sidekiq" +--query '@return || @param || '@yield' || @api.text == "public"' \ No newline at end of file From be7b305e15e5fdb1cb12421252127ce8a2e34929 Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Tue, 14 Jun 2022 09:15:56 -0700 Subject: [PATCH 07/22] Make sidekiq api module public, so the classes show up --- lib/sidekiq/api.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index bf0177fe..3de980d0 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -5,6 +5,7 @@ require "sidekiq" require "zlib" require "base64" +# @api public module Sidekiq class Stats def initialize @@ -610,7 +611,7 @@ module Sidekiq # Scan through each element of the sorted set, yielding each to the supplied block. # Please see Redis's SCAN documentation for implementation details. - # + # # @param match [String] a snippet or regexp to filter matches. # @param count [Integer] number of elements to retrieve at a time, default 100 # @yieldparam [Sidekiq::SortedEntry] each entry From 272a1bab7fb55846b68ffa3242e51ff9dc08e4f2 Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Tue, 14 Jun 2022 09:22:08 -0700 Subject: [PATCH 08/22] Redirect readme to menu.md --- .yardopts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.yardopts b/.yardopts index 14bf432e..0d82902a 100644 --- a/.yardopts +++ b/.yardopts @@ -1,2 +1,3 @@ --title "Sidekiq" ---query '@return || @param || '@yield' || @api.text == "public"' \ No newline at end of file +--query '@return || @param || '@yield' || @api.text == "public"' +--readme docs/menu.md \ No newline at end of file From 2a24d541bd14e0f2802486e90f61036cf4d50aca Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Tue, 14 Jun 2022 09:25:04 -0700 Subject: [PATCH 09/22] Include changelog, comment out unnecessary code --- .yardopts | 3 ++- Rakefile | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.yardopts b/.yardopts index 0d82902a..e08ba6c1 100644 --- a/.yardopts +++ b/.yardopts @@ -1,3 +1,4 @@ --title "Sidekiq" --query '@return || @param || '@yield' || @api.text == "public"' ---readme docs/menu.md \ No newline at end of file +--readme docs/menu.md +- Changes.md \ No newline at end of file diff --git a/Rakefile b/Rakefile index abc6a3ff..84763fbe 100644 --- a/Rakefile +++ b/Rakefile @@ -5,14 +5,16 @@ require "yard" require "yard/rake/yardoc_task" YARD::Rake::YardocTask.new do |yard| - yard.files = [ - "lib/sidekiq/api.rb", - "lib/sidekiq/client.rb", - "lib/sidekiq/worker.rb", - # "lib/sidekiq/job.rb", - "-", - "Changes.md", - "docs/menu.md"] + # keeping this in here for now, but code can be deleted once we are ready + # to push. .yardopts takes care of this. + # yard.files = [ + # "lib/sidekiq/api.rb", + # "lib/sidekiq/client.rb", + # "lib/sidekiq/worker.rb", + # # "lib/sidekiq/job.rb", + # "-", + # "Changes.md", + # "docs/menu.md"] end Rake::TestTask.new(:test) do |test| From 67283ce73b02c3cca882c62e175803afd5a44896 Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Tue, 14 Jun 2022 17:26:58 -0700 Subject: [PATCH 10/22] Add yarddoc files to gitignore, remove yardoc from rake tasks, update api file --- .gitignore | 2 ++ .yardopts | 4 ++-- Rakefile | 14 ++---------- lib/sidekiq/api.rb | 53 ++++++++++++++++++++++++++++++++++------------ 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index f105b73a..6fb35943 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ development.log /docker-compose.yml Gemfile.lock *.DS_Store +doc/ +.yardoc/ \ No newline at end of file diff --git a/.yardopts b/.yardopts index e08ba6c1..e5223e5e 100644 --- a/.yardopts +++ b/.yardopts @@ -1,4 +1,4 @@ --title "Sidekiq" ---query '@return || @param || '@yield' || @api.text == "public"' +--query '(@return || @param || @yield || @attr_reader || @api.text == "public") && @api.text != "private"' --readme docs/menu.md -- Changes.md \ No newline at end of file +- Changes.md diff --git a/Rakefile b/Rakefile index 84763fbe..e6f65c0e 100644 --- a/Rakefile +++ b/Rakefile @@ -4,18 +4,8 @@ require "standard/rake" require "yard" require "yard/rake/yardoc_task" -YARD::Rake::YardocTask.new do |yard| - # keeping this in here for now, but code can be deleted once we are ready - # to push. .yardopts takes care of this. - # yard.files = [ - # "lib/sidekiq/api.rb", - # "lib/sidekiq/client.rb", - # "lib/sidekiq/worker.rb", - # # "lib/sidekiq/job.rb", - # "-", - # "Changes.md", - # "docs/menu.md"] -end +# If you want to generate the docs, run yarddoc from your terminal +# https://rubydoc.info/gems/yard/file/README.md Rake::TestTask.new(:test) do |test| test.warning = true diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 3de980d0..04e8809d 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -7,6 +7,7 @@ require "base64" # @api public module Sidekiq + # @api private class Stats def initialize fetch_stats_fast! @@ -214,7 +215,6 @@ module Sidekiq # job.args # => [1, 2, 3] # job.delete if job.jid == 'abcdef1234567890' # end - # class Queue include Enumerable @@ -310,7 +310,9 @@ module Sidekiq end alias_method :💣, :clear - def as_json(options = nil) # :nodoc: + # :nodoc: + # @api private + def as_json(options = nil) {name: name} # 5336 end end @@ -320,24 +322,30 @@ module Sidekiq # # The job should be considered immutable but may be # removed from the queue via JobRecord#delete. - # class JobRecord # the parsed Hash of job data + # @!attribute [r] Item attr_reader :item # the underlying String in Redis + # @!attribute [r] Value attr_reader :value # the queue associated with this job + # @!attribute [r] Queue attr_reader :queue - def initialize(item, queue_name = nil) # :nodoc: + # :nodoc: + # @api private + def initialize(item, queue_name = nil) @args = nil @value = item @item = item.is_a?(Hash) ? item : parse(item) @queue = queue_name || @item["queue"] end - def parse(item) # :nodoc: + # :nodoc: + # @api private + def parse(item) Sidekiq.load_json(item) rescue JSON::ParserError # If the job payload in Redis is invalid JSON, we'll load @@ -496,7 +504,9 @@ module Sidekiq attr_reader :score attr_reader :parent - def initialize(parent, score, item) # :nodoc: + # :nodoc: + # @api private + def initialize(parent, score, item) super(item) @score = Float(score) @parent = parent @@ -597,9 +607,12 @@ module Sidekiq include Enumerable # Redis key of the set + # @!attribute [r] Name attr_reader :name - def initialize(name) # :nodoc: + # :nodoc: + # @api private + def initialize(name) @name = name @_size = size end @@ -635,7 +648,9 @@ module Sidekiq end alias_method :💣, :clear - def as_json(options = nil) # :nodoc: + # :nodoc: + # @api private + def as_json(options = nil) {name: name} # 5336 end end @@ -718,7 +733,9 @@ module Sidekiq nil end - def delete_by_value(name, value) # :nodoc: + # :nodoc: + # @api private + def delete_by_value(name, value) Sidekiq.redis do |conn| ret = conn.zrem(name, value) @_size -= 1 if ret @@ -726,7 +743,9 @@ module Sidekiq end end - def delete_by_jid(score, jid) # :nodoc: + # :nodoc: + # @api private + def delete_by_jid(score, jid) Sidekiq.redis do |conn| elements = conn.zrangebyscore(name, score, score) elements.each do |element| @@ -852,13 +871,17 @@ module Sidekiq class ProcessSet include Enumerable - def initialize(clean_plz = true) # :nodoc: + # :nodoc: + # @api private + def initialize(clean_plz = true) cleanup if clean_plz end # Cleans up dead processes recorded in Redis. # Returns the number of processes cleaned. - def cleanup # :nodoc: + # :nodoc: + # @api private + def cleanup count = 0 Sidekiq.redis do |conn| procs = conn.sscan_each("processes").to_a.sort @@ -934,6 +957,8 @@ module Sidekiq # Returns the identity of the current cluster leader or "" if no leader. # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq # or Sidekiq Pro. + # @return [String] Identity of cluster leader + # @return [String] empty string if no leader def leader @leader ||= begin x = Sidekiq.redis { |c| c.get("dear-leader") } @@ -960,7 +985,9 @@ module Sidekiq # 'identity' => , # } class Process - def initialize(hash) # :nodoc: + # :nodoc: + # @api private + def initialize(hash) @attribs = hash end From 311bee55696298eca9d189b265b82574bad46593 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 15 Jun 2022 08:15:22 -0700 Subject: [PATCH 11/22] polish --- Gemfile | 2 +- Rakefile | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 4a73b6dd..a4443c47 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "rails", "~> 6.0" gem "sqlite3", platforms: :ruby gem "activerecord-jdbcsqlite3-adapter", platforms: :jruby gem "after_commit_everywhere" -gem "yard" + # mail dependencies gem "net-smtp", platforms: :mri, require: false diff --git a/Rakefile b/Rakefile index e6f65c0e..df7d3371 100644 --- a/Rakefile +++ b/Rakefile @@ -1,11 +1,12 @@ require "bundler/gem_tasks" require "rake/testtask" require "standard/rake" -require "yard" -require "yard/rake/yardoc_task" -# If you want to generate the docs, run yarddoc from your terminal -# https://rubydoc.info/gems/yard/file/README.md +# If you want to generate API docs: +# gem install yard && yard && open doc/index.html +# YARD readme: https://rubydoc.info/gems/yard/file/README.md +# YARD tags: https://www.rubydoc.info/gems/yard/file/docs/Tags.md +# YARD cheatsheet: https://gist.github.com/phansch/db18a595d2f5f1ef16646af72fe1fb0e Rake::TestTask.new(:test) do |test| test.warning = true From 8e3dd2e6e898869b5026a76d2e01634737f41744 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 15 Jun 2022 10:02:49 -0700 Subject: [PATCH 12/22] fix missing component usage --- lib/sidekiq/job_retry.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sidekiq/job_retry.rb b/lib/sidekiq/job_retry.rb index 3e53a3a7..5fefdf35 100644 --- a/lib/sidekiq/job_retry.rb +++ b/lib/sidekiq/job_retry.rb @@ -176,7 +176,7 @@ module Sidekiq # logger.debug { "Failure! Retry #{count} in #{delay} seconds" } retry_at = Time.now.to_f + delay payload = Sidekiq.dump_json(msg) - Sidekiq.redis do |conn| + redis do |conn| conn.zadd("retry", retry_at.to_s, payload) end else @@ -195,7 +195,7 @@ module Sidekiq send_to_morgue(msg) unless msg["dead"] == false - Sidekiq.death_handlers.each do |handler| + config.death_handlers.each do |handler| handler.call(msg, exception) rescue => e handle_exception(e, {context: "Error calling death handler", job: msg}) From f57c1a45b5b164c667ebfdf48f647c8b1f66a921 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 15 Jun 2022 10:05:14 -0700 Subject: [PATCH 13/22] bump --- Changes.md | 5 +++++ lib/sidekiq/version.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index a573fc50..2c0d887a 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,11 @@ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md) +6.5.1 +---------- + +- Fix `push_bulk` breakage [#5387] + 6.5.0 --------- diff --git a/lib/sidekiq/version.rb b/lib/sidekiq/version.rb index adf8eba4..23123176 100644 --- a/lib/sidekiq/version.rb +++ b/lib/sidekiq/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Sidekiq - VERSION = "6.5.0" + VERSION = "6.5.1" end From 8559fc1d7bafc90d6dce7f161daabba4b2f2c6b2 Mon Sep 17 00:00:00 2001 From: SamArdrey Date: Wed, 15 Jun 2022 10:06:37 -0700 Subject: [PATCH 14/22] Get opts working: --- .yardopts | 3 +-- lib/sidekiq/api.rb | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.yardopts b/.yardopts index e5223e5e..9138bb53 100644 --- a/.yardopts +++ b/.yardopts @@ -1,4 +1,3 @@ --title "Sidekiq" ---query '(@return || @param || @yield || @attr_reader || @api.text == "public") && @api.text != "private"' --readme docs/menu.md -- Changes.md +lib/sidekiq/api.rb lib/sidekiq/middleware/chain.rb - Changes.md diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 04e8809d..7b42d4a6 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -5,7 +5,6 @@ require "sidekiq" require "zlib" require "base64" -# @api public module Sidekiq # @api private class Stats From 9d33caf487df94938bb39a8887983c1b28df5662 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 15 Jun 2022 10:08:20 -0700 Subject: [PATCH 15/22] comm notes --- Ent-Changes.md | 2 +- Pro-Changes.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Ent-Changes.md b/Ent-Changes.md index d63e6d3e..88e97af1 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -4,7 +4,7 @@ Please see [sidekiq.org](https://sidekiq.org) for more details and how to buy. -HEAD +2.5.1 ------------- - Fix crash with empty periodic data [#5374] diff --git a/Pro-Changes.md b/Pro-Changes.md index 8e920f9b..4cf2fb08 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -4,7 +4,7 @@ Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy. -HEAD +5.5.1 --------- - Unbreak queue pausing [#5382] From e830ecd7ac1539877de03e57fb4807765f5fa392 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Thu, 16 Jun 2022 11:25:25 +0800 Subject: [PATCH 16/22] Delete .DS_Store (#5392) --- lib/sidekiq/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/sidekiq/.DS_Store diff --git a/lib/sidekiq/.DS_Store b/lib/sidekiq/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Mon, 20 Jun 2022 15:06:49 -0700 Subject: [PATCH 17/22] Hide endpoints tagged as private from build --- .yardopts | 1 + 1 file changed, 1 insertion(+) diff --git a/.yardopts b/.yardopts index 9138bb53..8f04328b 100644 --- a/.yardopts +++ b/.yardopts @@ -1,3 +1,4 @@ --title "Sidekiq" --readme docs/menu.md +--hide-api private lib/sidekiq/api.rb lib/sidekiq/middleware/chain.rb - Changes.md From 1e7ace608b5e10f77133ee21c2d70671a413fee9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 06:29:08 +0200 Subject: [PATCH 18/22] Bump actions/dependency-review-action from 1 to 2 (#5399) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 1 to 2. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/v1...v2) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/depsreview.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml index f2605b7a..a25de591 100644 --- a/.github/workflows/depsreview.yaml +++ b/.github/workflows/depsreview.yaml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v3 - name: 'Dependency Review' - uses: actions/dependency-review-action@v1 + uses: actions/dependency-review-action@v2 From 43ff39529445dabed5ef97c2bcc8fe014be38156 Mon Sep 17 00:00:00 2001 From: Takuya Kato <51745522+taakuuyaa@users.noreply.github.com> Date: Sun, 3 Jul 2022 12:55:31 +0900 Subject: [PATCH 19/22] fix typo (#5411) --- .github/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/contributing.md b/.github/contributing.md index c1d8a153..ec8a0bb4 100644 --- a/.github/contributing.md +++ b/.github/contributing.md @@ -66,7 +66,7 @@ bundle install redis-server ``` -#### 7. Navivate to myapp (small Rails app inside Sidekiq repository used for development) +#### 7. Navigate to myapp (small Rails app inside Sidekiq repository used for development) ``` cd myapp/ From ec589de5ef122cb985f8b4774154bb793b205440 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 12 Jul 2022 13:15:10 -0700 Subject: [PATCH 20/22] better yard for middleware --- lib/sidekiq/api.rb | 1 - lib/sidekiq/middleware/chain.rb | 105 +++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 36 deletions(-) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 7b42d4a6..503ab6d0 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -6,7 +6,6 @@ require "zlib" require "base64" module Sidekiq - # @api private class Stats def initialize fetch_stats_fast! diff --git a/lib/sidekiq/middleware/chain.rb b/lib/sidekiq/middleware/chain.rb index d19a21ef..7757441a 100644 --- a/lib/sidekiq/middleware/chain.rb +++ b/lib/sidekiq/middleware/chain.rb @@ -4,84 +4,98 @@ require "sidekiq/middleware/modules" module Sidekiq # Middleware is code configured to run before/after - # a message is processed. It is patterned after Rack + # a job is processed. It is patterned after Rack # middleware. Middleware exists for the client side # (pushing jobs onto the queue) as well as the server # side (when jobs are actually processed). # + # Callers will register middleware Classes and Sidekiq will + # create new instances of the middleware for every job. This + # is important so that instance state is not shared accidentally + # between job executions. + # # To add middleware for the client: # - # Sidekiq.configure_client do |config| - # config.client_middleware do |chain| - # chain.add MyClientHook + # Sidekiq.configure_client do |config| + # config.client_middleware do |chain| + # chain.add MyClientHook + # end # end - # end # # To modify middleware for the server, just call # with another block: # - # Sidekiq.configure_server do |config| - # config.server_middleware do |chain| - # chain.add MyServerHook - # chain.remove ActiveRecord + # Sidekiq.configure_server do |config| + # config.server_middleware do |chain| + # chain.add MyServerHook + # chain.remove ActiveRecord + # end # end - # end # # To insert immediately preceding another entry: # - # Sidekiq.configure_client do |config| - # config.client_middleware do |chain| - # chain.insert_before ActiveRecord, MyClientHook + # Sidekiq.configure_client do |config| + # config.client_middleware do |chain| + # chain.insert_before ActiveRecord, MyClientHook + # end # end - # end # # To insert immediately after another entry: # - # Sidekiq.configure_client do |config| - # config.client_middleware do |chain| - # chain.insert_after ActiveRecord, MyClientHook + # Sidekiq.configure_client do |config| + # config.client_middleware do |chain| + # chain.insert_after ActiveRecord, MyClientHook + # end # end - # end # # This is an example of a minimal server middleware: # - # class MyServerHook - # include Sidekiq::ServerMiddleware - # def call(job_instance, msg, queue) - # logger.info "Before job" - # redis {|conn| conn.get("foo") } # do something in Redis - # yield - # logger.info "After job" + # class MyServerHook + # include Sidekiq::ServerMiddleware + # + # def call(job_instance, msg, queue) + # logger.info "Before job" + # redis {|conn| conn.get("foo") } # do something in Redis + # yield + # logger.info "After job" + # end # end - # end # # This is an example of a minimal client middleware, note # the method must return the result or the job will not push # to Redis: # - # class MyClientHook - # include Sidekiq::ClientMiddleware - # def call(job_class, msg, queue, redis_pool) - # logger.info "Before push" - # result = yield - # logger.info "After push" - # result + # class MyClientHook + # include Sidekiq::ClientMiddleware + # + # def call(job_class, msg, queue, redis_pool) + # logger.info "Before push" + # result = yield + # logger.info "After push" + # result + # end # end - # end # module Middleware class Chain include Enumerable + # A unique instance of the middleware chain is created for + # each job executed in order to be thread-safe. + # @param copy [Sidekiq::Middleware::Chain] New instance of Chain + # @returns nil def initialize_copy(copy) copy.instance_variable_set(:@entries, entries.dup) + nil end + # Iterate through each middleware in the chain def each(&block) entries.each(&block) end - def initialize(config = nil) + # @api private + def initialize(config = nil) # :nodoc: @config = config @entries = nil yield self if block_given? @@ -91,20 +105,33 @@ module Sidekiq @entries ||= [] end + # Remove all middleware matching the given Class + # @param klass [Class] def remove(klass) entries.delete_if { |entry| entry.klass == klass } end + # Add the given middleware to the end of the chain. + # Sidekiq will call `klass.new(*args)` to create a clean + # copy of your middleware for every job executed. + # + # chain.add(Statsd::Metrics, { collector: "localhost:8125" }) + # + # @param klass [Class] Your middleware class + # @param *args [Array] Set of arguments to pass to every instance of your middleware def add(klass, *args) remove(klass) entries << Entry.new(@config, klass, *args) end + # Identical to {#add} except the middleware is added to the front of the chain. def prepend(klass, *args) remove(klass) entries.insert(0, Entry.new(@config, klass, *args)) end + # Inserts +newklass+ before +oldklass+ in the chain. + # Useful if one middleware must run before another middleware. def insert_before(oldklass, newklass, *args) i = entries.index { |entry| entry.klass == newklass } new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i) @@ -112,6 +139,8 @@ module Sidekiq entries.insert(i, new_entry) end + # Inserts +newklass+ after +oldklass+ in the chain. + # Useful if one middleware must run after another middleware. def insert_after(oldklass, newklass, *args) i = entries.index { |entry| entry.klass == newklass } new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i) @@ -119,10 +148,12 @@ module Sidekiq entries.insert(i + 1, new_entry) end + # @return [Boolean] if the given class is already in the chain def exists?(klass) any? { |entry| entry.klass == klass } end + # @return [Boolean] if the chain contains no middleware def empty? @entries.nil? || @entries.empty? end @@ -135,6 +166,8 @@ module Sidekiq entries.clear end + # Used by Sidekiq to execute the middleware at runtime + # @api private def invoke(*args) return yield if empty? @@ -152,6 +185,8 @@ module Sidekiq private + # Represents each link in the middleware chain + # @api private class Entry attr_reader :klass From cdb1eac495c24f7b0e7b69f758b1af3022e32606 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 12 Jul 2022 13:23:35 -0700 Subject: [PATCH 21/22] cleanup stats yard --- lib/sidekiq/api.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 503ab6d0..2c3ff694 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -6,6 +6,12 @@ require "zlib" require "base64" module Sidekiq + + # Retrieve runtime statistics from Redis regarding + # this Sidekiq cluster. + # + # stat = Sidekiq::Stats.new + # stat.processed class Stats def initialize fetch_stats_fast! @@ -52,6 +58,7 @@ module Sidekiq end # O(1) redis calls + # @api private def fetch_stats_fast! pipe1_res = Sidekiq.redis { |conn| conn.pipelined do |pipeline| @@ -91,6 +98,7 @@ module Sidekiq end # O(number of processes + number of queues) redis calls + # @api private def fetch_stats_slow! processes = Sidekiq.redis { |conn| conn.sscan_each("processes").to_a @@ -116,11 +124,13 @@ module Sidekiq @stats end + # @api private def fetch_stats! fetch_stats_fast! fetch_stats_slow! end + # @api private def reset(*stats) all = %w[failed processed] stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s) From b73994056d7e07395c8dfeaa5edc469ede56c936 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 12 Jul 2022 13:35:39 -0700 Subject: [PATCH 22/22] formatting --- lib/sidekiq/api.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 2c3ff694..9f3669a8 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -6,7 +6,6 @@ require "zlib" require "base64" module Sidekiq - # Retrieve runtime statistics from Redis regarding # this Sidekiq cluster. # @@ -331,7 +330,6 @@ module Sidekiq # The job should be considered immutable but may be # removed from the queue via JobRecord#delete. class JobRecord - # the parsed Hash of job data # @!attribute [r] Item attr_reader :item @@ -667,7 +665,6 @@ module Sidekiq # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data, # e.g. Batches. class JobSet < SortedSet - # Add a job with the associated timestamp to this set. # @param timestamp [Time] the score for the job # @param job [Hash] the job data