mperham--sidekiq/test/redis_connection_test.rb

251 lines
7.0 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require_relative "helper"
require "sidekiq/redis_connection"
require "sidekiq/capsule"
2019-03-01 21:25:56 +00:00
describe Sidekiq::RedisConnection do
describe "create" do
before do
@config = reset!
2022-09-02 14:37:29 +00:00
@config.default_capsule.concurrency = 12
end
def client_for(redis)
2022-05-12 15:41:40 +00:00
redis._client
end
def config_for(redis)
2022-05-12 15:41:40 +00:00
redis._config
end
2022-05-11 19:40:19 +00:00
def client_class
Sidekiq::RedisClientAdapter::CompatClient
end
2013-01-20 00:03:12 +00:00
it "creates a pooled redis connection" do
pool = Sidekiq::RedisConnection.create
assert_equal 5, pool.size
assert_equal client_class, pool.checkout.class
2013-01-20 00:03:12 +00:00
end
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
# Readers for these ivars should be available in the next release of
# `connection_pool`, until then we need to reach into the internal state to
# verify the setting.
describe "size" do
def client_connection(args = {})
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
Sidekiq.stub(:server?, nil) do
@config.redis = args
@config.redis_pool
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
end
end
def server_connection(args = {})
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
Sidekiq.stub(:server?, "constant") do
@config.redis = args
@config.redis_pool
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
end
end
it "sizes default pool" do
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
pool = server_connection
assert_equal 5, pool.size
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
end
it "defaults client pool sizes to 5" do
pool = client_connection
assert_equal 5, pool.size
end
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
it "sizes capsule pools based on concurrency" do
2022-09-02 14:37:29 +00:00
assert_equal 12, @config.default_capsule.redis_pool.size
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
end
2022-09-15 20:07:43 +00:00
it "does not change client pool sizes with ENV" do
# Only Sidekiq::CLI looks at ENV
ENV["RAILS_MAX_THREADS"] = "9"
pool = client_connection
2022-09-15 20:07:43 +00:00
assert_equal 5, pool.size
ensure
ENV.delete("RAILS_MAX_THREADS")
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 20:53:59 +00:00
end
end
it "disables client setname with nil id" do
pool = Sidekiq::RedisConnection.create(id: nil)
assert_equal client_class, pool.checkout.class
client = client_for(pool.checkout)
2022-05-11 19:40:19 +00:00
assert_nil client.id
end
describe "network_timeout" do
it "sets a custom network_timeout if specified" do
pool = Sidekiq::RedisConnection.create(network_timeout: 8)
redis = pool.checkout
assert_equal 8, client_for(redis).read_timeout
end
it "uses the default network_timeout if none specified" do
pool = Sidekiq::RedisConnection.create
redis = pool.checkout
2022-05-11 19:40:19 +00:00
assert_equal 1.0, client_for(redis).read_timeout
end
end
2013-01-20 00:03:12 +00:00
describe "namespace" do
it "isn't supported" do
error = assert_raises ArgumentError do
Sidekiq::RedisConnection.create(namespace: "xxx")
end
assert_includes error.message, "Your Redis configuration uses the namespace 'xxx' but this feature isn't supported by redis-client"
2013-01-20 00:03:12 +00:00
end
end
2013-10-21 19:53:34 +00:00
describe "socket path" do
it "uses a given :path" do
2022-05-12 15:41:40 +00:00
pool = Sidekiq::RedisConnection.create(path: "/tmp/redis.sock")
config = config_for(pool.checkout)
2022-05-12 15:41:40 +00:00
assert_equal "/tmp/redis.sock", config.path
2013-10-21 19:53:34 +00:00
end
it "uses a given :path and :db" do
2022-05-12 15:41:40 +00:00
pool = Sidekiq::RedisConnection.create(path: "/tmp/redis.sock", db: 8)
config = config_for(pool.checkout)
2022-05-12 15:41:40 +00:00
assert_equal "/tmp/redis.sock", config.path
assert_equal 8, config.db
2013-10-21 19:53:34 +00:00
end
end
describe "pool_timeout" do
it "uses a given :timeout over the default of 1" do
pool = Sidekiq::RedisConnection.create(pool_timeout: 5)
assert_equal 5, pool.instance_eval { @timeout }
end
it "uses the default timeout of 1 if no override" do
pool = Sidekiq::RedisConnection.create
assert_equal 1, pool.instance_eval { @timeout }
end
end
describe "driver" do
2022-05-11 19:40:19 +00:00
it "uses ruby driver by default" do
pool = Sidekiq::RedisConnection.create
config = config_for(pool.checkout)
2022-05-11 19:40:19 +00:00
assert_equal RedisClient::RubyConnection, config.driver
end
end
describe "logging redis options" do
it "redacts credentials" do
options = {
role: "master",
master_name: "mymaster",
sentinels: [
{host: "host1", port: 26379, password: "secret"},
{host: "host2", port: 26379, password: "secret"},
{host: "host3", port: 26379, password: "secret"}
],
password: "secret"
}
output = capture_logging(@config) do |logger|
Sidekiq::RedisConnection.create(options.merge(logger: logger))
end
refute_includes(options.inspect, "REDACTED")
assert_includes(output, ':host=>"host1", :port=>26379, :password=>"REDACTED"')
assert_includes(output, ':host=>"host2", :port=>26379, :password=>"REDACTED"')
assert_includes(output, ':host=>"host3", :port=>26379, :password=>"REDACTED"')
2020-03-23 17:40:44 +00:00
assert_includes(output, ':password=>"REDACTED"')
end
it "prunes SSL parameters from the logging" do
output = capture_logging(@config) do |logger|
options = {
ssl_params: {
cert_store: OpenSSL::X509::Store.new
},
logger: logger
}
Sidekiq::RedisConnection.create(options)
assert_includes(options.inspect, "ssl_params")
end
refute_includes(output, "ssl_params")
end
end
2013-01-20 00:03:12 +00:00
end
describe ".determine_redis_provider" do
before do
@old_env = ENV.to_hash
end
after do
ENV.update(@old_env)
end
def with_env_var(var, uri, skip_provider = false)
vars = ["REDISTOGO_URL", "REDIS_PROVIDER", "REDIS_URL"] - [var]
vars.each do |v|
next if skip_provider
ENV[v] = nil
end
ENV[var] = uri
assert_equal uri, Sidekiq::RedisConnection.__send__(:determine_redis_provider)
ENV[var] = nil
end
describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
it "sets connection URI to the provider" do
uri = "redis://sidekiq-redis-provider:6379/0"
provider = "SIDEKIQ_REDIS_PROVIDER"
ENV["REDIS_PROVIDER"] = provider
ENV[provider] = uri
ENV["REDISTOGO_URL"] = "redis://redis-to-go:6379/0"
with_env_var provider, uri, true
ENV[provider] = nil
end
end
describe "with REDIS_PROVIDER set" do
2018-09-17 18:51:27 +00:00
it "rejects URLs in REDIS_PROVIDER" do
uri = "redis://sidekiq-redis-provider:6379/0"
2018-09-17 18:51:27 +00:00
ENV["REDIS_PROVIDER"] = uri
2018-09-17 18:51:27 +00:00
2018-09-17 18:58:34 +00:00
assert_raises RuntimeError do
2018-09-17 18:51:27 +00:00
Sidekiq::RedisConnection.__send__(:determine_redis_provider)
2018-09-17 18:58:34 +00:00
end
2018-09-17 18:51:27 +00:00
ENV["REDIS_PROVIDER"] = nil
2018-09-17 18:51:27 +00:00
end
it "sets connection URI to the provider" do
uri = "redis://sidekiq-redis-provider:6379/0"
provider = "SIDEKIQ_REDIS_PROVIDER"
ENV["REDIS_PROVIDER"] = provider
ENV[provider] = uri
with_env_var provider, uri, true
ENV[provider] = nil
end
end
describe "with REDIS_URL set" do
it "sets connection URI to custom uri" do
with_env_var "REDIS_URL", "redis://redis-uri:6379/0"
end
end
end
end