1
0
Fork 0
mirror of https://github.com/mperham/sidekiq.git synced 2022-11-09 13:52:34 -05:00
mperham--sidekiq/test/test_redis_connection.rb

259 lines
7.5 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require_relative 'helper'
require 'sidekiq/cli'
2019-03-01 16:25:56 -05:00
describe Sidekiq::RedisConnection do
describe "create" do
before do
Sidekiq.options = Sidekiq::DEFAULTS.dup
@old = ENV['REDIS_URL']
ENV['REDIS_URL'] = 'redis://localhost/15'
end
after do
ENV['REDIS_URL'] = @old
end
# To support both redis-rb 3.3.x #client and 4.0.x #_client
def client_for(redis)
if redis.respond_to?(:_client)
redis._client
else
redis.client
end
end
2013-01-19 19:03:12 -05:00
it "creates a pooled redis connection" do
pool = Sidekiq::RedisConnection.create
assert_equal Redis, pool.checkout.class
assert_equal "Sidekiq-server-PID-#{$$}", pool.checkout.connection.fetch(:id)
2013-01-19 19:03:12 -05:00
end
Use `RAILS_MAX_THREADS` for client pool size This is a follow up to #2985 (52828e4) adding similar support for the client connection pool. For Rails servers, Sidekiq is not loaded from the CLI so the prior change to support setting the concurrency via `RAILS_MAX_THREADS` is not applied to the web server process. This means for Rails servers which do not configure a custom size through an initializer they will run with the default connection pool size of 5. When the Rails server runs the initial Redis connection may be made through `Sidekiq::Client` (e.g. from [`ActiveJob::QueueAdapters::SidekiqAdapter`](https://github.com/rails/rails/blob/v5.1.5/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb#L20)). This causes the `redis_pool` to be initialized without any options, setting the pool size to the default of 5. .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq.rb:125:in `redis_pool' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:42:in `initialize' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `new' .gem/ruby/2.5.0/gems/sidekiq-5.1.1/lib/sidekiq/client.rb:131:in `push' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/queue_adapters/sidekiq_adapter.rb:20:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:51:in `block in enqueue' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:108:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:15:in `block (3 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `block in tag_logger' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `block in tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:26:in `tagged' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/tagged_logging.rb:69:in `tagged' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:44:in `tag_logger' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/logging.rb:14:in `block (2 levels) in <module:Logging>' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `instance_exec' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:117:in `block in run_callbacks' .gem/ruby/2.5.0/gems/activesupport-5.1.5/lib/active_support/callbacks.rb:135:in `run_callbacks' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:47:in `enqueue' .gem/ruby/2.5.0/gems/activejob-5.1.5/lib/active_job/enqueuing.rb:18:in `perform_later' For the majority of cases, a client pool size of 5 is sufficient. However, servers which utilize a high number of threads, with large job payloads, and which may experience some network latency issues can see `Timeout::Error` crashes. This may be further exacerbated by the ~2-20x performance decrease through `ActiveJob` (#3782). Rails addresses this general type of connection issue for the main database by suggesting that the DB pool size match the number of threads running. This change applies that logic to the default client pool size by leveraging the same environment setting; this way there's a connection available per thread. This may also have the side effect of a slight performance boost, as there is less of a chance that threads will be blocked waiting on connections. The trade-off is that there may be a memory profile increase to handle the additional Redis connections in the pool; note the pool only creates new connections as necessary to handle the requests. Resolves #3806
2018-03-27 16:53:59 -04:00
# Readers for these ivars should be available in the next release of
# `connection_pool`, until then we need to reach into the internal state to
# verify the setting.
describe "size" do
def client_connection(*args)
Sidekiq.stub(:server?, nil) do
Sidekiq::RedisConnection.create(*args)
end
end
def server_connection(*args)
Sidekiq.stub(:server?, "constant") do
Sidekiq::RedisConnection.create(*args)
end
end
it "uses the specified custom pool size" do
pool = client_connection(size: 42)
assert_equal 42, pool.instance_eval{ @size }
assert_equal 42, pool.instance_eval{ @available.length }
pool = server_connection(size: 42)
assert_equal 42, pool.instance_eval{ @size }
assert_equal 42, pool.instance_eval{ @available.length }
end
it "defaults server pool sizes based on concurrency with padding" do
_expected_padding = 5
Sidekiq.options[:concurrency] = 6
pool = server_connection
assert_equal 11, pool.instance_eval{ @size }
assert_equal 11, pool.instance_eval{ @available.length }
end
it "defaults client pool sizes to 5" do
pool = client_connection
assert_equal 5, pool.instance_eval{ @size }
assert_equal 5, pool.instance_eval{ @available.length }
end
it "changes client pool sizes with ENV" do
begin
ENV['RAILS_MAX_THREADS'] = '9'
pool = client_connection
assert_equal 9, pool.instance_eval{ @size }
assert_equal 9, pool.instance_eval{ @available.length }
ensure
ENV.delete('RAILS_MAX_THREADS')
end
end
end
it "disables client setname with nil id" do
pool = Sidekiq::RedisConnection.create(:id => nil)
assert_equal Redis, pool.checkout.class
assert_equal "redis://localhost:6379/15", pool.checkout.connection.fetch(:id)
end
describe "network_timeout" do
it "sets a custom network_timeout if specified" do
pool = Sidekiq::RedisConnection.create(:network_timeout => 8)
redis = pool.checkout
assert_equal 8, client_for(redis).timeout
end
it "uses the default network_timeout if none specified" do
pool = Sidekiq::RedisConnection.create
redis = pool.checkout
assert_equal 5, client_for(redis).timeout
end
end
2013-01-19 19:03:12 -05:00
describe "namespace" do
it "uses a given :namespace set by a symbol key" do
2013-01-19 19:03:12 -05:00
pool = Sidekiq::RedisConnection.create(:namespace => "xxx")
assert_equal "xxx", pool.checkout.namespace
end
it "uses a given :namespace set by a string key" do
pool = Sidekiq::RedisConnection.create("namespace" => "xxx")
assert_equal "xxx", pool.checkout.namespace
end
2013-01-19 19:03:12 -05:00
it "uses given :namespace over :namespace from Sidekiq.options" do
Sidekiq.options[:namespace] = "xxx"
2013-01-19 19:03:12 -05:00
pool = Sidekiq::RedisConnection.create(:namespace => "yyy")
assert_equal "yyy", pool.checkout.namespace
end
end
2013-10-21 15:53:34 -04:00
describe "socket path" do
it "uses a given :path" do
pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock")
assert_equal "unix", client_for(pool.checkout).scheme
assert_equal "/var/run/redis.sock", pool.checkout.connection.fetch(:location)
assert_equal 15, pool.checkout.connection.fetch(:db)
2013-10-21 15:53:34 -04:00
end
it "uses a given :path and :db" do
pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
assert_equal "unix", client_for(pool.checkout).scheme
assert_equal "/var/run/redis.sock", pool.checkout.connection.fetch(:location)
assert_equal 8, pool.checkout.connection.fetch(:db)
2013-10-21 15:53:34 -04:00
end
end
describe "pool_timeout" do
it "uses a given :timeout over the default of 1" do
pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)
assert_equal 5, pool.instance_eval{ @timeout }
end
it "uses the default timeout of 1 if no override" do
pool = Sidekiq::RedisConnection.create
assert_equal 1, pool.instance_eval{ @timeout }
end
end
describe "driver" do
it "uses redis' ruby driver" do
pool = Sidekiq::RedisConnection.create
redis = pool.checkout
assert_equal Redis::Connection::Ruby, redis.instance_variable_get(:@client).driver
end
it "uses redis' default driver if there are many available" do
begin
redis_driver = Object.new
Redis::Connection.drivers << redis_driver
pool = Sidekiq::RedisConnection.create
redis = pool.checkout
assert_equal redis_driver, redis.instance_variable_get(:@client).driver
ensure
Redis::Connection.drivers.pop
end
end
it "uses a given :driver" do
redis_driver = Object.new
pool = Sidekiq::RedisConnection.create(:driver => redis_driver)
redis = pool.checkout
assert_equal redis_driver, redis.instance_variable_get(:@client).driver
end
end
2013-01-19 19:03:12 -05:00
end
describe ".determine_redis_provider" do
before do
@old_env = ENV.to_hash
end
after do
ENV.update(@old_env)
end
def with_env_var(var, uri, skip_provider=false)
vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
vars.each do |v|
next if skip_provider
ENV[v] = nil
end
ENV[var] = uri
assert_equal uri, Sidekiq::RedisConnection.__send__(:determine_redis_provider)
ENV[var] = nil
end
describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
it "sets connection URI to the provider" do
uri = 'redis://sidekiq-redis-provider:6379/0'
provider = 'SIDEKIQ_REDIS_PROVIDER'
ENV['REDIS_PROVIDER'] = provider
ENV[provider] = uri
ENV['REDISTOGO_URL'] = 'redis://redis-to-go:6379/0'
with_env_var provider, uri, true
ENV[provider] = nil
end
end
describe "with REDIS_PROVIDER set" do
2018-09-17 14:51:27 -04:00
it "rejects URLs in REDIS_PROVIDER" do
uri = 'redis://sidekiq-redis-provider:6379/0'
ENV['REDIS_PROVIDER'] = uri
2018-09-17 14:58:34 -04:00
assert_raises RuntimeError do
2018-09-17 14:51:27 -04:00
Sidekiq::RedisConnection.__send__(:determine_redis_provider)
2018-09-17 14:58:34 -04:00
end
2018-09-17 14:51:27 -04:00
ENV['REDIS_PROVIDER'] = nil
end
it "sets connection URI to the provider" do
uri = 'redis://sidekiq-redis-provider:6379/0'
provider = 'SIDEKIQ_REDIS_PROVIDER'
ENV['REDIS_PROVIDER'] = provider
ENV[provider] = uri
with_env_var provider, uri, true
ENV[provider] = nil
end
end
describe "with REDIS_URL set" do
it "sets connection URI to custom uri" do
with_env_var 'REDIS_URL', 'redis://redis-uri:6379/0'
end
end
end
end