144 lines
4.4 KiB
Ruby
144 lines
4.4 KiB
Ruby
module Gitlab
|
|
module Git
|
|
module Storage
|
|
class CircuitBreaker
|
|
FailureInfo = Struct.new(:last_failure, :failure_count)
|
|
|
|
attr_reader :storage,
|
|
:hostname,
|
|
:storage_path,
|
|
:failure_count_threshold,
|
|
:failure_wait_time,
|
|
:failure_reset_time,
|
|
:storage_timeout
|
|
|
|
delegate :last_failure, :failure_count, to: :failure_info
|
|
|
|
def self.reset_all!
|
|
pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*"
|
|
|
|
Gitlab::Git::Storage.redis.with do |redis|
|
|
all_storage_keys = redis.keys(pattern)
|
|
redis.del(*all_storage_keys) unless all_storage_keys.empty?
|
|
end
|
|
|
|
RequestStore.delete(:circuitbreaker_cache)
|
|
end
|
|
|
|
def self.for_storage(storage)
|
|
cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
|
|
Hash.new do |hash, storage_name|
|
|
hash[storage_name] = new(storage_name)
|
|
end
|
|
end
|
|
|
|
cached_circuitbreakers[storage]
|
|
end
|
|
|
|
def initialize(storage, hostname = Gitlab::Environment.hostname)
|
|
@storage = storage
|
|
@hostname = hostname
|
|
|
|
config = Gitlab.config.repositories.storages[@storage]
|
|
@storage_path = config['path']
|
|
@failure_count_threshold = config['failure_count_threshold']
|
|
@failure_wait_time = config['failure_wait_time']
|
|
@failure_reset_time = config['failure_reset_time']
|
|
@storage_timeout = config['storage_timeout']
|
|
end
|
|
|
|
def perform
|
|
return yield unless Feature.enabled?('git_storage_circuit_breaker')
|
|
|
|
check_storage_accessible!
|
|
|
|
yield
|
|
end
|
|
|
|
def circuit_broken?
|
|
return false if no_failures?
|
|
|
|
recent_failure = last_failure > failure_wait_time.seconds.ago
|
|
too_many_failures = failure_count > failure_count_threshold
|
|
|
|
recent_failure || too_many_failures
|
|
end
|
|
|
|
# Memoizing the `storage_available` call means we only do it once per
|
|
# request when the storage is available.
|
|
#
|
|
# When the storage appears not available, and the memoized value is `false`
|
|
# we might want to try again.
|
|
def storage_available?
|
|
return @storage_available if @storage_available
|
|
|
|
if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
|
|
.storage_available?(storage_path, storage_timeout)
|
|
track_storage_accessible
|
|
else
|
|
track_storage_inaccessible
|
|
end
|
|
|
|
@storage_available
|
|
end
|
|
|
|
def check_storage_accessible!
|
|
if circuit_broken?
|
|
raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time)
|
|
end
|
|
|
|
unless storage_available?
|
|
raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time)
|
|
end
|
|
end
|
|
|
|
def no_failures?
|
|
last_failure.blank? && failure_count == 0
|
|
end
|
|
|
|
def track_storage_inaccessible
|
|
@failure_info = FailureInfo.new(Time.now, failure_count + 1)
|
|
|
|
Gitlab::Git::Storage.redis.with do |redis|
|
|
redis.pipelined do
|
|
redis.hset(cache_key, :last_failure, last_failure.to_i)
|
|
redis.hincrby(cache_key, :failure_count, 1)
|
|
redis.expire(cache_key, failure_reset_time)
|
|
end
|
|
end
|
|
end
|
|
|
|
def track_storage_accessible
|
|
return if no_failures?
|
|
|
|
@failure_info = FailureInfo.new(nil, 0)
|
|
|
|
Gitlab::Git::Storage.redis.with do |redis|
|
|
redis.pipelined do
|
|
redis.hset(cache_key, :last_failure, nil)
|
|
redis.hset(cache_key, :failure_count, 0)
|
|
end
|
|
end
|
|
end
|
|
|
|
def failure_info
|
|
@failure_info ||= get_failure_info
|
|
end
|
|
|
|
def get_failure_info
|
|
last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
|
|
redis.hmget(cache_key, :last_failure, :failure_count)
|
|
end
|
|
|
|
last_failure = Time.at(last_failure.to_i) if last_failure.present?
|
|
|
|
FailureInfo.new(last_failure, failure_count.to_i)
|
|
end
|
|
|
|
def cache_key
|
|
@cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|