diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 2ee8d78b2f0..96f436fa7c3 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -131,6 +131,15 @@ mount is reset. **Seconds to wait for a storage access attempt:** The time in seconds GitLab will try to access storage. After this time a timeout error will be raised. +To enable the circuitbreaker for repository storage you can flip the feature flag from a rails console: + +``` +Feature.enable('git_storage_circuit_breaker') +``` + +Alternatively it can be enabled by setting `true` in the `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. +This approach would be used when enabling the circuit breaker on a single host. + When storage failures occur, this will be visible in the admin interface like this: ![failing storage](img/failing_storage.png) diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index e53171fa3c5..be7598ef011 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -54,7 +54,7 @@ module Gitlab end def perform - return yield unless Feature.enabled?('git_storage_circuit_breaker') + return yield unless enabled? check_storage_accessible! @@ -78,6 +78,15 @@ module Gitlab private + # The circuitbreaker can be enabled for the entire fleet using a Feature + # flag. + # + # Enabling it for a single host can be done setting the + # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. + def enabled? + ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') + end + def failure_info @failure_info ||= get_failure_info end diff --git a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb index c723ab18e67..72dabca793a 100644 --- a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb +++ b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb @@ -200,10 +200,21 @@ describe Gitlab::Git::Storage::CircuitBreaker, clean_gitlab_redis_shared_state: end context 'with the feature disabled' do - it 'returns the block without checking accessibility' do + before do stub_feature_flags(git_storage_circuit_breaker: false) + end - expect(circuit_breaker).not_to receive(:circuit_broken?) + it 'returns the block without checking accessibility' do + expect(circuit_breaker).not_to receive(:check_storage_accessible!) + + result = circuit_breaker.perform { 'hello' } + + expect(result).to eq('hello') + end + + it 'allows enabling the feature using an ENV var' do + stub_env('GIT_STORAGE_CIRCUIT_BREAKER', 'true') + expect(circuit_breaker).to receive(:check_storage_accessible!) result = circuit_breaker.perform { 'hello' }