gitlab-org--gitlab-foss/lib/gitlab/gitaly_client.rb
2017-10-03 08:03:19 +00:00

283 lines
8.7 KiB
Ruby

require 'base64'
require 'gitaly'
module Gitlab
module GitalyClient
module MigrationStatus
DISABLED = 1
OPT_IN = 2
OPT_OUT = 3
end
class TooManyInvocationsError < StandardError
attr_reader :call_site, :invocation_count, :max_call_stack
def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
@call_site = call_site
@invocation_count = invocation_count
@max_call_stack = max_call_stack
stacks = most_invoked_stack.join('\n') if most_invoked_stack
msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
super(msg)
end
end
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
MAXIMUM_GITALY_CALLS = 30
MUTEX = Mutex.new
private_constant :MUTEX
def self.stub(name, storage)
MUTEX.synchronize do
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
addr = address(storage)
addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
klass.new(addr, :this_channel_is_insecure)
end
end
end
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
end
end
def self.address(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
address = params['gitaly_address']
unless address.present?
raise "storage #{storage.inspect} is missing a gitaly_address"
end
unless URI(address).scheme.in?(%w(tcp unix))
raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'"
end
address
end
# All Gitaly RPC call sites should use GitalyClient.call. This method
# makes sure that per-request authentication headers are set.
def self.call(storage, service, rpc, request)
enforce_gitaly_request_limits(:call)
metadata = request_metadata(storage)
metadata = yield(metadata) if block_given?
stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend
end
def self.request_metadata(storage)
encoded_token = Base64.strict_encode64(token(storage).to_s)
{ metadata: { 'authorization' => "Bearer #{encoded_token}" } }
end
def self.token(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end
# Evaluates whether a feature toggle is on or off
def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN)
# Disabled features are always off!
return false if status == MigrationStatus::DISABLED
feature = Feature.get("gitaly_#{feature_name}")
# If the feature has been set, always evaluate
if Feature.persisted?(feature)
if feature.percentage_of_time_value > 0
# Probabilistically enable this feature
return Random.rand() * 100 < feature.percentage_of_time_value
end
return feature.enabled?
end
# If the feature has not been set, the default depends
# on it's status
case status
when MigrationStatus::OPT_OUT
true
when MigrationStatus::OPT_IN
opt_into_all_features?
else
false
end
end
# opt_into_all_features? returns true when the current environment
# is one in which we opt into features automatically
def self.opt_into_all_features?
Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1"
end
private_class_method :opt_into_all_features?
def self.migrate(feature, status: MigrationStatus::OPT_IN)
# Enforce limits at both the `migrate` and `call` sites to ensure that
# problems are not hidden by a feature being disabled
enforce_gitaly_request_limits(:migrate)
is_enabled = feature_enabled?(feature, status: status)
metric_name = feature.to_s
metric_name += "_gitaly" if is_enabled
Gitlab::Metrics.measure(metric_name) do
# Some migrate calls wrap other migrate calls
allow_n_plus_1_calls do
yield is_enabled
end
end
end
# Ensures that Gitaly is not being abuse through n+1 misuse etc
def self.enforce_gitaly_request_limits(call_site)
# Only count limits in request-response environments (not sidekiq for example)
return unless RequestStore.active?
# This is this actual number of times this call was made. Used for information purposes only
actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
# Do no enforce limits in production
return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"]
# Check if this call is nested within a allow_n_plus_1_calls
# block and skip check if it is
return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
# This is the count of calls outside of a `allow_n_plus_1_calls` block
# It is used for enforcement but not statistics
permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
count_stack
return if permitted_call_count <= MAXIMUM_GITALY_CALLS
raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end
def self.allow_n_plus_1_calls
return yield unless RequestStore.active?
begin
increment_call_count(:gitaly_call_count_exception_block_depth)
yield
ensure
decrement_call_count(:gitaly_call_count_exception_block_depth)
end
end
def self.get_call_count(key)
RequestStore.store[key] || 0
end
private_class_method :get_call_count
def self.increment_call_count(key)
RequestStore.store[key] ||= 0
RequestStore.store[key] += 1
end
private_class_method :increment_call_count
def self.decrement_call_count(key)
RequestStore.store[key] -= 1
end
private_class_method :decrement_call_count
# Returns an estimate of the number of Gitaly calls made for this
# request
def self.get_request_count
return 0 unless RequestStore.active?
gitaly_migrate_count = get_call_count("gitaly_migrate_actual")
gitaly_call_count = get_call_count("gitaly_call_actual")
# Using the maximum of migrate and call_count will provide an
# indicator of how many Gitaly calls will be made, even
# before a feature is enabled. This provides us with a single
# metric, but not an exact number, but this tradeoff is acceptable
if gitaly_migrate_count > gitaly_call_count
gitaly_migrate_count
else
gitaly_call_count
end
end
def self.reset_counts
return unless RequestStore.active?
%w[migrate call].each do |call_site|
RequestStore.store["gitaly_#{call_site}_actual"] = 0
RequestStore.store["gitaly_#{call_site}_permitted"] = 0
end
end
def self.expected_server_version
path = Rails.root.join(SERVER_VERSION_FILE)
path.read.chomp
end
def self.timestamp(t)
Google::Protobuf::Timestamp.new(seconds: t.to_i)
end
def self.encode(s)
return "" if s.nil?
s.dup.force_encoding(Encoding::ASCII_8BIT)
end
def self.encode_repeated(a)
Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } )
end
# Count a stack. Used for n+1 detection
def self.count_stack
return unless RequestStore.active?
stack_string = caller.drop(1).join("\n")
RequestStore.store[:stack_counter] ||= Hash.new
count = RequestStore.store[:stack_counter][stack_string] || 0
RequestStore.store[:stack_counter][stack_string] = count + 1
end
private_class_method :count_stack
# Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
def self.max_call_count
return 0 unless RequestStore.active?
stack_counter = RequestStore.store[:stack_counter]
return 0 unless stack_counter
stack_counter.values.max
end
private_class_method :max_call_count
# Returns the stacks that calls Gitaly the most times. Used for n+1 detection
def self.max_stacks
return nil unless RequestStore.active?
stack_counter = RequestStore.store[:stack_counter]
return nil unless stack_counter
max = max_call_count
return nil if max.zero?
stack_counter.select { |_, v| v == max }.keys
end
private_class_method :max_stacks
end
end