From 31136f16549b687fe005b27d9c4d64b8b7ba42eb Mon Sep 17 00:00:00 2001 From: Michael Sawyer Date: Thu, 20 Apr 2017 13:38:23 -0700 Subject: [PATCH 1/3] ET-2352 Implements exponential backoff for compute requests. No new test failures. --- lib/fog/aws/compute.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/fog/aws/compute.rb b/lib/fog/aws/compute.rb index eef90a04d..75c6e8690 100644 --- a/lib/fog/aws/compute.rb +++ b/lib/fog/aws/compute.rb @@ -537,7 +537,10 @@ module Fog end end - def _request(body, headers, idempotent, parser) + def _request(body, headers, idempotent, parser, retries = 0) + + max_retries = 10 + @connection.request({ :body => body, :expects => 200, @@ -552,6 +555,13 @@ module Fog raise case match[:code] when 'NotFound', 'Unknown' Fog::Compute::AWS::NotFound.slurp(error, match[:message]) + when 'RequestLimitExceeded' + if retries < max_retries + sleep (2.0 ** (1.0 + retries) * 100) / 1000.0 + _request(body, headers, idempotent, parser, retries + 1) + else + Fog::Compute::AWS::Error.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}") + end else Fog::Compute::AWS::Error.slurp(error, "#{match[:code]} => #{match[:message]}") end From 8ffa53309060c3ddbf4ea986ff09518dd343d542 Mon Sep 17 00:00:00 2001 From: Michael Sawyer Date: Thu, 20 Apr 2017 15:36:39 -0700 Subject: [PATCH 2/3] ET-2352 Throw RequestLimitExceeded error if out of retries. --- lib/fog/aws/compute.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/fog/aws/compute.rb b/lib/fog/aws/compute.rb index 75c6e8690..4e31f091a 100644 --- a/lib/fog/aws/compute.rb +++ b/lib/fog/aws/compute.rb @@ -3,6 +3,8 @@ module Fog class AWS < Fog::Service extend Fog::AWS::CredentialFetcher::ServiceMethods + class RequestLimitExceeded < Fog::Errors::Error; end + requires :aws_access_key_id, :aws_secret_access_key recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version @@ -560,7 +562,7 @@ module Fog sleep (2.0 ** (1.0 + retries) * 100) / 1000.0 _request(body, headers, idempotent, parser, retries + 1) else - Fog::Compute::AWS::Error.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}") + Fog::Compute::AWS::RequestLimitExceeded.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}") end else Fog::Compute::AWS::Error.slurp(error, "#{match[:code]} => #{match[:message]}") From 5f9439ca169e5ee5eec11632f95251dc09b3d93a Mon Sep 17 00:00:00 2001 From: Michael Sawyer Date: Tue, 25 Apr 2017 18:12:18 -0700 Subject: [PATCH 3/3] ET-2352 Use retries instead of recursion. --- lib/fog/aws/compute.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/fog/aws/compute.rb b/lib/fog/aws/compute.rb index 4e31f091a..cdc4ac065 100644 --- a/lib/fog/aws/compute.rb +++ b/lib/fog/aws/compute.rb @@ -543,6 +543,7 @@ module Fog max_retries = 10 + begin @connection.request({ :body => body, :expects => 200, @@ -551,22 +552,24 @@ module Fog :method => 'POST', :parser => parser }) - rescue Excon::Errors::HTTPStatusError => error - match = Fog::AWS::Errors.match_error(error) - raise if match.empty? - raise case match[:code] + rescue Excon::Errors::HTTPStatusError => error + match = Fog::AWS::Errors.match_error(error) + raise if match.empty? + raise case match[:code] when 'NotFound', 'Unknown' Fog::Compute::AWS::NotFound.slurp(error, match[:message]) when 'RequestLimitExceeded' if retries < max_retries sleep (2.0 ** (1.0 + retries) * 100) / 1000.0 - _request(body, headers, idempotent, parser, retries + 1) + retries += 1 + retry else Fog::Compute::AWS::RequestLimitExceeded.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}") end else Fog::Compute::AWS::Error.slurp(error, "#{match[:code]} => #{match[:message]}") end + end end end end