From 4a8c2dd4192517fa030b6585ae82bf925c80516a Mon Sep 17 00:00:00 2001 From: Alexander Pakulov Date: Wed, 3 Jul 2019 16:35:59 -0700 Subject: [PATCH] [rubygems/rubygems] Extracting sign_s3_url & s3_source_auth into a separate S3URISigner class https://github.com/rubygems/rubygems/commit/c30d21ec7a --- lib/rubygems/remote_fetcher.rb | 116 ++-------------- lib/rubygems/s3_uri_signer.rb | 166 +++++++++++++++++++++++ test/rubygems/test_gem_remote_fetcher.rb | 15 +- 3 files changed, 187 insertions(+), 110 deletions(-) create mode 100644 lib/rubygems/s3_uri_signer.rb diff --git a/lib/rubygems/remote_fetcher.rb b/lib/rubygems/remote_fetcher.rb index 529fdab620..1e0c23dc33 100644 --- a/lib/rubygems/remote_fetcher.rb +++ b/lib/rubygems/remote_fetcher.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'rubygems' require 'rubygems/request' +require 'rubygems/request/connection_pools' +require 'rubygems/s3_uri_signer' require 'rubygems/uri_formatter' require 'rubygems/user_interaction' -require 'rubygems/request/connection_pools' require 'resolv' ## @@ -284,10 +285,19 @@ class Gem::RemoteFetcher end def fetch_s3(uri, mtime = nil, head = false) - public_uri = sign_s3_url(uri) + begin + public_uri = s3_uri_signer(uri).sign + rescue Gem::S3URISigner::ConfigurationError, Gem::S3URISigner::InstanceProfileError => e + raise FetchError.new(e.message, "s3://#{uri.host}") + end fetch_https public_uri, mtime, head end + # we have our own signing code here to avoid a dependency on the aws-sdk gem + def s3_uri_signer(uri) + Gem::S3URISigner.new(uri) + end + ## # Downloads +uri+ to +path+ if necessary. If no path is given, it just # passes the data. @@ -341,73 +351,8 @@ class Gem::RemoteFetcher @pools.each_value {|pool| pool.close_all} end - protected - - S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region - - # we have our own signing code here to avoid a dependency on the aws-sdk gem - def sign_s3_url(uri, expiration = nil) - require 'base64' - require 'digest' - require 'openssl' - - s3_config = s3_source_auth uri - expiration ||= 86400 - - current_time = Time.now.utc - date_time = current_time.strftime("%Y%m%dT%H%m%SZ") - date = date_time[0,8] - - credential_info = "#{date}/#{s3_config.region}/s3/aws4_request" - canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com" - - canonical_params = {} - canonical_params['X-Amz-Algorithm'] = "AWS4-HMAC-SHA256" - canonical_params['X-Amz-Credential'] = "#{s3_config.access_key_id}/#{credential_info}" - canonical_params['X-Amz-Date'] = date_time - canonical_params['X-Amz-Expires'] = expiration.to_s - canonical_params['X-Amz-SignedHeaders'] = "host" - canonical_params['X-Amz-Security-Token'] = s3_config.security_token if s3_config.security_token - - # Sorting is required to generate proper signature - query_params = canonical_params.sort.to_h.map do |key, value| - "#{base64_uri_escape(key)}=#{base64_uri_escape(value)}" - end.join('&') - - canonical_request = [ - 'GET', - uri.path, - query_params, - "host:#{canonical_host}", - '', # empty params - 'host', - 'UNSIGNED-PAYLOAD', - ].join("\n") - - string_to_sign = [ - "AWS4-HMAC-SHA256", - date_time, - credential_info, - Digest::SHA256.hexdigest(canonical_request) - ].join("\n") - - date_key = OpenSSL::HMAC.digest('sha256', "AWS4" + s3_config.secret_access_key, date) - date_region_key = OpenSSL::HMAC.digest('sha256', date_key, s3_config.region) - date_region_service_key = OpenSSL::HMAC.digest('sha256', date_region_key, "s3") - signing_key = OpenSSL::HMAC.digest('sha256', date_region_service_key, "aws4_request") - signature = OpenSSL::HMAC.hexdigest('sha256', signing_key, string_to_sign) - - URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}") - end - - BASE64_URI_TRANSLATE = { '+' => '%2B', '/' => '%2F', '=' => '%3D' }.freeze - private - def base64_uri_escape(str) - str.gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] } - end - def proxy_for(proxy, uri) Gem::Request.proxy_uri(proxy || Gem::Request.get_proxy_from_env(uri.scheme)) end @@ -418,41 +363,4 @@ class Gem::RemoteFetcher end end - def s3_source_auth(uri) - return S3Config.new(uri.user, uri.password, nil, 'us-east-1') if uri.user && uri.password - - s3_source = Gem.configuration[:s3_source] || Gem.configuration['s3_source'] - host = uri.host - raise FetchError.new("no s3_source key exists in .gemrc", "s3://#{host}") unless s3_source - - auth = s3_source[host] || s3_source[host.to_sym] - raise FetchError.new("no key for host #{host} in s3_source in .gemrc", "s3://#{host}") unless auth - - provider = auth[:provider] || auth['provider'] - case provider - when 'env' - id = ENV['AWS_ACCESS_KEY_ID'] - secret = ENV['AWS_SECRET_ACCESS_KEY'] - security_token = ENV['AWS_SESSION_TOKEN'] - when 'instance_profile' - require 'json' - credentials_response = fetch_http URI(EC2_METADATA_CREDENTIALS) - credentials = JSON.parse(credentials_response) - id = credentials['AccessKeyId'] - secret = credentials['SecretAccessKey'] - security_token = credentials['Token'] - else - id = auth[:id] || auth['id'] - secret = auth[:secret] || auth['secret'] - raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id && secret - - security_token = auth[:security_token] || auth['security_token'] - end - - region = auth[:region] || auth['region'] || 'us-east-1' - S3Config.new(id, secret, security_token, region) - end - - EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance" - end diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb new file mode 100644 index 0000000000..ff1f10e2bb --- /dev/null +++ b/lib/rubygems/s3_uri_signer.rb @@ -0,0 +1,166 @@ +require 'base64' +require 'digest' +require 'openssl' +require 'rubygems/request' +require 'rubygems/request/connection_pools' + +## +# S3URISigner implements AWS SigV4 for S3 Source to avoid a dependency on the aws-sdk-* gems +# More on AWS SigV4: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +class Gem::S3URISigner + + class ConfigurationError < Gem::Exception + + def initialize(message) + super message + end + + def to_s # :nodoc: + "#{super}" + end + + end + + class InstanceProfileError < Gem::Exception + + def initialize(message) + super message + end + + def to_s # :nodoc: + "#{super}" + end + + end + + attr_accessor :uri + + def initialize(uri) + @uri = uri + end + + ## + # Signs S3 URI using query-params according to the reference: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html + def sign(expiration = nil) + expiration ||= 86400 + + s3_config = fetch_s3_config + + current_time = Time.now.utc + date_time = current_time.strftime("%Y%m%dT%H%m%SZ") + date = date_time[0,8] + + credential_info = "#{date}/#{s3_config.region}/s3/aws4_request" + canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com" + + canonical_params = {} + canonical_params["X-Amz-Algorithm"] = "AWS4-HMAC-SHA256" + canonical_params["X-Amz-Credential"] = "#{s3_config.access_key_id}/#{credential_info}" + canonical_params["X-Amz-Date"] = date_time + canonical_params["X-Amz-Expires"] = expiration.to_s + canonical_params["X-Amz-SignedHeaders"] = "host" + canonical_params["X-Amz-Security-Token"] = s3_config.security_token if s3_config.security_token + + # Sorting is required to generate proper signature + query_params = canonical_params.sort.to_h.map do |key, value| + "#{base64_uri_escape(key)}=#{base64_uri_escape(value)}" + end.join("&") + + canonical_request = [ + "GET", + uri.path, + query_params, + "host:#{canonical_host}", + "", # empty params + "host", + "UNSIGNED-PAYLOAD", + ].join("\n") + + string_to_sign = [ + "AWS4-HMAC-SHA256", + date_time, + credential_info, + Digest::SHA256.hexdigest(canonical_request) + ].join("\n") + + date_key = OpenSSL::HMAC.digest("sha256", "AWS4" + s3_config.secret_access_key, date) + date_region_key = OpenSSL::HMAC.digest("sha256", date_key, s3_config.region) + date_region_service_key = OpenSSL::HMAC.digest("sha256", date_region_key, "s3") + signing_key = OpenSSL::HMAC.digest("sha256", date_region_service_key, "aws4_request") + signature = OpenSSL::HMAC.hexdigest("sha256", signing_key, string_to_sign) + + URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}") + end + + private + + S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region + + ## + # Extracts S3 configuration for S3 bucket + def fetch_s3_config + return S3Config.new(uri.user, uri.password, nil, "us-east-1") if uri.user && uri.password + + s3_source = Gem.configuration[:s3_source] || Gem.configuration["s3_source"] + host = uri.host + raise ConfigurationError.new("no s3_source key exists in .gemrc") unless s3_source + + auth = s3_source[host] || s3_source[host.to_sym] + raise ConfigurationError.new("no key for host #{host} in s3_source in .gemrc") unless auth + + provider = auth[:provider] || auth["provider"] + case provider + when "env" + id = ENV["AWS_ACCESS_KEY_ID"] + secret = ENV["AWS_SECRET_ACCESS_KEY"] + security_token = ENV["AWS_SESSION_TOKEN"] + when "instance_profile" + require "json" + credentials_response = ec2_metadata + credentials = JSON.parse(credentials_response) + id = credentials["AccessKeyId"] + secret = credentials["SecretAccessKey"] + security_token = credentials["Token"] + else + id = auth[:id] || auth["id"] + secret = auth[:secret] || auth["secret"] + raise ConfigurationError.new("s3_source for #{host} missing id or secret") unless id && secret + + security_token = auth[:security_token] || auth["security_token"] + end + + region = auth[:region] || auth["region"] || "us-east-1" + S3Config.new(id, secret, security_token, region) + end + + def base64_uri_escape(str) + str.gsub("\n", "").gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] } + end + + def ec2_metadata + require 'net/http' + + metadata_uri = URI(EC2_METADATA_CREDENTIALS) + + @request_pool ||= create_request_pool(metadata_uri) + request = Gem::Request.new(metadata_uri, Net::HTTP::Get, nil, @request_pool) + response = request.fetch + + case response + when Net::HTTPOK then + response.body + else + raise InstanceProfileError.new("Unable to fetch AWS credentials from #{metadata_uri}: #{response.message} #{response.code}") + end + end + + def create_request_pool(uri) + proxy_uri = Gem::Request.proxy_uri(Gem::Request.get_proxy_from_env(uri.scheme)) + certs = Gem::Request.get_cert_files + Gem::Request::ConnectionPools.new(proxy_uri, certs).pool_for(uri) + end + + BASE64_URI_TRANSLATE = { "+" => "%2B", "/" => "%2F", "=" => "%3D" }.freeze + EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance".freeze + +end diff --git a/test/rubygems/test_gem_remote_fetcher.rb b/test/rubygems/test_gem_remote_fetcher.rb index 5b2ea7d257..66e9bfca22 100644 --- a/test/rubygems/test_gem_remote_fetcher.rb +++ b/test/rubygems/test_gem_remote_fetcher.rb @@ -659,15 +659,18 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg== def fetcher.request(uri, request_class, last_modified = nil) $fetched_uri = uri res = Net::HTTPOK.new nil, 200, nil - case uri.to_s - when /^http:\/\/169\.254\.169\.254.*/ - def res.body() $instance_profile end - else - def res.body() 'success' end - end + def res.body() 'success' end res end + def fetcher.s3_uri_signer(uri) + s3_uri_signer = Gem::S3URISigner.new(uri) + def s3_uri_signer.ec2_metadata + $instance_profile + end + s3_uri_signer + end + data = fetcher.fetch_s3 URI.parse(url) assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T050641Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s