mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
[rubygems/rubygems] Extracting sign_s3_url & s3_source_auth into a separate S3URISigner class
https://github.com/rubygems/rubygems/commit/c30d21ec7a
This commit is contained in:
Alexander Pakulov
Hiroshi SHIBATA
parent
1dfe132aa5
commit
4a8c2dd419
3 changed files with 187 additions and 110 deletions
|
|
@ -1,9 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
require 'rubygems'
|
||||
require 'rubygems/request'
|
||||
require 'rubygems/request/connection_pools'
|
||||
require 'rubygems/s3_uri_signer'
|
||||
require 'rubygems/uri_formatter'
|
||||
require 'rubygems/user_interaction'
|
||||
require 'rubygems/request/connection_pools'
|
||||
require 'resolv'
|
||||
|
||||
##
|
||||
|
|
@ -284,10 +285,19 @@ class Gem::RemoteFetcher
|
|||
end
|
||||
|
||||
def fetch_s3(uri, mtime = nil, head = false)
|
||||
public_uri = sign_s3_url(uri)
|
||||
begin
|
||||
public_uri = s3_uri_signer(uri).sign
|
||||
rescue Gem::S3URISigner::ConfigurationError, Gem::S3URISigner::InstanceProfileError => e
|
||||
raise FetchError.new(e.message, "s3://#{uri.host}")
|
||||
end
|
||||
fetch_https public_uri, mtime, head
|
||||
end
|
||||
|
||||
# we have our own signing code here to avoid a dependency on the aws-sdk gem
|
||||
def s3_uri_signer(uri)
|
||||
Gem::S3URISigner.new(uri)
|
||||
end
|
||||
|
||||
##
|
||||
# Downloads +uri+ to +path+ if necessary. If no path is given, it just
|
||||
# passes the data.
|
||||
|
|
@ -341,73 +351,8 @@ class Gem::RemoteFetcher
|
|||
@pools.each_value {|pool| pool.close_all}
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region
|
||||
|
||||
# we have our own signing code here to avoid a dependency on the aws-sdk gem
|
||||
def sign_s3_url(uri, expiration = nil)
|
||||
require 'base64'
|
||||
require 'digest'
|
||||
require 'openssl'
|
||||
|
||||
s3_config = s3_source_auth uri
|
||||
expiration ||= 86400
|
||||
|
||||
current_time = Time.now.utc
|
||||
date_time = current_time.strftime("%Y%m%dT%H%m%SZ")
|
||||
date = date_time[0,8]
|
||||
|
||||
credential_info = "#{date}/#{s3_config.region}/s3/aws4_request"
|
||||
canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com"
|
||||
|
||||
canonical_params = {}
|
||||
canonical_params['X-Amz-Algorithm'] = "AWS4-HMAC-SHA256"
|
||||
canonical_params['X-Amz-Credential'] = "#{s3_config.access_key_id}/#{credential_info}"
|
||||
canonical_params['X-Amz-Date'] = date_time
|
||||
canonical_params['X-Amz-Expires'] = expiration.to_s
|
||||
canonical_params['X-Amz-SignedHeaders'] = "host"
|
||||
canonical_params['X-Amz-Security-Token'] = s3_config.security_token if s3_config.security_token
|
||||
|
||||
# Sorting is required to generate proper signature
|
||||
query_params = canonical_params.sort.to_h.map do |key, value|
|
||||
"#{base64_uri_escape(key)}=#{base64_uri_escape(value)}"
|
||||
end.join('&')
|
||||
|
||||
canonical_request = [
|
||||
'GET',
|
||||
uri.path,
|
||||
query_params,
|
||||
"host:#{canonical_host}",
|
||||
'', # empty params
|
||||
'host',
|
||||
'UNSIGNED-PAYLOAD',
|
||||
].join("\n")
|
||||
|
||||
string_to_sign = [
|
||||
"AWS4-HMAC-SHA256",
|
||||
date_time,
|
||||
credential_info,
|
||||
Digest::SHA256.hexdigest(canonical_request)
|
||||
].join("\n")
|
||||
|
||||
date_key = OpenSSL::HMAC.digest('sha256', "AWS4" + s3_config.secret_access_key, date)
|
||||
date_region_key = OpenSSL::HMAC.digest('sha256', date_key, s3_config.region)
|
||||
date_region_service_key = OpenSSL::HMAC.digest('sha256', date_region_key, "s3")
|
||||
signing_key = OpenSSL::HMAC.digest('sha256', date_region_service_key, "aws4_request")
|
||||
signature = OpenSSL::HMAC.hexdigest('sha256', signing_key, string_to_sign)
|
||||
|
||||
URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}")
|
||||
end
|
||||
|
||||
BASE64_URI_TRANSLATE = { '+' => '%2B', '/' => '%2F', '=' => '%3D' }.freeze
|
||||
|
||||
private
|
||||
|
||||
def base64_uri_escape(str)
|
||||
str.gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] }
|
||||
end
|
||||
|
||||
def proxy_for(proxy, uri)
|
||||
Gem::Request.proxy_uri(proxy || Gem::Request.get_proxy_from_env(uri.scheme))
|
||||
end
|
||||
|
|
@ -418,41 +363,4 @@ class Gem::RemoteFetcher
|
|||
end
|
||||
end
|
||||
|
||||
def s3_source_auth(uri)
|
||||
return S3Config.new(uri.user, uri.password, nil, 'us-east-1') if uri.user && uri.password
|
||||
|
||||
s3_source = Gem.configuration[:s3_source] || Gem.configuration['s3_source']
|
||||
host = uri.host
|
||||
raise FetchError.new("no s3_source key exists in .gemrc", "s3://#{host}") unless s3_source
|
||||
|
||||
auth = s3_source[host] || s3_source[host.to_sym]
|
||||
raise FetchError.new("no key for host #{host} in s3_source in .gemrc", "s3://#{host}") unless auth
|
||||
|
||||
provider = auth[:provider] || auth['provider']
|
||||
case provider
|
||||
when 'env'
|
||||
id = ENV['AWS_ACCESS_KEY_ID']
|
||||
secret = ENV['AWS_SECRET_ACCESS_KEY']
|
||||
security_token = ENV['AWS_SESSION_TOKEN']
|
||||
when 'instance_profile'
|
||||
require 'json'
|
||||
credentials_response = fetch_http URI(EC2_METADATA_CREDENTIALS)
|
||||
credentials = JSON.parse(credentials_response)
|
||||
id = credentials['AccessKeyId']
|
||||
secret = credentials['SecretAccessKey']
|
||||
security_token = credentials['Token']
|
||||
else
|
||||
id = auth[:id] || auth['id']
|
||||
secret = auth[:secret] || auth['secret']
|
||||
raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id && secret
|
||||
|
||||
security_token = auth[:security_token] || auth['security_token']
|
||||
end
|
||||
|
||||
region = auth[:region] || auth['region'] || 'us-east-1'
|
||||
S3Config.new(id, secret, security_token, region)
|
||||
end
|
||||
|
||||
EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance"
|
||||
|
||||
end
|
||||
|
|
|
|||
166
lib/rubygems/s3_uri_signer.rb
Normal file
166
lib/rubygems/s3_uri_signer.rb
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
require 'base64'
|
||||
require 'digest'
|
||||
require 'openssl'
|
||||
require 'rubygems/request'
|
||||
require 'rubygems/request/connection_pools'
|
||||
|
||||
##
|
||||
# S3URISigner implements AWS SigV4 for S3 Source to avoid a dependency on the aws-sdk-* gems
|
||||
# More on AWS SigV4: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
|
||||
class Gem::S3URISigner
|
||||
|
||||
class ConfigurationError < Gem::Exception
|
||||
|
||||
def initialize(message)
|
||||
super message
|
||||
end
|
||||
|
||||
def to_s # :nodoc:
|
||||
"#{super}"
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
class InstanceProfileError < Gem::Exception
|
||||
|
||||
def initialize(message)
|
||||
super message
|
||||
end
|
||||
|
||||
def to_s # :nodoc:
|
||||
"#{super}"
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
attr_accessor :uri
|
||||
|
||||
def initialize(uri)
|
||||
@uri = uri
|
||||
end
|
||||
|
||||
##
|
||||
# Signs S3 URI using query-params according to the reference: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
|
||||
def sign(expiration = nil)
|
||||
expiration ||= 86400
|
||||
|
||||
s3_config = fetch_s3_config
|
||||
|
||||
current_time = Time.now.utc
|
||||
date_time = current_time.strftime("%Y%m%dT%H%m%SZ")
|
||||
date = date_time[0,8]
|
||||
|
||||
credential_info = "#{date}/#{s3_config.region}/s3/aws4_request"
|
||||
canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com"
|
||||
|
||||
canonical_params = {}
|
||||
canonical_params["X-Amz-Algorithm"] = "AWS4-HMAC-SHA256"
|
||||
canonical_params["X-Amz-Credential"] = "#{s3_config.access_key_id}/#{credential_info}"
|
||||
canonical_params["X-Amz-Date"] = date_time
|
||||
canonical_params["X-Amz-Expires"] = expiration.to_s
|
||||
canonical_params["X-Amz-SignedHeaders"] = "host"
|
||||
canonical_params["X-Amz-Security-Token"] = s3_config.security_token if s3_config.security_token
|
||||
|
||||
# Sorting is required to generate proper signature
|
||||
query_params = canonical_params.sort.to_h.map do |key, value|
|
||||
"#{base64_uri_escape(key)}=#{base64_uri_escape(value)}"
|
||||
end.join("&")
|
||||
|
||||
canonical_request = [
|
||||
"GET",
|
||||
uri.path,
|
||||
query_params,
|
||||
"host:#{canonical_host}",
|
||||
"", # empty params
|
||||
"host",
|
||||
"UNSIGNED-PAYLOAD",
|
||||
].join("\n")
|
||||
|
||||
string_to_sign = [
|
||||
"AWS4-HMAC-SHA256",
|
||||
date_time,
|
||||
credential_info,
|
||||
Digest::SHA256.hexdigest(canonical_request)
|
||||
].join("\n")
|
||||
|
||||
date_key = OpenSSL::HMAC.digest("sha256", "AWS4" + s3_config.secret_access_key, date)
|
||||
date_region_key = OpenSSL::HMAC.digest("sha256", date_key, s3_config.region)
|
||||
date_region_service_key = OpenSSL::HMAC.digest("sha256", date_region_key, "s3")
|
||||
signing_key = OpenSSL::HMAC.digest("sha256", date_region_service_key, "aws4_request")
|
||||
signature = OpenSSL::HMAC.hexdigest("sha256", signing_key, string_to_sign)
|
||||
|
||||
URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}")
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region
|
||||
|
||||
##
|
||||
# Extracts S3 configuration for S3 bucket
|
||||
def fetch_s3_config
|
||||
return S3Config.new(uri.user, uri.password, nil, "us-east-1") if uri.user && uri.password
|
||||
|
||||
s3_source = Gem.configuration[:s3_source] || Gem.configuration["s3_source"]
|
||||
host = uri.host
|
||||
raise ConfigurationError.new("no s3_source key exists in .gemrc") unless s3_source
|
||||
|
||||
auth = s3_source[host] || s3_source[host.to_sym]
|
||||
raise ConfigurationError.new("no key for host #{host} in s3_source in .gemrc") unless auth
|
||||
|
||||
provider = auth[:provider] || auth["provider"]
|
||||
case provider
|
||||
when "env"
|
||||
id = ENV["AWS_ACCESS_KEY_ID"]
|
||||
secret = ENV["AWS_SECRET_ACCESS_KEY"]
|
||||
security_token = ENV["AWS_SESSION_TOKEN"]
|
||||
when "instance_profile"
|
||||
require "json"
|
||||
credentials_response = ec2_metadata
|
||||
credentials = JSON.parse(credentials_response)
|
||||
id = credentials["AccessKeyId"]
|
||||
secret = credentials["SecretAccessKey"]
|
||||
security_token = credentials["Token"]
|
||||
else
|
||||
id = auth[:id] || auth["id"]
|
||||
secret = auth[:secret] || auth["secret"]
|
||||
raise ConfigurationError.new("s3_source for #{host} missing id or secret") unless id && secret
|
||||
|
||||
security_token = auth[:security_token] || auth["security_token"]
|
||||
end
|
||||
|
||||
region = auth[:region] || auth["region"] || "us-east-1"
|
||||
S3Config.new(id, secret, security_token, region)
|
||||
end
|
||||
|
||||
def base64_uri_escape(str)
|
||||
str.gsub("\n", "").gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] }
|
||||
end
|
||||
|
||||
def ec2_metadata
|
||||
require 'net/http'
|
||||
|
||||
metadata_uri = URI(EC2_METADATA_CREDENTIALS)
|
||||
|
||||
@request_pool ||= create_request_pool(metadata_uri)
|
||||
request = Gem::Request.new(metadata_uri, Net::HTTP::Get, nil, @request_pool)
|
||||
response = request.fetch
|
||||
|
||||
case response
|
||||
when Net::HTTPOK then
|
||||
response.body
|
||||
else
|
||||
raise InstanceProfileError.new("Unable to fetch AWS credentials from #{metadata_uri}: #{response.message} #{response.code}")
|
||||
end
|
||||
end
|
||||
|
||||
def create_request_pool(uri)
|
||||
proxy_uri = Gem::Request.proxy_uri(Gem::Request.get_proxy_from_env(uri.scheme))
|
||||
certs = Gem::Request.get_cert_files
|
||||
Gem::Request::ConnectionPools.new(proxy_uri, certs).pool_for(uri)
|
||||
end
|
||||
|
||||
BASE64_URI_TRANSLATE = { "+" => "%2B", "/" => "%2F", "=" => "%3D" }.freeze
|
||||
EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance".freeze
|
||||
|
||||
end
|
||||
|
|
@ -659,15 +659,18 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg==
|
|||
def fetcher.request(uri, request_class, last_modified = nil)
|
||||
$fetched_uri = uri
|
||||
res = Net::HTTPOK.new nil, 200, nil
|
||||
case uri.to_s
|
||||
when /^http:\/\/169\.254\.169\.254.*/
|
||||
def res.body() $instance_profile end
|
||||
else
|
||||
def res.body() 'success' end
|
||||
end
|
||||
def res.body() 'success' end
|
||||
res
|
||||
end
|
||||
|
||||
def fetcher.s3_uri_signer(uri)
|
||||
s3_uri_signer = Gem::S3URISigner.new(uri)
|
||||
def s3_uri_signer.ec2_metadata
|
||||
$instance_profile
|
||||
end
|
||||
s3_uri_signer
|
||||
end
|
||||
|
||||
data = fetcher.fetch_s3 URI.parse(url)
|
||||
|
||||
assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T050641Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s
|
||||
|
|
|
|||
Loading…
Reference in a new issue