2012-05-15 14:54:20 -04:00
require 'fog/aws'
2011-08-31 16:52:53 -04:00
require 'fog/storage'
2010-09-08 17:40:02 -04:00
module Fog
2011-06-15 17:26:43 -04:00
module Storage
class AWS < Fog :: Service
2012-06-18 04:44:51 -04:00
extend Fog :: AWS :: CredentialFetcher :: ServiceMethods
2010-09-08 17:40:02 -04:00
2013-04-12 06:31:10 -04:00
COMPLIANT_BUCKET_NAMES = / ^(?:[a-z]| \ d(?! \ d{0,2}(?: \ . \ d{1,3}){3}$))(?:[a-z0-9]| \ .(?![ \ . \ -])| \ -(?![ \ .])){1,61}[a-z0-9]$ /
2013-04-12 12:37:01 -04:00
DEFAULT_REGION = 'us-east-1'
2013-04-12 06:31:10 -04:00
DEFAULT_SCHEME = 'https'
DEFAULT_SCHEME_PORT = {
'http' = > 80 ,
'https' = > 443
}
VALID_QUERY_KEYS = %w[
acl
cors
delete
lifecycle
location
logging
notification
partNumber
policy
requestPayment
response - cache - control
response - content - disposition
response - content - encoding
response - content - language
response - content - type
response - expires
2013-04-08 20:51:47 -04:00
restore
2013-04-12 06:31:10 -04:00
torrent
uploadId
uploads
versionId
versioning
versions
website
]
2010-12-16 18:31:24 -05:00
requires :aws_access_key_id , :aws_secret_access_key
2013-03-06 09:29:22 -05:00
recognizes :endpoint , :region , :host , :path , :port , :scheme , :persistent , :use_iam_profile , :aws_session_token , :aws_credentials_expire_at , :path_style
2011-01-21 23:45:37 -05:00
2012-06-08 19:36:08 -04:00
secrets :aws_secret_access_key , :hmac
2011-08-24 14:50:42 -04:00
model_path 'fog/aws/models/storage'
2010-09-08 17:40:02 -04:00
collection :directories
model :directory
collection :files
model :file
2011-08-24 14:50:42 -04:00
request_path 'fog/aws/requests/storage'
2010-11-15 20:17:37 -05:00
request :abort_multipart_upload
request :complete_multipart_upload
2010-09-08 17:40:02 -04:00
request :copy_object
request :delete_bucket
2012-12-05 13:30:41 -05:00
request :delete_bucket_cors
2012-01-19 14:25:39 -05:00
request :delete_bucket_lifecycle
2011-06-07 21:41:24 -04:00
request :delete_bucket_policy
2011-02-18 15:06:51 -05:00
request :delete_bucket_website
2010-09-08 17:40:02 -04:00
request :delete_object
2012-01-04 08:19:32 -05:00
request :delete_multiple_objects
2010-09-08 17:40:02 -04:00
request :get_bucket
request :get_bucket_acl
2012-12-05 13:30:41 -05:00
request :get_bucket_cors
2012-01-19 14:25:39 -05:00
request :get_bucket_lifecycle
2010-09-08 17:40:02 -04:00
request :get_bucket_location
request :get_bucket_logging
request :get_bucket_object_versions
2011-06-07 21:41:24 -04:00
request :get_bucket_policy
2010-09-08 17:40:02 -04:00
request :get_bucket_versioning
2011-02-18 15:06:51 -05:00
request :get_bucket_website
2010-09-08 17:40:02 -04:00
request :get_object
request :get_object_acl
request :get_object_torrent
2011-06-27 17:50:32 -04:00
request :get_object_http_url
request :get_object_https_url
2010-09-08 17:40:02 -04:00
request :get_object_url
request :get_request_payment
request :get_service
request :head_object
2010-11-15 20:17:37 -05:00
request :initiate_multipart_upload
request :list_multipart_uploads
request :list_parts
2010-10-18 20:56:01 -04:00
request :post_object_hidden_fields
2013-04-08 20:51:47 -04:00
request :post_object_restore
2010-09-08 17:40:02 -04:00
request :put_bucket
request :put_bucket_acl
2012-12-05 13:30:41 -05:00
request :put_bucket_cors
2012-01-19 14:25:39 -05:00
request :put_bucket_lifecycle
2010-09-08 17:40:02 -04:00
request :put_bucket_logging
2011-06-07 21:41:24 -04:00
request :put_bucket_policy
2010-09-08 17:40:02 -04:00
request :put_bucket_versioning
2011-02-18 15:06:51 -05:00
request :put_bucket_website
2010-09-08 17:40:02 -04:00
request :put_object
2010-11-22 02:44:44 -05:00
request :put_object_acl
2010-09-08 17:40:02 -04:00
request :put_object_url
request :put_request_payment
2010-12-21 15:12:30 -05:00
request :sync_clock
2010-11-15 20:17:37 -05:00
request :upload_part
2010-09-08 17:40:02 -04:00
module Utils
2011-08-04 10:41:02 -04:00
attr_accessor :region
2010-11-05 14:37:12 -04:00
def cdn
@cdn || = Fog :: AWS :: CDN . new (
:aws_access_key_id = > @aws_access_key_id ,
2012-06-18 04:44:51 -04:00
:aws_secret_access_key = > @aws_secret_access_key ,
:use_iam_profile = > @use_iam_profile
2010-11-05 14:37:12 -04:00
)
end
2011-06-27 17:50:32 -04:00
def http_url ( params , expires )
2013-04-12 06:31:10 -04:00
signed_url ( params . merge ( :scheme = > 'http' ) , expires )
2011-06-27 17:50:32 -04:00
end
def https_url ( params , expires )
2013-04-12 06:31:10 -04:00
signed_url ( params . merge ( :scheme = > 'https' ) , expires )
2011-06-27 17:50:32 -04:00
end
2010-09-08 17:40:02 -04:00
def url ( params , expires )
2011-10-19 15:49:34 -04:00
Fog :: Logger . deprecation ( " Fog::Storage::AWS => # url is deprecated, use # https_url instead [light_black]( #{ caller . first } )[/] " )
2011-06-27 17:50:32 -04:00
https_url ( params , expires )
end
2013-04-12 06:31:10 -04:00
def request_url ( params )
params = request_params ( params )
params_to_url ( params )
end
def signed_url ( params , expires )
expires = expires . to_i
signature = signature ( params , expires )
params = request_params ( params )
params [ :query ] = ( params [ :query ] || { } ) . merge ( {
'AWSAccessKeyId' = > @aws_access_key_id ,
'Signature' = > signature ,
'Expires' = > expires ,
} )
params [ :query ] [ 'x-amz-security-token' ] = @aws_session_token if @aws_session_token
params_to_url ( params )
end
2011-06-27 17:50:32 -04:00
private
2013-04-12 06:31:10 -04:00
def region_to_host ( region = nil )
case region . to_s
when DEFAULT_REGION , ''
's3.amazonaws.com'
else
" s3- #{ region } .amazonaws.com "
2012-04-22 22:16:19 -04:00
end
2013-04-12 06:31:10 -04:00
end
def object_to_path ( object_name = nil )
'/' + Fog :: AWS . escape ( object_name . to_s ) . gsub ( '%2F' , '/' )
end
2013-04-16 09:40:52 -04:00
def bucket_to_path ( bucket_name , path = nil )
" / #{ Fog :: AWS . escape ( bucket_name . to_s ) } #{ path } "
end
2013-04-12 06:31:10 -04:00
# Transforms things like bucket_name, object_name, region
#
# Should yield the same result when called f*f
def request_params ( params )
headers = params [ :headers ] || { }
if params [ :scheme ]
scheme = params [ :scheme ]
port = params [ :port ]
else
scheme = @scheme || DEFAULT_SCHEME
port = @port
2012-04-22 22:16:19 -04:00
end
2013-04-12 06:31:10 -04:00
if DEFAULT_SCHEME_PORT [ scheme ] == port
port = nil
end
if params [ :region ]
region = params [ :region ]
host = params [ :host ] || region_to_host ( region )
else
region = @region || DEFAULT_REGION
host = params [ :host ] || @host || region_to_host ( region )
end
path = params [ :path ] || object_to_path ( params [ :object_name ] )
path = '/' + path if path [ 0 .. 0 ] != '/'
if params [ :bucket_name ]
bucket_name = params [ :bucket_name ]
path_style = params . fetch ( :path_style , @path_style )
if ! path_style && COMPLIANT_BUCKET_NAMES !~ bucket_name
Fog :: Logger . warning ( " fog: the specified s3 bucket name( #{ bucket_name } ) is not a valid dns name, which will negatively impact performance. For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html " )
path_style = true
end
if path_style
2013-04-16 09:40:52 -04:00
path = bucket_to_path bucket_name , path
2013-04-12 06:31:10 -04:00
else
host = [ bucket_name , host ] . join ( '.' )
2011-06-08 13:37:52 -04:00
end
2011-06-07 18:25:12 -04:00
end
2013-04-12 06:31:10 -04:00
ret = params . merge ( {
:scheme = > scheme ,
:host = > host ,
:port = > port ,
:path = > path ,
:headers = > headers ,
} )
2013-04-08 20:51:47 -04:00
2013-04-12 06:31:10 -04:00
#
ret . delete ( :path_style )
ret . delete ( :bucket_name )
ret . delete ( :object_name )
ret . delete ( :region )
ret
end
def params_to_url ( params )
query = params [ :query ] && params [ :query ] . map do | key , value |
if value
[ key , Fog :: AWS . escape ( value . to_s ) ] . join ( '=' )
else
key
end
end . join ( '&' )
URI :: Generic . build ( {
:scheme = > params [ :scheme ] ,
:host = > params [ :host ] ,
:port = > params [ :port ] ,
:path = > params [ :path ] ,
:query = > query ,
} ) . to_s
2010-09-08 17:40:02 -04:00
end
2011-08-04 10:41:02 -04:00
2010-09-08 17:40:02 -04:00
end
class Mock
include Utils
2010-11-18 17:17:11 -05:00
def self . acls ( type )
case type
when 'private'
2011-01-21 23:45:37 -05:00
{
2010-11-18 17:17:11 -05:00
" AccessControlList " = > [
{
" Permission " = > " FULL_CONTROL " ,
" Grantee " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
}
] ,
" Owner " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
}
when 'public-read'
2011-01-21 23:45:37 -05:00
{
" AccessControlList " = > [
{
" Permission " = > " FULL_CONTROL " ,
" Grantee " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
} ,
{
" Permission " = > " READ " ,
" Grantee " = > { " URI " = > " http://acs.amazonaws.com/groups/global/AllUsers " }
}
] ,
" Owner " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
}
2010-11-18 17:17:11 -05:00
when 'public-read-write'
2011-01-21 23:45:37 -05:00
{
" AccessControlList " = > [
{
" Permission " = > " FULL_CONTROL " ,
" Grantee " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
} ,
{
" Permission " = > " READ " ,
" Grantee " = > { " URI " = > " http://acs.amazonaws.com/groups/global/AllUsers " }
} ,
{
" Permission " = > " WRITE " ,
" Grantee " = > { " URI " = > " http://acs.amazonaws.com/groups/global/AllUsers " }
}
] ,
" Owner " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
}
2010-11-18 17:17:11 -05:00
when 'authenticated-read'
2011-01-21 23:45:37 -05:00
{
" AccessControlList " = > [
{
" Permission " = > " FULL_CONTROL " ,
" Grantee " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
} ,
{
" Permission " = > " READ " ,
" Grantee " = > { " URI " = > " http://acs.amazonaws.com/groups/global/AuthenticatedUsers " }
}
] ,
" Owner " = > { " DisplayName " = > " me " , " ID " = > " 2744ccd10c7533bd736ad890f9dd5cab2adb27b07d500b9493f29cdc420cb2e0 " }
}
2010-11-18 17:17:11 -05:00
end
end
2010-09-08 17:40:02 -04:00
def self . data
2010-09-24 16:50:44 -04:00
@data || = Hash . new do | hash , region |
2010-10-13 16:20:18 -04:00
hash [ region ] = Hash . new do | region_hash , key |
region_hash [ key ] = {
2010-11-18 17:17:11 -05:00
:acls = > {
:bucket = > { } ,
:object = > { }
} ,
2013-01-03 04:15:38 -05:00
:buckets = > { } ,
:cors = > {
:bucket = > { }
}
2010-09-24 16:50:44 -04:00
}
end
2010-09-08 17:40:02 -04:00
end
end
2011-05-19 10:05:33 -04:00
def self . reset
@data = nil
end
2010-09-08 17:40:02 -04:00
def initialize ( options = { } )
2012-06-18 04:44:51 -04:00
@use_iam_profile = options [ :use_iam_profile ]
setup_credentials ( options )
2013-04-12 06:31:10 -04:00
@region = options [ :region ] || DEFAULT_REGION
@host = options [ :host ] || region_to_host ( @region )
@scheme = options [ :scheme ] || DEFAULT_SCHEME
2011-05-19 18:35:33 -04:00
end
def data
self . class . data [ @region ] [ @aws_access_key_id ]
2011-03-10 14:16:55 -05:00
end
def reset_data
self . class . data [ @region ] . delete ( @aws_access_key_id )
2010-09-08 17:40:02 -04:00
end
2013-04-12 06:31:10 -04:00
def signature ( params , expires )
2010-09-08 17:40:02 -04:00
" foo "
end
2011-03-10 14:16:55 -05:00
2012-06-23 17:44:10 -04:00
def setup_credentials ( options )
2012-06-18 04:44:51 -04:00
@aws_access_key_id = options [ :aws_access_key_id ]
@aws_secret_access_key = options [ :aws_secret_access_key ]
@aws_session_token = options [ :aws_session_token ]
@aws_credentials_expire_at = options [ :aws_credentials_expire_at ]
end
2010-09-08 17:40:02 -04:00
end
class Real
include Utils
2012-06-18 04:44:51 -04:00
include Fog :: AWS :: CredentialFetcher :: ConnectionMethods
2010-09-08 17:40:02 -04:00
# Initialize connection to S3
#
# ==== Notes
2010-09-27 19:08:04 -04:00
# options parameter must include values for :aws_access_key_id and
2010-09-08 17:40:02 -04:00
# :aws_secret_access_key in order to create a connection
#
# ==== Examples
2011-10-12 09:45:34 -04:00
# s3 = Fog::Storage.new(
# :provider => "AWS",
2010-09-08 17:40:02 -04:00
# :aws_access_key_id => your_aws_access_key_id,
# :aws_secret_access_key => your_aws_secret_access_key
# )
#
# ==== Parameters
# * options<~Hash> - config arguments for connection. Defaults to {}.
#
# ==== Returns
# * S3 object with connection to aws.
def initialize ( options = { } )
2011-02-16 20:25:50 -05:00
require 'fog/core/parser'
2012-06-18 04:44:51 -04:00
@use_iam_profile = options [ :use_iam_profile ]
setup_credentials ( options )
2011-09-12 11:01:48 -04:00
@connection_options = options [ :connection_options ] || { }
2013-04-12 06:31:10 -04:00
@persistent = options . fetch ( :persistent , false )
2012-01-04 08:19:32 -05:00
2010-09-29 20:53:32 -04:00
if @endpoint = options [ :endpoint ]
endpoint = URI . parse ( @endpoint )
@host = endpoint . host
2011-11-04 11:28:27 -04:00
@path = if endpoint . path . empty?
'/'
else
endpoint . path
end
2010-09-29 20:53:32 -04:00
@scheme = endpoint . scheme
2013-04-12 06:31:10 -04:00
@port = endpoint . port
2010-09-29 20:53:32 -04:00
else
2013-04-12 06:31:10 -04:00
@region = options [ :region ] || DEFAULT_REGION
@host = options [ :host ] || region_to_host ( @region )
2011-09-12 11:01:48 -04:00
@path = options [ :path ] || '/'
2013-04-12 06:31:10 -04:00
@scheme = options [ :scheme ] || DEFAULT_SCHEME
@port = options [ :port ] || DEFAULT_SCHEME_PORT [ @scheme ]
2013-03-06 09:29:22 -05:00
@path_style = options [ :path_style ] || false
2010-09-29 20:53:32 -04:00
end
2013-04-12 06:31:10 -04:00
2011-09-12 11:01:48 -04:00
@connection = Fog :: Connection . new ( " #{ @scheme } :// #{ @host } : #{ @port } #{ @path } " , @persistent , @connection_options )
2010-09-08 17:40:02 -04:00
end
def reload
@connection . reset
end
2013-04-12 06:31:10 -04:00
def signature ( params , expires )
headers = params [ :headers ] || { }
2010-09-08 17:40:02 -04:00
string_to_sign =
<<-DATA
2011-06-07 18:28:57 -04:00
#{params[:method].to_s.upcase}
2013-04-12 06:31:10 -04:00
#{headers['Content-MD5']}
#{headers['Content-Type']}
#{expires}
2010-09-08 17:40:02 -04:00
DATA
amz_headers , canonical_amz_headers = { } , ''
2013-04-12 06:31:10 -04:00
for key , value in headers
2010-09-08 17:40:02 -04:00
if key [ 0 .. 5 ] == 'x-amz-'
amz_headers [ key ] = value
end
end
amz_headers = amz_headers . sort { | x , y | x [ 0 ] < = > y [ 0 ] }
for key , value in amz_headers
canonical_amz_headers << " #{ key } : #{ value } \n "
end
2010-11-15 20:17:37 -05:00
string_to_sign << canonical_amz_headers
2010-09-08 17:40:02 -04:00
2013-04-12 06:31:10 -04:00
query_string = ''
if params [ :query ]
query_args = [ ]
for key in params [ :query ] . keys . sort
if VALID_QUERY_KEYS . include? ( key )
value = params [ :query ] [ key ]
if value
2013-05-18 21:43:28 -04:00
query_args << " #{ key } = #{ value } "
2013-04-12 06:31:10 -04:00
else
query_args << key
end
2013-04-16 04:50:30 -04:00
end
2010-09-08 17:40:02 -04:00
end
2013-04-12 06:31:10 -04:00
if query_args . any?
query_string = '?' + query_args . join ( '&' )
end
2010-09-08 17:40:02 -04:00
end
2013-04-12 06:31:10 -04:00
canonical_path = ( params [ :path ] || object_to_path ( params [ :object_name ] ) ) . to_s
canonical_path = '/' + canonical_path if canonical_path [ 0 .. 0 ] != '/'
if params [ :bucket_name ]
canonical_resource = " / #{ params [ :bucket_name ] } #{ canonical_path } "
else
canonical_resource = canonical_path
2010-09-08 17:40:02 -04:00
end
2013-04-12 06:31:10 -04:00
canonical_resource << query_string
2010-11-15 20:17:37 -05:00
string_to_sign << canonical_resource
2010-09-08 17:40:02 -04:00
signed_string = @hmac . sign ( string_to_sign )
2012-05-14 15:26:41 -04:00
Base64 . encode64 ( signed_string ) . chomp!
2010-09-08 17:40:02 -04:00
end
2011-02-08 13:06:44 -05:00
private
2012-06-18 04:44:51 -04:00
def setup_credentials ( options )
@aws_access_key_id = options [ :aws_access_key_id ]
@aws_secret_access_key = options [ :aws_secret_access_key ]
@aws_session_token = options [ :aws_session_token ]
@aws_credentials_expire_at = options [ :aws_credentials_expire_at ]
@hmac = Fog :: HMAC . new ( 'sha1' , @aws_secret_access_key )
end
2011-02-08 13:06:44 -05:00
def request ( params , & block )
2012-06-18 04:44:51 -04:00
refresh_credentials_if_expired
2013-04-12 06:31:10 -04:00
expires = Fog :: Time . now . to_date_header
2013-07-03 16:11:13 -04:00
params [ :headers ] [ 'x-amz-security-token' ] = @aws_session_token if @aws_session_token
2013-04-12 06:31:10 -04:00
signature = signature ( params , expires )
2013-04-08 20:51:47 -04:00
2013-04-12 06:31:10 -04:00
params = request_params ( params )
params . delete ( :port ) unless params [ :port ]
params [ :headers ] [ 'Date' ] = expires
params [ :headers ] [ 'Authorization' ] = " AWS #{ @aws_access_key_id } : #{ signature } "
2011-02-08 13:06:44 -05:00
# FIXME: ToHashParser should make this not needed
original_params = params . dup
2011-02-09 07:36:12 -05:00
begin
response = @connection . request ( params , & block )
rescue Excon :: Errors :: TemporaryRedirect = > error
2013-04-12 06:31:10 -04:00
headers = ( error . response . is_a? ( Hash ) ? error . response [ :headers ] : error . response . headers )
uri = URI . parse ( headers [ 'Location' ] )
2011-09-02 13:30:10 -04:00
Fog :: Logger . warning ( " fog: followed redirect to #{ uri . host } , connecting to the matching region will be more performant " )
2011-09-12 11:01:48 -04:00
response = Fog :: Connection . new ( " #{ @scheme } :// #{ uri . host } : #{ @port } " , false , @connection_options ) . request ( original_params , & block )
2011-02-08 13:06:44 -05:00
end
response
end
2010-09-08 17:40:02 -04:00
end
end
end
end