kotovalexarian-likes-github/fog--fog
1
0
Fork 0
mirror of https://github.com/fog/fog.git synced 2022-11-09 13:51:43 -05:00
Code Releases Activity

[rackspace|storage] add methods for SLO support

Browse source 
Adds #put_static_obj_manifest and #delete_static_large_object methods.
This commit is contained in:
Brian D. Burns 2013-07-02 18:21:37 -04:00
parent d713739d97
commit 23dbccd760
4 changed files with 435 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
lib/fog/rackspace/requests/storage/delete_static_large_object.rb Normal file
View file

@ -0,0 +1,47 @@
module Fog
module Storage
class Rackspace
class Real
# Delete a static large object.
#
# Deletes the SLO manifest +object+ and all segments that it references.
# The server will respond with +200 OK+ for all requests.
# +response.body+ must be inspected for actual results.
#
# @param container [String] Name of container.
# @param object [String] Name of the SLO manifest object.
# @param options [Hash] Additional request headers.
#
# @return [Excon::Response]
# * body [Hash] - Results of the operation.
# * "Number Not Found" [Integer] - Number of missing segments.
# * "Response Status" [String] - Response code for the subrequest of the last failed operation.
# * "Errors" [Array<object_name, response_status>]
# * object_name [String] - Object that generated an error when the delete was attempted.
# * response_status [String] - Response status from the subrequest for object_name.
# * "Number Deleted" [Integer] - Number of segments deleted.
# * "Response Body" [String] - Response body for Response Status.
#
# @raise [Fog::Storage::Rackspace::NotFound] HTTP 404
# @raise [Fog::Storage::Rackspace::BadRequest] HTTP 400
# @raise [Fog::Storage::Rackspace::InternalServerError] HTTP 500
# @raise [Fog::Storage::Rackspace::ServiceError]
# @raise [Excon::Errors::Unauthorized] HTTP 401
#
# @see http://docs.rackspace.com/files/api/v1/cf-devguide/content/Deleting_a_Large_Object-d1e2228.html
def delete_static_large_object(container, object, options = {})
request(
:expects => 200,
:method => 'DELETE',
:headers => options.merge('Content-Type' => 'text/plain'),
:path => "#{Fog::Rackspace.escape(container)}/#{Fog::Rackspace.escape(object)}",
:query => { 'multipart-manifest' => 'delete' }
)
end
end
end
end
end

60
lib/fog/rackspace/requests/storage/put_static_obj_manifest.rb Normal file
View file

@ -0,0 +1,60 @@
module Fog
module Storage
class Rackspace
class Real
# Create a new static large object manifest.
#
# A static large object is similar to a dynamic large object. Whereas a GET for a dynamic large object manifest
# will stream segments based on the manifest's +X-Object-Manifest+ object name prefix, a static large object
# manifest streams segments which are defined by the user within the manifest. Information about each segment is
# provided in +segments+ as an Array of Hash objects, ordered in the sequence which the segments should be streamed.
#
# When the SLO manifest is received, each segment's +etag+ and +size_bytes+ will be verified.
# The +etag+ for each segment is returned in the response to {#put_object}, but may also be calculated.
# e.g. +Digest::MD5.hexdigest(segment_data)+
#
# The maximum number of segments for a static large object is 1000, and all segments (except the last) must be
# at least 1 MiB in size. Unlike a dynamic large object, segments are not required to be in the same container.
#
# @example
# segments = [
# { :path => 'segments_container/first_segment',
# :etag => 'md5 for first_segment',
# :size_bytes => 'byte size of first_segment' },
# { :path => 'segments_container/second_segment',
# :etag => 'md5 for second_segment',
# :size_bytes => 'byte size of second_segment' }
# ]
# put_static_obj_manifest('my_container', 'my_large_object', segments)
#
# @param container [String] Name for container where +object+ will be stored.
# Should be < 256 bytes and must not contain '/'
# @param object [String] Name for manifest object.
# @param segments [Array<Hash>] Segment data for the object.
# @param options [Hash] Config headers for +object+.
#
# @raise [Fog::Storage::Rackspace::NotFound] HTTP 404
# @raise [Fog::Storage::Rackspace::BadRequest] HTTP 400
# @raise [Fog::Storage::Rackspace::InternalServerError] HTTP 500
# @raise [Fog::Storage::Rackspace::ServiceError]
# @raise [Excon::Errors::Unauthorized] HTTP 401
#
# @see http://docs.rackspace.com/files/api/v1/cf-devguide/content/Uploading_the_Manifext-d1e2227.html
def put_static_obj_manifest(container, object, segments, options = {})
request(
:expects => 201,
:method => 'PUT',
:headers => options,
:body => Fog::JSON.encode(segments),
:path => "#{Fog::Rackspace.escape(container)}/#{Fog::Rackspace.escape(object)}",
:query => { 'multipart-manifest' => 'put' }
)
end
end
end
end
end

26
lib/fog/rackspace/storage.rb
View file

@ -25,6 +25,7 @@ module Fog
request :copy_object
request :delete_container
request :delete_object
request :delete_static_large_object
request :get_container
request :get_containers
request :get_object
@ -35,6 +36,7 @@ module Fog
request :put_container
request :put_object
request :put_object_manifest
request :put_static_obj_manifest
request :post_set_meta_temp_url_key
module Utils
@ -54,7 +56,7 @@ module Fog
end
end
end
end
class Mock < Fog::Rackspace::Service
include Utils
@ -107,7 +109,7 @@ module Fog
@rackspace_servicenet = options[:rackspace_servicenet]
@rackspace_auth_token = options[:rackspace_auth_token]
@rackspace_storage_url = options[:rackspace_storage_url]
@rackspace_cdn_url = options[:rackspace_cdn_url]
@rackspace_cdn_url = options[:rackspace_cdn_url]
@rackspace_region = options[:rackspace_region] || :dfw
@rackspace_temp_url_key = options[:rackspace_temp_url_key]
@rackspace_must_reauthenticate = false
@ -117,8 +119,8 @@ module Fog
@persistent = options[:persistent] || false
Excon.defaults[:ssl_verify_peer] = false if service_net?
@connection = Fog::Connection.new(endpoint_uri.to_s, @persistent, @connection_options)
end
end
# Return Account Details
# @return [Fog::Storage::Rackspace::Account] account details object
def account
@ -152,8 +154,8 @@ module Fog
def service_net?
@rackspace_servicenet == true
end
end
def authenticate
if @rackspace_must_reauthenticate || @rackspace_auth_token.nil?
options = {
@ -161,14 +163,14 @@ module Fog
:rackspace_username => @rackspace_username,
:rackspace_auth_url => @rackspace_auth_url,
:connection_options => @connection_options
}
}
super(options)
else
@auth_token = @rackspace_auth_token
@uri = URI.parse(@rackspace_storage_url)
end
end
def service_name
:cloudFiles
end
@ -184,15 +186,15 @@ module Fog
@uri.host = "snet-#{@uri.host}" if service_net?
@uri
end
private
private
def authenticate_v1(options)
credentials = Fog::Rackspace.authenticate(options, @connection_options)
endpoint_uri credentials['X-Storage-Url']
@auth_token = credentials['X-Auth-Token']
end
end
end
end

391
tests/rackspace/requests/storage/large_object_tests.rb
View file

@ -1,106 +1,343 @@
Shindo.tests('Fog::Storage[:rackspace] | large object requests', ["rackspace"]) do
Shindo.tests('Fog::Storage[:rackspace] | large object requests', ['rackspace']) do
unless Fog.mocking?
@directory = Fog::Storage[:rackspace].directories.create(:key => 'foglargeobjecttests')
@directory = Fog::Storage[:rackspace].directories.create(:key => 'foglargeobjecttests')
@directory2 = Fog::Storage[:rackspace].directories.create(:key => 'foglargeobjecttests2')
@segments = {
:a => {
:container => @directory.identity,
:name => 'fog_large_object/a',
:data => 'a' * (1024**2 + 10),
:size => 1024**2 + 10,
:etag => 'c2e97007d59f0c19b850debdcb80cca5'
},
:b => {
:container => @directory.identity,
:name => 'fog_large_object/b',
:data => 'b' * (1024**2 + 20),
:size => 1024**2 + 20,
:etag => 'd35f50622a1259daad75ff7d5512c7ef'
},
:c => {
:container => @directory.identity,
:name => 'fog_large_object2/a',
:data => 'c' * (1024**2 + 30),
:size => 1024**2 + 30,
:etag => '901d3531a87d188041d4d5b43cb464c1'
},
:d => {
:container => @directory2.identity,
:name => 'fog_large_object2/b',
:data => 'd' * (1024**2 + 40),
:size => 1024**2 + 40,
:etag => '350c0e00525198813920a157df185c8d'
}
}
end
tests('success') do
tests("#put_object('foglargeobjecttests', 'fog_large_object/1', ('x' * 4 * 1024 * 1024))").succeeds do
tests('upload test segments').succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object/1', ('x' * 4 * 1024 * 1024))
@segments.each_value do |segment|
Fog::Storage[:rackspace].put_object(segment[:container], segment[:name], segment[:data])
end
end
tests("#put_object('foglargeobjecttests', 'fog_large_object/2', ('x' * 2 * 1024 * 1024))").succeeds do
tests('dynamic large object requests') do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object/2', ('x' * 2 * 1024 * 1024))
tests('using default X-Object-Manifest header') do
tests('#put_object_manifest').succeeds do
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object')
end
tests('#get_object streams all segments matching the default prefix').succeeds do
expected = @segments[:a][:data] + @segments[:b][:data] + @segments[:c][:data]
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == expected
end
# When the manifest object name is equal to the segment prefix, OpenStack treats it as if it's the first segment.
# So you must prepend the manifest object's Etag - Digest::MD5.hexdigest('')
tests('#head_object returns Etag that includes manifest object in calculation').succeeds do
etags = ['d41d8cd98f00b204e9800998ecf8427e', @segments[:a][:etag], @segments[:b][:etag], @segments[:c][:etag]]
expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"2577f38428e895c50de6ea78ccc7da2a"\"
Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
end
end
tests('specifying X-Object-Manifest segment prefix') do
tests('#put_object_manifest').succeeds do
options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object', options)
end
tests('#get_object streams segments only matching the specified prefix').succeeds do
expected = @segments[:a][:data] + @segments[:b][:data]
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == expected
end
tests('#head_object returns Etag that does not include manifest object in calculation').succeeds do
etags = [@segments[:a][:etag], @segments[:b][:etag]]
expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"0f035ed3cc38aa0ef46dda3478fad44d"\"
Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
end
end
tests('storing manifest in a different container than the segments') do
tests('#put_object_manifest').succeeds do
options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', options)
end
tests('#get_object').succeeds do
expected = @segments[:a][:data] + @segments[:b][:data]
Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == expected
end
end
end
tests("#put_object('foglargeobjecttests', 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))").succeeds do
tests('static large object requests') do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))
end
tests("using default X-Object-Manifest header") do
tests('single container') do
tests('#put_static_obj_manifest').succeeds do
segments = [
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
:etag => @segments[:a][:etag],
:size_bytes => @segments[:a][:size] },
{ :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
:etag => @segments[:c][:etag],
:size_bytes => @segments[:c][:size] }
]
Fog::Storage[:rackspace].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
end
tests('#head_object') do
etags = [@segments[:a][:etag], @segments[:c][:etag]]
etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"ad7e633a12e8a4915b45e6dd1d4b0b4b\""
content_length = (@segments[:a][:size] + @segments[:c][:size]).to_s
response = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object')
returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
end
tests('#get_object').succeeds do
expected = @segments[:a][:data] + @segments[:c][:data]
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == expected
end
tests('#delete_static_large_object') do
expected = {
'Number Not Found' => 0,
'Response Status' => '200 OK',
'Errors' => [],
'Number Deleted' => 3,
'Response Body' => ''
}
returns(expected, 'deletes manifest and segments') do
Fog::Storage[:rackspace].delete_static_large_object(@directory.identity, 'fog_large_object').body
end
end
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object')
end
tests("#get_object streams all segments matching the default prefix").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 7 * 1024 * 1024)
tests('multiple containers') do
tests('#put_static_obj_manifest').succeeds do
segments = [
{ :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
:etag => @segments[:b][:etag],
:size_bytes => @segments[:b][:size] },
{ :path => "#{ @segments[:d][:container] }/#{ @segments[:d][:name] }",
:etag => @segments[:d][:etag],
:size_bytes => @segments[:d][:size] }
]
Fog::Storage[:rackspace].put_static_obj_manifest(@directory2.identity, 'fog_large_object', segments)
end
tests('#head_object') do
etags = [@segments[:b][:etag], @segments[:d][:etag]]
etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"9801a4cc4472896a1e975d03f0d2c3f8\""
content_length = (@segments[:b][:size] + @segments[:d][:size]).to_s
response = Fog::Storage[:rackspace].head_object(@directory2.identity, 'fog_large_object')
returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
end
tests('#get_object').succeeds do
expected = @segments[:b][:data] + @segments[:d][:data]
Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == expected
end
tests('#delete_static_large_object') do
expected = {
'Number Not Found' => 0,
'Response Status' => '200 OK',
'Errors' => [],
'Number Deleted' => 3,
'Response Body' => ''
}
returns(expected, 'deletes manifest and segments') do
Fog::Storage[:rackspace].delete_static_large_object(@directory2.identity, 'fog_large_object').body
end
end
end
tests("#head_object returns Etag that includes manifest object in calculation").succeeds do
pending if Fog.mocking?
etags = []
# When the manifest object name is equal to the prefix, OpenStack treats it as if it's the first segment.
etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e"
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
etags << Digest::MD5.hexdigest('x' * 1 * 1024 * 1024) # => "b561f87202d04959e37588ee05cf5b10"
expected = Digest::MD5.hexdigest(etags.join) # => "42e92048bd2c8085e7072b0b55fd76ab"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"42e92048bd2c8085e7072b0b55fd76abu"\"
end
end
tests("specifying X-Object-Manifest segment prefix") do
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/')").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
end
tests("#get_object streams segments only matching the specified prefix").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end
tests("#head_object returns Etag that does not include manifest object in calculation").succeeds do
pending if Fog.mocking?
etags = []
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84"
actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag']
actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\"
end
end
tests("storing manifest object in a different container than the segments") do
tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/'})").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
end
tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do
pending if Fog.mocking?
Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
end
end
unless Fog.mocking?
['fog_large_object', 'fog_large_object/1', 'fog_large_object/2', 'fog_large_object2/1'].each do |key|
@directory.files.new(:key => key).destroy
end
@directory2.files.new(:key => 'fog_large_object').destroy
end
end
tests('failure') do
tests("put_object_manifest")
tests('dynamic large object requests') do
pending if Fog.mocking?
tests('#put_object_manifest with missing container').raises(Fog::Storage::Rackspace::NotFound) do
Fog::Storage[:rackspace].put_object_manifest('fognoncontainer', 'fog_large_object')
end
end
tests('static large object requests') do
pending if Fog.mocking?
tests('upload test segments').succeeds do
Fog::Storage[:rackspace].put_object(@segments[:a][:container], @segments[:a][:name], @segments[:a][:data])
Fog::Storage[:rackspace].put_object(@segments[:b][:container], @segments[:b][:name], @segments[:b][:data])
end
tests('#put_static_obj_manifest with missing container').raises(Fog::Storage::Rackspace::NotFound) do
Fog::Storage[:rackspace].put_static_obj_manifest('fognoncontainer', 'fog_large_object', [])
end
tests('#put_static_obj_manifest with missing object') do
segments = [
{ :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
:etag => @segments[:c][:etag],
:size_bytes => @segments[:c][:size] }
]
expected = { 'Errors' => [[segments[0][:path], '404 Not Found']] }
error = nil
begin
Fog::Storage[:rackspace].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
rescue => err
error = err
end
raises(Fog::Storage::Rackspace::BadRequest) do
raise error if error
end
returns(expected, 'returns error information') do
error.response_data
end
end
tests('#put_static_obj_manifest with invalid etag') do
segments = [
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
:etag => @segments[:b][:etag],
:size_bytes => @segments[:a][:size] }
]
expected = { 'Errors' => [[segments[0][:path], 'Etag Mismatch']] }
error = nil
begin
Fog::Storage[:rackspace].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
rescue => err
error = err
end
raises(Fog::Storage::Rackspace::BadRequest) do
raise error if error
end
returns(expected, 'returns error information') do
error.response_data
end
end
tests('#put_static_obj_manifest with invalid byte_size') do
segments = [
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
:etag => @segments[:a][:etag],
:size_bytes => @segments[:b][:size] }
]
expected = { 'Errors' => [[segments[0][:path], 'Size Mismatch']] }
error = nil
begin
Fog::Storage[:rackspace].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
rescue => err
error = err
end
raises(Fog::Storage::Rackspace::BadRequest) do
raise error if error
end
returns(expected, 'returns error information') do
error.response_data
end
end
tests('#delete_static_large_object with missing container').raises(Fog::Storage::Rackspace::NotFound) do
Fog::Storage[:rackspace].delete_static_large_object('fognoncontainer', 'fog_large_object')
end
tests('#delete_static_large_object with missing manifest').raises(Fog::Storage::Rackspace::NotFound) do
Fog::Storage[:rackspace].delete_static_large_object(@directory.identity, 'fog_non_object')
end
tests('#delete_static_large_object with missing segment') do
tests('#put_static_obj_manifest for segments :a and :b').succeeds do
segments = [
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
:etag => @segments[:a][:etag],
:size_bytes => @segments[:a][:size] },
{ :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
:etag => @segments[:b][:etag],
:size_bytes => @segments[:b][:size] }
]
Fog::Storage[:rackspace].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
end
tests('#delete_object segment :b').succeeds do
Fog::Storage[:rackspace].delete_object(@segments[:b][:container], @segments[:b][:name])
end
tests('#delete_static_large_object') do
expected = {
'Number Not Found' => 1,
'Response Status' => '200 OK',
'Errors' => [],
'Number Deleted' => 2,
'Response Body' => ''
}
returns(expected, 'deletes manifest and segment :a, and reports missing segment :b') do
Fog::Storage[:rackspace].delete_static_large_object(@directory.identity, 'fog_large_object').body
end
end
end
end
end