mirror of
https://github.com/fog/fog.git
synced 2022-11-09 13:51:43 -05:00
Merge pull request #2191 from burns/openstack_bulk_delete_slo
[openstack|storage] add bulk delete and SLO support
This commit is contained in:
Kyle Rames
commit
dc545e42f2
8 changed files with 613 additions and 99 deletions
|
|
@ -0,0 +1,67 @@
|
|||
module Fog
|
||||
module Storage
|
||||
class OpenStack
|
||||
class Real
|
||||
|
||||
# Deletes multiple objects or containers with a single request.
|
||||
#
|
||||
# To delete objects from a single container, +container+ may be provided
|
||||
# and +object_names+ should be an Array of object names within the container.
|
||||
#
|
||||
# To delete objects from multiple containers or delete containers,
|
||||
# +container+ should be +nil+ and all +object_names+ should be prefixed with a container name.
|
||||
#
|
||||
# Containers must be empty when deleted. +object_names+ are processed in the order given,
|
||||
# so objects within a container should be listed first to empty the container.
|
||||
#
|
||||
# Up to 10,000 objects may be deleted in a single request.
|
||||
# The server will respond with +200 OK+ for all requests.
|
||||
# +response.body+ must be inspected for actual results.
|
||||
#
|
||||
# @example Delete objects from a container
|
||||
# object_names = ['object', 'another/object']
|
||||
# conn.delete_multiple_objects('my_container', object_names)
|
||||
#
|
||||
# @example Delete objects from multiple containers
|
||||
# object_names = ['container_a/object', 'container_b/object']
|
||||
# conn.delete_multiple_objects(nil, object_names)
|
||||
#
|
||||
# @example Delete a container and all it's objects
|
||||
# object_names = ['my_container/object_a', 'my_container/object_b', 'my_container']
|
||||
# conn.delete_multiple_objects(nil, object_names)
|
||||
#
|
||||
# @param container [String,nil] Name of container.
|
||||
# @param object_names [Array<String>] Object names to be deleted.
|
||||
# @param options [Hash] Additional request headers.
|
||||
#
|
||||
# @return [Excon::Response]
|
||||
# * body [Hash] - Results of the operation.
|
||||
# * "Number Not Found" [Integer] - Number of missing objects or containers.
|
||||
# * "Response Status" [String] - Response code for the subrequest of the last failed operation.
|
||||
# * "Errors" [Array<object_name, response_status>]
|
||||
# * object_name [String] - Object that generated an error when the delete was attempted.
|
||||
# * response_status [String] - Response status from the subrequest for object_name.
|
||||
# * "Number Deleted" [Integer] - Number of objects or containers deleted.
|
||||
# * "Response Body" [String] - Response body for "Response Status".
|
||||
def delete_multiple_objects(container, object_names, options = {})
|
||||
body = object_names.map do |name|
|
||||
object_name = container ? "#{ container }/#{ name }" : name
|
||||
URI.encode(object_name)
|
||||
end.join("\n")
|
||||
|
||||
response = request({
|
||||
:expects => 200,
|
||||
:method => 'DELETE',
|
||||
:headers => options.merge('Content-Type' => 'text/plain',
|
||||
'Accept' => 'application/json'),
|
||||
:body => body,
|
||||
:query => { 'bulk-delete' => true }
|
||||
}, false)
|
||||
response.body = Fog::JSON.decode(response.body)
|
||||
response
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
module Fog
|
||||
module Storage
|
||||
class OpenStack
|
||||
class Real
|
||||
|
||||
# Delete a static large object.
|
||||
#
|
||||
# Deletes the SLO manifest +object+ and all segments that it references.
|
||||
# The server will respond with +200 OK+ for all requests.
|
||||
# +response.body+ must be inspected for actual results.
|
||||
#
|
||||
# @param container [String] Name of container.
|
||||
# @param object [String] Name of the SLO manifest object.
|
||||
# @param options [Hash] Additional request headers.
|
||||
#
|
||||
# @return [Excon::Response]
|
||||
# * body [Hash] - Results of the operation.
|
||||
# * "Number Not Found" [Integer] - Number of missing segments.
|
||||
# * "Response Status" [String] - Response code for the subrequest of the last failed operation.
|
||||
# * "Errors" [Array<object_name, response_status>]
|
||||
# * object_name [String] - Object that generated an error when the delete was attempted.
|
||||
# * response_status [String] - Response status from the subrequest for object_name.
|
||||
# * "Number Deleted" [Integer] - Number of segments deleted.
|
||||
# * "Response Body" [String] - Response body for Response Status.
|
||||
#
|
||||
# @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html
|
||||
def delete_static_large_object(container, object, options = {})
|
||||
response = request({
|
||||
:expects => 200,
|
||||
:method => 'DELETE',
|
||||
:headers => options.merge('Content-Type' => 'text/plain',
|
||||
'Accept' => 'application/json'),
|
||||
:path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}",
|
||||
:query => { 'multipart-manifest' => 'delete' }
|
||||
}, false)
|
||||
response.body = Fog::JSON.decode(response.body)
|
||||
response
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
module Fog
|
||||
module Storage
|
||||
class OpenStack
|
||||
class Real
|
||||
|
||||
# Create a new dynamic large object manifest
|
||||
#
|
||||
# Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("<container>/<prefix>")
|
||||
# for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix.
|
||||
# Segments must sort in the order they should be concatenated. Note that any future objects stored in the container
|
||||
# along with the segments that match the prefix will be included when retrieving the manifest object.
|
||||
#
|
||||
# All segments must be stored in the same container, but may be in a different container than the manifest object.
|
||||
# The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+
|
||||
# to specify the prefix and/or the container where segments were stored.
|
||||
# If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::OpenStack.escape}).
|
||||
#
|
||||
# @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/'
|
||||
# @param object [String] Name for manifest object.
|
||||
# @param options [Hash] Config headers for +object+.
|
||||
# @option options [String] 'X-Object-Manifest' ("container/object") "<container>/<prefix>" for segment objects.
|
||||
#
|
||||
# @raise [Fog::Storage::OpenStack::NotFound] HTTP 404
|
||||
# @raise [Excon::Errors::BadRequest] HTTP 400
|
||||
# @raise [Excon::Errors::Unauthorized] HTTP 401
|
||||
# @raise [Excon::Errors::HTTPStatusError]
|
||||
#
|
||||
# @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/dynamic-large-object-creation.html
|
||||
def put_dynamic_obj_manifest(container, object, options = {})
|
||||
path = "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}"
|
||||
headers = {'X-Object-Manifest' => path}.merge(options)
|
||||
request(
|
||||
:expects => 201,
|
||||
:headers => headers,
|
||||
:method => 'PUT',
|
||||
:path => path
|
||||
)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -3,33 +3,11 @@ module Fog
|
|||
class OpenStack
|
||||
class Real
|
||||
|
||||
# Create a new manifest object
|
||||
# Create a new dynamic large object manifest
|
||||
#
|
||||
# Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("<container>/<prefix>")
|
||||
# for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix.
|
||||
# Segments must sort in the order they should be concatenated. Note that any future objects stored in the container
|
||||
# along with the segments that match the prefix will be included when retrieving the manifest object.
|
||||
#
|
||||
# All segments must be stored in the same container, but may be in a different container than the manifest object.
|
||||
# The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+
|
||||
# to specify the prefix and/or the container where segments were stored.
|
||||
# If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::Rackspace.escape}).
|
||||
#
|
||||
# @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/'
|
||||
# @param object [String] Name for manifest object.
|
||||
# @param options [Hash] Config headers for +object+.
|
||||
# @option options [String] 'X-Object-Manifest' ("container/object") "<container>/<prefix>" for segment objects.
|
||||
#
|
||||
# @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/large-object-creation.html
|
||||
# This is an alias for {#put_dynamic_obj_manifest} for backward compatibility.
|
||||
def put_object_manifest(container, object, options = {})
|
||||
path = "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}"
|
||||
headers = {'X-Object-Manifest' => path}.merge(options)
|
||||
request(
|
||||
:expects => 201,
|
||||
:headers => headers,
|
||||
:method => 'PUT',
|
||||
:path => path
|
||||
)
|
||||
put_dynamic_obj_manifest(container, object, options)
|
||||
end
|
||||
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
module Fog
|
||||
module Storage
|
||||
class OpenStack
|
||||
class Real
|
||||
|
||||
# Create a new static large object manifest.
|
||||
#
|
||||
# A static large object is similar to a dynamic large object. Whereas a GET for a dynamic large object manifest
|
||||
# will stream segments based on the manifest's +X-Object-Manifest+ object name prefix, a static large object
|
||||
# manifest streams segments which are defined by the user within the manifest. Information about each segment is
|
||||
# provided in +segments+ as an Array of Hash objects, ordered in the sequence which the segments should be streamed.
|
||||
#
|
||||
# When the SLO manifest is received, each segment's +etag+ and +size_bytes+ will be verified.
|
||||
# The +etag+ for each segment is returned in the response to {#put_object}, but may also be calculated.
|
||||
# e.g. +Digest::MD5.hexdigest(segment_data)+
|
||||
#
|
||||
# The maximum number of segments for a static large object is 1000, and all segments (except the last) must be
|
||||
# at least 1 MiB in size. Unlike a dynamic large object, segments are not required to be in the same container.
|
||||
#
|
||||
# @example
|
||||
# segments = [
|
||||
# { :path => 'segments_container/first_segment',
|
||||
# :etag => 'md5 for first_segment',
|
||||
# :size_bytes => 'byte size of first_segment' },
|
||||
# { :path => 'segments_container/second_segment',
|
||||
# :etag => 'md5 for second_segment',
|
||||
# :size_bytes => 'byte size of second_segment' }
|
||||
# ]
|
||||
# put_static_obj_manifest('my_container', 'my_large_object', segments)
|
||||
#
|
||||
# @param container [String] Name for container where +object+ will be stored.
|
||||
# Should be < 256 bytes and must not contain '/'
|
||||
# @param object [String] Name for manifest object.
|
||||
# @param segments [Array<Hash>] Segment data for the object.
|
||||
# @param options [Hash] Config headers for +object+.
|
||||
#
|
||||
# @raise [Fog::Storage::OpenStack::NotFound] HTTP 404
|
||||
# @raise [Excon::Errors::BadRequest] HTTP 400
|
||||
# @raise [Excon::Errors::Unauthorized] HTTP 401
|
||||
# @raise [Excon::Errors::HTTPStatusError]
|
||||
#
|
||||
# @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html
|
||||
def put_static_obj_manifest(container, object, segments, options = {})
|
||||
request(
|
||||
:expects => 201,
|
||||
:method => 'PUT',
|
||||
:headers => options,
|
||||
:body => Fog::JSON.encode(segments),
|
||||
:path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}",
|
||||
:query => { 'multipart-manifest' => 'put' }
|
||||
)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -21,6 +21,8 @@ module Fog
|
|||
request :copy_object
|
||||
request :delete_container
|
||||
request :delete_object
|
||||
request :delete_multiple_objects
|
||||
request :delete_static_large_object
|
||||
request :get_container
|
||||
request :get_containers
|
||||
request :get_object
|
||||
|
|
@ -31,6 +33,8 @@ module Fog
|
|||
request :put_container
|
||||
request :put_object
|
||||
request :put_object_manifest
|
||||
request :put_dynamic_obj_manifest
|
||||
request :put_static_obj_manifest
|
||||
|
||||
class Mock
|
||||
|
||||
|
|
@ -145,6 +149,7 @@ module Fog
|
|||
response = @connection.request(params.merge({
|
||||
:headers => {
|
||||
'Content-Type' => 'application/json',
|
||||
'Accept' => 'application/json',
|
||||
'X-Auth-Token' => @auth_token
|
||||
}.merge!(params[:headers] || {}),
|
||||
:host => @host,
|
||||
|
|
|
|||
|
|
@ -1,106 +1,367 @@
|
|||
Shindo.tests('Fog::Storage[:openstack] | large object requests', ["openstack"]) do
|
||||
Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) do
|
||||
|
||||
unless Fog.mocking?
|
||||
@directory = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests')
|
||||
@directory = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests')
|
||||
@directory2 = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests2')
|
||||
@segments = {
|
||||
:a => {
|
||||
:container => @directory.identity,
|
||||
:name => 'fog_large_object/a',
|
||||
:data => 'a' * (1024**2 + 10),
|
||||
:size => 1024**2 + 10,
|
||||
:etag => 'c2e97007d59f0c19b850debdcb80cca5'
|
||||
},
|
||||
:b => {
|
||||
:container => @directory.identity,
|
||||
:name => 'fog_large_object/b',
|
||||
:data => 'b' * (1024**2 + 20),
|
||||
:size => 1024**2 + 20,
|
||||
:etag => 'd35f50622a1259daad75ff7d5512c7ef'
|
||||
},
|
||||
:c => {
|
||||
:container => @directory.identity,
|
||||
:name => 'fog_large_object2/a',
|
||||
:data => 'c' * (1024**2 + 30),
|
||||
:size => 1024**2 + 30,
|
||||
:etag => '901d3531a87d188041d4d5b43cb464c1'
|
||||
},
|
||||
:d => {
|
||||
:container => @directory2.identity,
|
||||
:name => 'fog_large_object2/b',
|
||||
:data => 'd' * (1024**2 + 40),
|
||||
:size => 1024**2 + 40,
|
||||
:etag => '350c0e00525198813920a157df185c8d'
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
tests('success') do
|
||||
|
||||
tests("#put_object('foglargeobjecttests', 'fog_large_object/1', ('x' * 4 * 1024 * 1024))").succeeds do
|
||||
tests('upload test segments').succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/1', ('x' * 4 * 1024 * 1024))
|
||||
|
||||
@segments.each_value do |segment|
|
||||
Fog::Storage[:openstack].put_object(segment[:container], segment[:name], segment[:data])
|
||||
end
|
||||
end
|
||||
|
||||
tests("#put_object('foglargeobjecttests', 'fog_large_object/2', ('x' * 2 * 1024 * 1024))").succeeds do
|
||||
tests('dynamic large object requests') do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/2', ('x' * 2 * 1024 * 1024))
|
||||
end
|
||||
|
||||
tests("#put_object('foglargeobjecttests', 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))
|
||||
end
|
||||
|
||||
tests("using default X-Object-Manifest header") do
|
||||
|
||||
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do
|
||||
pending if Fog.mocking?
|
||||
tests('#put_object_manifest alias').succeeds do
|
||||
Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object')
|
||||
end
|
||||
|
||||
tests("#get_object streams all segments matching the default prefix").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == ('x' * 7 * 1024 * 1024)
|
||||
tests('using default X-Object-Manifest header') do
|
||||
|
||||
tests('#put_dynamic_obj_manifest').succeeds do
|
||||
Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object')
|
||||
end
|
||||
|
||||
tests('#get_object streams all segments matching the default prefix').succeeds do
|
||||
expected = @segments[:a][:data] + @segments[:b][:data] + @segments[:c][:data]
|
||||
Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
|
||||
end
|
||||
|
||||
# When the manifest object name is equal to the segment prefix, OpenStack treats it as if it's the first segment.
|
||||
# So you must prepend the manifest object's Etag - Digest::MD5.hexdigest('')
|
||||
tests('#head_object returns Etag that includes manifest object in calculation').succeeds do
|
||||
etags = ['d41d8cd98f00b204e9800998ecf8427e', @segments[:a][:etag], @segments[:b][:etag], @segments[:c][:etag]]
|
||||
expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"2577f38428e895c50de6ea78ccc7da2a"\"
|
||||
Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests("#head_object returns Etag that includes manifest object in calculation").succeeds do
|
||||
pending if Fog.mocking?
|
||||
tests('specifying X-Object-Manifest segment prefix') do
|
||||
|
||||
tests('#put_dynamic_obj_manifest').succeeds do
|
||||
options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
|
||||
Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object', options)
|
||||
end
|
||||
|
||||
tests('#get_object streams segments only matching the specified prefix').succeeds do
|
||||
expected = @segments[:a][:data] + @segments[:b][:data]
|
||||
Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
|
||||
end
|
||||
|
||||
tests('#head_object returns Etag that does not include manifest object in calculation').succeeds do
|
||||
etags = [@segments[:a][:etag], @segments[:b][:etag]]
|
||||
expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"0f035ed3cc38aa0ef46dda3478fad44d"\"
|
||||
Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests('storing manifest in a different container than the segments') do
|
||||
|
||||
tests('#put_dynamic_obj_manifest').succeeds do
|
||||
options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
|
||||
Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory2.identity, 'fog_large_object', options)
|
||||
end
|
||||
|
||||
tests('#get_object').succeeds do
|
||||
expected = @segments[:a][:data] + @segments[:b][:data]
|
||||
Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected
|
||||
end
|
||||
|
||||
etags = []
|
||||
# When the manifest object name is equal to the prefix, OpenStack treats it as if it's the first segment.
|
||||
etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e"
|
||||
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
|
||||
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
|
||||
etags << Digest::MD5.hexdigest('x' * 1 * 1024 * 1024) # => "b561f87202d04959e37588ee05cf5b10"
|
||||
expected = Digest::MD5.hexdigest(etags.join) # => "42e92048bd2c8085e7072b0b55fd76ab"
|
||||
actual = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag']
|
||||
actual.gsub('"', '') == expected # actual is returned in quotes "\"42e92048bd2c8085e7072b0b55fd76abu"\"
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests("specifying X-Object-Manifest segment prefix") do
|
||||
tests('static large object requests') do
|
||||
pending if Fog.mocking?
|
||||
|
||||
tests('single container') do
|
||||
|
||||
tests('#put_static_obj_manifest').succeeds do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
|
||||
:etag => @segments[:a][:etag],
|
||||
:size_bytes => @segments[:a][:size] },
|
||||
{ :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
|
||||
:etag => @segments[:c][:etag],
|
||||
:size_bytes => @segments[:c][:size] }
|
||||
]
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
|
||||
end
|
||||
|
||||
tests('#head_object') do
|
||||
etags = [@segments[:a][:etag], @segments[:c][:etag]]
|
||||
etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"ad7e633a12e8a4915b45e6dd1d4b0b4b\""
|
||||
content_length = (@segments[:a][:size] + @segments[:c][:size]).to_s
|
||||
response = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object')
|
||||
|
||||
returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
|
||||
returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
|
||||
returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
|
||||
end
|
||||
|
||||
tests('#get_object').succeeds do
|
||||
expected = @segments[:a][:data] + @segments[:c][:data]
|
||||
Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object') do
|
||||
expected = {
|
||||
'Number Not Found' => 0,
|
||||
'Response Status' => '200 OK',
|
||||
'Errors' => [],
|
||||
'Number Deleted' => 3,
|
||||
'Response Body' => ''
|
||||
}
|
||||
returns(expected, 'deletes manifest and segments') do
|
||||
Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body
|
||||
end
|
||||
end
|
||||
|
||||
tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/')").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
|
||||
end
|
||||
|
||||
tests("#get_object streams segments only matching the specified prefix").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
|
||||
tests('multiple containers') do
|
||||
|
||||
tests('#put_static_obj_manifest').succeeds do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
|
||||
:etag => @segments[:b][:etag],
|
||||
:size_bytes => @segments[:b][:size] },
|
||||
{ :path => "#{ @segments[:d][:container] }/#{ @segments[:d][:name] }",
|
||||
:etag => @segments[:d][:etag],
|
||||
:size_bytes => @segments[:d][:size] }
|
||||
]
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory2.identity, 'fog_large_object', segments)
|
||||
end
|
||||
|
||||
tests('#head_object') do
|
||||
etags = [@segments[:b][:etag], @segments[:d][:etag]]
|
||||
etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"9801a4cc4472896a1e975d03f0d2c3f8\""
|
||||
content_length = (@segments[:b][:size] + @segments[:d][:size]).to_s
|
||||
response = Fog::Storage[:openstack].head_object(@directory2.identity, 'fog_large_object')
|
||||
|
||||
returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
|
||||
returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
|
||||
returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
|
||||
end
|
||||
|
||||
tests('#get_object').succeeds do
|
||||
expected = @segments[:b][:data] + @segments[:d][:data]
|
||||
Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object') do
|
||||
expected = {
|
||||
'Number Not Found' => 0,
|
||||
'Response Status' => '200 OK',
|
||||
'Errors' => [],
|
||||
'Number Deleted' => 3,
|
||||
'Response Body' => ''
|
||||
}
|
||||
returns(expected, 'deletes manifest and segments') do
|
||||
Fog::Storage[:openstack].delete_static_large_object(@directory2.identity, 'fog_large_object').body
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests("#head_object returns Etag that does not include manifest object in calculation").succeeds do
|
||||
pending if Fog.mocking?
|
||||
|
||||
etags = []
|
||||
etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355"
|
||||
etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51"
|
||||
expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84"
|
||||
actual = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag']
|
||||
actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\"
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests("storing manifest object in a different container than the segments") do
|
||||
|
||||
tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/'})").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].put_object_manifest(@directory2.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"})
|
||||
end
|
||||
|
||||
tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
unless Fog.mocking?
|
||||
['fog_large_object', 'fog_large_object/1', 'fog_large_object/2', 'fog_large_object2/1'].each do |key|
|
||||
@directory.files.new(:key => key).destroy
|
||||
end
|
||||
@directory2.files.new(:key => 'fog_large_object').destroy
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests('failure') do
|
||||
|
||||
tests("put_object_manifest")
|
||||
tests('dynamic large object requests') do
|
||||
pending if Fog.mocking?
|
||||
|
||||
tests('#put_dynamic_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do
|
||||
Fog::Storage[:openstack].put_dynamic_obj_manifest('fognoncontainer', 'fog_large_object')
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
tests('static large object requests') do
|
||||
pending if Fog.mocking?
|
||||
|
||||
tests('upload test segments').succeeds do
|
||||
Fog::Storage[:openstack].put_object(@segments[:a][:container], @segments[:a][:name], @segments[:a][:data])
|
||||
Fog::Storage[:openstack].put_object(@segments[:b][:container], @segments[:b][:name], @segments[:b][:data])
|
||||
end
|
||||
|
||||
tests('#put_static_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do
|
||||
Fog::Storage[:openstack].put_static_obj_manifest('fognoncontainer', 'fog_large_object', [])
|
||||
end
|
||||
|
||||
tests('#put_static_obj_manifest with missing object') do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
|
||||
:etag => @segments[:c][:etag],
|
||||
:size_bytes => @segments[:c][:size] }
|
||||
]
|
||||
expected = { 'Errors' => [[segments[0][:path], '404 Not Found']] }
|
||||
|
||||
error = nil
|
||||
begin
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
|
||||
rescue => err
|
||||
error = err
|
||||
end
|
||||
|
||||
raises(Excon::Errors::BadRequest) do
|
||||
raise error if error
|
||||
end
|
||||
|
||||
returns(expected, 'returns error information') do
|
||||
Fog::JSON.decode(error.response.body)
|
||||
end
|
||||
end
|
||||
|
||||
tests('#put_static_obj_manifest with invalid etag') do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
|
||||
:etag => @segments[:b][:etag],
|
||||
:size_bytes => @segments[:a][:size] }
|
||||
]
|
||||
expected = { 'Errors' => [[segments[0][:path], 'Etag Mismatch']] }
|
||||
|
||||
error = nil
|
||||
begin
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
|
||||
rescue => err
|
||||
error = err
|
||||
end
|
||||
|
||||
raises(Excon::Errors::BadRequest) do
|
||||
raise error if error
|
||||
end
|
||||
|
||||
returns(expected, 'returns error information') do
|
||||
Fog::JSON.decode(error.response.body)
|
||||
end
|
||||
end
|
||||
|
||||
tests('#put_static_obj_manifest with invalid byte_size') do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
|
||||
:etag => @segments[:a][:etag],
|
||||
:size_bytes => @segments[:b][:size] }
|
||||
]
|
||||
expected = { 'Errors' => [[segments[0][:path], 'Size Mismatch']] }
|
||||
|
||||
error = nil
|
||||
begin
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
|
||||
rescue => err
|
||||
error = err
|
||||
end
|
||||
|
||||
raises(Excon::Errors::BadRequest) do
|
||||
raise error if error
|
||||
end
|
||||
|
||||
returns(expected, 'returns error information') do
|
||||
Fog::JSON.decode(error.response.body)
|
||||
end
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object with missing container') do
|
||||
expected = {
|
||||
'Number Not Found' => 1,
|
||||
'Response Status' => '200 OK',
|
||||
'Errors' => [],
|
||||
'Number Deleted' => 0,
|
||||
'Response Body' => ''
|
||||
}
|
||||
|
||||
returns(expected, 'reports missing object') do
|
||||
Fog::Storage[:openstack].delete_static_large_object('fognoncontainer', 'fog_large_object').body
|
||||
end
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object with missing manifest') do
|
||||
expected = {
|
||||
'Number Not Found' => 1,
|
||||
'Response Status' => '200 OK',
|
||||
'Errors' => [],
|
||||
'Number Deleted' => 0,
|
||||
'Response Body' => ''
|
||||
}
|
||||
|
||||
returns(expected, 'reports missing manifest') do
|
||||
Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_non_object').body
|
||||
end
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object with missing segment') do
|
||||
|
||||
tests('#put_static_obj_manifest for segments :a and :b').succeeds do
|
||||
segments = [
|
||||
{ :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
|
||||
:etag => @segments[:a][:etag],
|
||||
:size_bytes => @segments[:a][:size] },
|
||||
{ :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
|
||||
:etag => @segments[:b][:etag],
|
||||
:size_bytes => @segments[:b][:size] }
|
||||
]
|
||||
Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
|
||||
end
|
||||
|
||||
tests('#delete_object segment :b').succeeds do
|
||||
Fog::Storage[:openstack].delete_object(@segments[:b][:container], @segments[:b][:name])
|
||||
end
|
||||
|
||||
tests('#delete_static_large_object') do
|
||||
expected = {
|
||||
'Number Not Found' => 1,
|
||||
'Response Status' => '200 OK',
|
||||
'Errors' => [],
|
||||
'Number Deleted' => 2,
|
||||
'Response Body' => ''
|
||||
}
|
||||
returns(expected, 'deletes manifest and segment :a, and reports missing segment :b') do
|
||||
Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -17,18 +17,18 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do
|
|||
Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
|
||||
end
|
||||
|
||||
tests("#get_object('fogobjectests', 'fog_object')").returns(lorem_file.read) do
|
||||
tests("#get_object('fogobjectests', 'fog_object')").succeeds do
|
||||
pending if Fog.mocking?
|
||||
Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object').body
|
||||
Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object').body == lorem_file.read
|
||||
end
|
||||
|
||||
tests("#get_object('fogobjecttests', 'fog_object', &block)").returns(lorem_file.read) do
|
||||
tests("#get_object('fogobjecttests', 'fog_object', &block)").succeeds do
|
||||
pending if Fog.mocking?
|
||||
data = ''
|
||||
Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object') do |chunk, remaining_bytes, total_bytes|
|
||||
data << chunk
|
||||
end
|
||||
data
|
||||
data == lorem_file.read
|
||||
end
|
||||
|
||||
tests("#head_object('fogobjectests', 'fog_object')").succeeds do
|
||||
|
|
@ -65,6 +65,30 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do
|
|||
end
|
||||
end
|
||||
|
||||
tests('#delete_multiple_objects') do
|
||||
pending if Fog.mocking?
|
||||
|
||||
Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
|
||||
Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object2', lorem_file)
|
||||
Fog::Storage[:openstack].directories.create(:key => 'fogobjecttests2')
|
||||
Fog::Storage[:openstack].put_object('fogobjecttests2', 'fog_object', lorem_file)
|
||||
|
||||
expected = {
|
||||
"Number Not Found" => 0,
|
||||
"Response Status" => "200 OK",
|
||||
"Errors" => [],
|
||||
"Number Deleted" => 2,
|
||||
"Response Body" => ""
|
||||
}
|
||||
|
||||
returns(expected, 'deletes multiple objects') do
|
||||
Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_object', 'fog_object2']).body
|
||||
end
|
||||
|
||||
returns(expected, 'deletes object and container') do
|
||||
Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests2/fog_object', 'fogobjecttests2']).body
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
|
|
@ -100,6 +124,42 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do
|
|||
Fog::Storage[:openstack].delete_object('fognoncontainer', 'fog_non_object')
|
||||
end
|
||||
|
||||
tests('#delete_multiple_objects') do
|
||||
pending if Fog.mocking?
|
||||
|
||||
expected = {
|
||||
"Number Not Found" => 2,
|
||||
"Response Status" => "200 OK",
|
||||
"Errors" => [],
|
||||
"Number Deleted" => 0,
|
||||
"Response Body" => ""
|
||||
}
|
||||
|
||||
returns(expected, 'reports missing objects') do
|
||||
Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_non_object', 'fog_non_object2']).body
|
||||
end
|
||||
|
||||
returns(expected, 'reports missing container') do
|
||||
Fog::Storage[:openstack].delete_multiple_objects('fognoncontainer', ['fog_non_object', 'fog_non_object2']).body
|
||||
end
|
||||
|
||||
tests('deleting non-empty container') do
|
||||
Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
|
||||
|
||||
expected = {
|
||||
"Number Not Found" => 0,
|
||||
"Response Status" => "400 Bad Request",
|
||||
"Errors" => [['fogobjecttests', '409 Conflict']],
|
||||
"Number Deleted" => 1,
|
||||
"Response Body" => ""
|
||||
}
|
||||
|
||||
returns(expected, 'deletes object but not container') do
|
||||
Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests', 'fogobjecttests/fog_object']).body
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
unless Fog.mocking?
|
||||
|
|
|
|||
Loading…
Reference in a new issue