From 890049968ac505b5b43922beec7114db2335af0d Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 2 Jun 2013 17:41:57 -0400 Subject: [PATCH] [rackspace|storage] allow headers to be specified for object manifest Allows additional headers to be set, as well as overriding the default X-Object-Manifest header. --- .../requests/storage/put_object_manifest.rb | 33 +++-- .../requests/storage/large_object_tests.rb | 120 +++++++++--------- 2 files changed, 82 insertions(+), 71 deletions(-) diff --git a/lib/fog/rackspace/requests/storage/put_object_manifest.rb b/lib/fog/rackspace/requests/storage/put_object_manifest.rb index 45d06d0b7..5ed20a1ed 100644 --- a/lib/fog/rackspace/requests/storage/put_object_manifest.rb +++ b/lib/fog/rackspace/requests/storage/put_object_manifest.rb @@ -5,22 +5,33 @@ module Fog # Create a new manifest object # - # ==== Parameters - # * container<~String> - Name for container, should be < 256 bytes and must not contain '/' - # * object<~String> - Name for manifest object - # * options<~Hash>: - # * 'segments_container'<~String> - Name of container where segments are stored. +container+ will be used if not given. - # * 'segments_prefix'<~String> - Name prefix used for segmented objects. +object+ will be used if not given. - # @raise [Fog::Storage::Rackspace::NotFound] - HTTP 404 - # @raise [Fog::Storage::Rackspace::BadRequest] - HTTP 400 - # @raise [Fog::Storage::Rackspace::InternalServerError] - HTTP 500 + # Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("/") + # for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix. + # Segments must sort in the order they should be concatenated. Note that any future objects stored in the container + # along with the segments that match the prefix will be included when retrieving the manifest object. + # + # All segments must be stored in the same container, but may be in a different container than the manifest object. + # The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+ + # to specify the prefix and/or the container where segments were stored. + # If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::Rackspace.escape}). + # + # @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/' + # @param object [String] Name for manifest object. + # @param options [Hash] Config headers for +object+. + # @option options [String] 'X-Object-Manifest' ("container/object") "/" for segment objects. + # + # @raise [Fog::Storage::Rackspace::NotFound] HTTP 404 + # @raise [Fog::Storage::Rackspace::BadRequest] HTTP 400 + # @raise [Fog::Storage::Rackspace::InternalServerError] HTTP 500 # @raise [Fog::Storage::Rackspace::ServiceError] + # + # @see http://docs.rackspace.com/files/api/v1/cf-devguide/content/Large_Object_Creation-d1e2019.html def put_object_manifest(container, object, options = {}) path = "#{Fog::Rackspace.escape(container)}/#{Fog::Rackspace.escape(object)}" - prefix = "#{Fog::Rackspace.escape(options['segments_container'] || container)}/#{Fog::Rackspace.escape(options['segments_prefix'] || object)}" + headers = {'X-Object-Manifest' => path}.merge(options) request( :expects => 201, - :headers => {'X-Object-Manifest' => prefix}, + :headers => headers, :method => 'PUT', :path => path ) diff --git a/tests/rackspace/requests/storage/large_object_tests.rb b/tests/rackspace/requests/storage/large_object_tests.rb index dfaac6a6a..e4b6be1af 100644 --- a/tests/rackspace/requests/storage/large_object_tests.rb +++ b/tests/rackspace/requests/storage/large_object_tests.rb @@ -17,79 +17,80 @@ Shindo.tests('Fog::Storage[:rackspace] | large object requests', ["rackspace"]) Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object/2', ('x' * 2 * 1024 * 1024)) end - tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do + tests("#put_object('foglargeobjecttests', 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))").succeeds do pending if Fog.mocking? - Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object') + Fog::Storage[:rackspace].put_object(@directory.identity, 'fog_large_object2/1', ('x' * 1 * 1024 * 1024)) end - tests("#get_object('foglargeobjecttests', 'fog_large_object').body").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) + tests("using default X-Object-Manifest header") do + + tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object') + end + + tests("#get_object streams all segments matching the default prefix").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 7 * 1024 * 1024) + end + + tests("#head_object returns Etag that includes manifest object in calculation").succeeds do + pending if Fog.mocking? + + etags = [] + # When the manifest object name is equal to the prefix, OpenStack treats it as if it's the first segment. + etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e" + etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" + etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" + etags << Digest::MD5.hexdigest('x' * 1 * 1024 * 1024) # => "b561f87202d04959e37588ee05cf5b10" + expected = Digest::MD5.hexdigest(etags.join) # => "42e92048bd2c8085e7072b0b55fd76ab" + actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag'] + actual.gsub('"', '') == expected # actual is returned in quotes "\"42e92048bd2c8085e7072b0b55fd76abu"\" + end + end - tests("returns manifest Etag, computed including manifest file").succeeds do - pending if Fog.mocking? + tests("specifying X-Object-Manifest segment prefix") do + + tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/')").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"}) + end + + tests("#get_object streams segments only matching the specified prefix").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) + end + + tests("#head_object returns Etag that does not include manifest object in calculation").succeeds do + pending if Fog.mocking? + + etags = [] + etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" + etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" + expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84" + actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag'] + actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\" + end - etags = [] - # When the manifest object name is equal to the prefix for the segments, - # OpenStack treats it as if it's the first segment. - etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e" - etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" - etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" - expected = Digest::MD5.hexdigest(etags.join) # => "2537ea40a5a8cac247a912906cb62fc2" - actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'fog_large_object').headers['Etag'] - actual.gsub('"', '') == expected # actual is returned in quotes "\"2537ea40a5a8cac247a912906cb62fc2"\" end - tests("#delete_object('foglargeobjecttests', 'fog_large_object')").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].delete_object('foglargeobjecttests', 'fog_large_object') - end + tests("storing manifest object in a different container than the segments") do - tests("#put_object_manifest('foglargeobjecttests', 'large_object_manifest', {'segments_prefix' => 'fog_large_object'})").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].put_object_manifest(@directory.identity, 'large_object_manifest', {'segments_prefix' => 'fog_large_object'}) - end + tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/'})").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"}) + end - tests("#get_object('foglargeobjecttests', 'large_object_manifest').body").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].get_object(@directory.identity, 'large_object_manifest').body == ('x' * 6 * 1024 * 1024) - end + tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do + pending if Fog.mocking? + Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) + end - tests("returns manifest Etag, computed without manifest file").succeeds do - pending if Fog.mocking? - - etags = [] - etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" - etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" - expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84" - actual = Fog::Storage[:rackspace].head_object(@directory.identity, 'large_object_manifest').headers['Etag'] - actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\" - end - - tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'segments_container' => 'foglargeobjecttests', 'segments_prefix' => 'fog_large_object'})").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].put_object_manifest(@directory2.identity, 'fog_large_object', {'segments_container' => @directory.identity, 'segments_prefix' => 'fog_large_object'}) - end - - tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do - pending if Fog.mocking? - Fog::Storage[:rackspace].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) - end - - tests("returns manifest Etag, computed without manifest file").succeeds do - pending if Fog.mocking? - - etags = [] - etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" - etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" - expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84" - actual = Fog::Storage[:rackspace].head_object(@directory2.identity, 'fog_large_object').headers['Etag'] - actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\" end unless Fog.mocking? - ['large_object_manifest', 'fog_large_object/1', 'fog_large_object/2'].each do |key| + ['fog_large_object', 'fog_large_object/1', 'fog_large_object/2', 'fog_large_object2/1'].each do |key| @directory.files.new(:key => key).destroy end @directory2.files.new(:key => 'fog_large_object').destroy @@ -107,5 +108,4 @@ Shindo.tests('Fog::Storage[:rackspace] | large object requests', ["rackspace"]) @directory.destroy @directory2.destroy end - end