From c97c5662779b8bc64ae0fd645d973e8a7906aa60 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 20:37:51 -0400 Subject: [PATCH 1/6] [openstack|storage] add default Accept header --- lib/fog/openstack/storage.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index 4fc2b7d81..8aaecbbe6 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -145,6 +145,7 @@ module Fog response = @connection.request(params.merge({ :headers => { 'Content-Type' => 'application/json', + 'Accept' => 'application/json', 'X-Auth-Token' => @auth_token }.merge!(params[:headers] || {}), :host => @host, From 912760e735fb08aebd57698fbbbf9cfeddfe3d01 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 20:37:56 -0400 Subject: [PATCH 2/6] [openstack|storage] add #delete_multiple_objects --- .../storage/delete_multiple_objects.rb | 64 +++++++++++++++++ lib/fog/openstack/storage.rb | 1 + .../requests/storage/object_tests.rb | 68 +++++++++++++++++-- 3 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 lib/fog/openstack/requests/storage/delete_multiple_objects.rb diff --git a/lib/fog/openstack/requests/storage/delete_multiple_objects.rb b/lib/fog/openstack/requests/storage/delete_multiple_objects.rb new file mode 100644 index 000000000..86b20cc1b --- /dev/null +++ b/lib/fog/openstack/requests/storage/delete_multiple_objects.rb @@ -0,0 +1,64 @@ +module Fog + module Storage + class OpenStack + class Real + + # Deletes multiple objects or containers with a single request. + # + # To delete objects from a single container, +container+ may be provided + # and +object_names+ should be an Array of object names within the container. + # + # To delete objects from multiple containers or delete containers, + # +container+ should be +nil+ and all +object_names+ should be prefixed with a container name. + # + # Containers must be empty when deleted. +object_names+ are processed in the order given, + # so objects within a container should be listed first to empty the container. + # + # Up to 10,000 objects may be deleted in a single request. + # The server will respond with +200 OK+ for all requests. + # +response.body+ must be inspected for actual results. + # + # @example Delete objects from a container + # object_names = ['object', 'another/object'] + # conn.delete_multiple_objects('my_container', object_names) + # + # @example Delete objects from multiple containers + # object_names = ['container_a/object', 'container_b/object'] + # conn.delete_multiple_objects(nil, object_names) + # + # @example Delete a container and all it's objects + # object_names = ['my_container/object_a', 'my_container/object_b', 'my_container'] + # conn.delete_multiple_objects(nil, object_names) + # + # @param container [String,nil] Name of container. + # @param object_names [Array] Object names to be deleted. + # @param options [Hash] Additional request headers. + # + # @return [Excon::Response] + # * body [Hash] - Results of the operation. + # * "Number Not Found" [Integer] - Number of missing objects or containers. + # * "Response Status" [String] - Response code for the subrequest of the last failed operation. + # * "Errors" [Array] + # * object_name [String] - Object that generated an error when the delete was attempted. + # * response_status [String] - Response status from the subrequest for object_name. + # * "Number Deleted" [Integer] - Number of objects or containers deleted. + # * "Response Body" [String] - Response body for "Response Status". + def delete_multiple_objects(container, object_names, options = {}) + body = object_names.map do |name| + object_name = container ? "#{ container }/#{ name }" : name + URI.encode(object_name) + end.join("\n") + + request( + :expects => 200, + :method => 'DELETE', + :headers => options.merge('Content-Type' => 'text/plain'), + :body => body, + :query => { 'bulk-delete' => true } + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index 8aaecbbe6..44badaadd 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -21,6 +21,7 @@ module Fog request :copy_object request :delete_container request :delete_object + request :delete_multiple_objects request :get_container request :get_containers request :get_object diff --git a/tests/openstack/requests/storage/object_tests.rb b/tests/openstack/requests/storage/object_tests.rb index dcd9ff65a..ac4f87cca 100644 --- a/tests/openstack/requests/storage/object_tests.rb +++ b/tests/openstack/requests/storage/object_tests.rb @@ -17,18 +17,18 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file) end - tests("#get_object('fogobjectests', 'fog_object')").returns(lorem_file.read) do + tests("#get_object('fogobjectests', 'fog_object')").succeeds do pending if Fog.mocking? - Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object').body + Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object').body == lorem_file.read end - tests("#get_object('fogobjecttests', 'fog_object', &block)").returns(lorem_file.read) do + tests("#get_object('fogobjecttests', 'fog_object', &block)").succeeds do pending if Fog.mocking? data = '' Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object') do |chunk, remaining_bytes, total_bytes| data << chunk end - data + data == lorem_file.read end tests("#head_object('fogobjectests', 'fog_object')").succeeds do @@ -65,6 +65,30 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do end end + tests('#delete_multiple_objects') do + pending if Fog.mocking? + + Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file) + Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object2', lorem_file) + Fog::Storage[:openstack].directories.create(:key => 'fogobjecttests2') + Fog::Storage[:openstack].put_object('fogobjecttests2', 'fog_object', lorem_file) + + expected = { + "Number Not Found" => 0, + "Response Status" => "200 OK", + "Errors" => [], + "Number Deleted" => 2, + "Response Body" => "" + } + + returns(expected, 'deletes multiple objects') do + Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_object', 'fog_object2']).body + end + + returns(expected, 'deletes object and container') do + Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests2/fog_object', 'fogobjecttests2']).body + end + end end @@ -100,6 +124,42 @@ Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do Fog::Storage[:openstack].delete_object('fognoncontainer', 'fog_non_object') end + tests('#delete_multiple_objects') do + pending if Fog.mocking? + + expected = { + "Number Not Found" => 2, + "Response Status" => "200 OK", + "Errors" => [], + "Number Deleted" => 0, + "Response Body" => "" + } + + returns(expected, 'reports missing objects') do + Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_non_object', 'fog_non_object2']).body + end + + returns(expected, 'reports missing container') do + Fog::Storage[:openstack].delete_multiple_objects('fognoncontainer', ['fog_non_object', 'fog_non_object2']).body + end + + tests('deleting non-empty container') do + Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file) + + expected = { + "Number Not Found" => 0, + "Response Status" => "400 Bad Request", + "Errors" => [['fogobjecttests', '409 Conflict']], + "Number Deleted" => 1, + "Response Body" => "" + } + + returns(expected, 'deletes object but not container') do + Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests', 'fogobjecttests/fog_object']).body + end + end + end + end unless Fog.mocking? From 7cf00f3251f51509ea856443ada4e5fda0b52356 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 20:37:59 -0400 Subject: [PATCH 3/6] [openstack|storage] patch #delete_multiple_objects for Swift v1.8 This commit addresses an issue where the Content-Type header is not being set correctly in the response. This has been fixed in Swift v1.9. openstack/swift@ad24cde120efe73fe5a9b866f74c7d3c766aa4cb --- .../requests/storage/delete_multiple_objects.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/fog/openstack/requests/storage/delete_multiple_objects.rb b/lib/fog/openstack/requests/storage/delete_multiple_objects.rb index 86b20cc1b..c623419ce 100644 --- a/lib/fog/openstack/requests/storage/delete_multiple_objects.rb +++ b/lib/fog/openstack/requests/storage/delete_multiple_objects.rb @@ -49,13 +49,16 @@ module Fog URI.encode(object_name) end.join("\n") - request( + response = request({ :expects => 200, :method => 'DELETE', - :headers => options.merge('Content-Type' => 'text/plain'), + :headers => options.merge('Content-Type' => 'text/plain', + 'Accept' => 'application/json'), :body => body, :query => { 'bulk-delete' => true } - ) + }, false) + response.body = Fog::JSON.decode(response.body) + response end end From d3ac285625d2a78c0c2cb441fd3179ae6421b894 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 21:49:40 -0400 Subject: [PATCH 4/6] [openstack|storage] add methods for SLO support Adds #put_static_obj_manifest and #delete_static_large_object methods. --- .../storage/delete_static_large_object.rb | 40 ++ .../storage/put_static_obj_manifest.rb | 57 +++ lib/fog/openstack/storage.rb | 2 + .../requests/storage/large_object_tests.rb | 411 ++++++++++++++---- 4 files changed, 433 insertions(+), 77 deletions(-) create mode 100644 lib/fog/openstack/requests/storage/delete_static_large_object.rb create mode 100644 lib/fog/openstack/requests/storage/put_static_obj_manifest.rb diff --git a/lib/fog/openstack/requests/storage/delete_static_large_object.rb b/lib/fog/openstack/requests/storage/delete_static_large_object.rb new file mode 100644 index 000000000..07c737b0b --- /dev/null +++ b/lib/fog/openstack/requests/storage/delete_static_large_object.rb @@ -0,0 +1,40 @@ +module Fog + module Storage + class OpenStack + class Real + + # Delete a static large object. + # + # Deletes the SLO manifest +object+ and all segments that it references. + # The server will respond with +200 OK+ for all requests. + # +response.body+ must be inspected for actual results. + # + # @param container [String] Name of container. + # @param object [String] Name of the SLO manifest object. + # @param options [Hash] Additional request headers. + # + # @return [Excon::Response] + # * body [Hash] - Results of the operation. + # * "Number Not Found" [Integer] - Number of missing segments. + # * "Response Status" [String] - Response code for the subrequest of the last failed operation. + # * "Errors" [Array] + # * object_name [String] - Object that generated an error when the delete was attempted. + # * response_status [String] - Response status from the subrequest for object_name. + # * "Number Deleted" [Integer] - Number of segments deleted. + # * "Response Body" [String] - Response body for Response Status. + # + # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html + def delete_static_large_object(container, object, options = {}) + request( + :expects => 200, + :method => 'DELETE', + :headers => options.merge('Content-Type' => 'text/plain'), + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}", + :query => { 'multipart-manifest' => 'delete' } + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/put_static_obj_manifest.rb b/lib/fog/openstack/requests/storage/put_static_obj_manifest.rb new file mode 100644 index 000000000..a1c0b1f0a --- /dev/null +++ b/lib/fog/openstack/requests/storage/put_static_obj_manifest.rb @@ -0,0 +1,57 @@ +module Fog + module Storage + class OpenStack + class Real + + # Create a new static large object manifest. + # + # A static large object is similar to a dynamic large object. Whereas a GET for a dynamic large object manifest + # will stream segments based on the manifest's +X-Object-Manifest+ object name prefix, a static large object + # manifest streams segments which are defined by the user within the manifest. Information about each segment is + # provided in +segments+ as an Array of Hash objects, ordered in the sequence which the segments should be streamed. + # + # When the SLO manifest is received, each segment's +etag+ and +size_bytes+ will be verified. + # The +etag+ for each segment is returned in the response to {#put_object}, but may also be calculated. + # e.g. +Digest::MD5.hexdigest(segment_data)+ + # + # The maximum number of segments for a static large object is 1000, and all segments (except the last) must be + # at least 1 MiB in size. Unlike a dynamic large object, segments are not required to be in the same container. + # + # @example + # segments = [ + # { :path => 'segments_container/first_segment', + # :etag => 'md5 for first_segment', + # :size_bytes => 'byte size of first_segment' }, + # { :path => 'segments_container/second_segment', + # :etag => 'md5 for second_segment', + # :size_bytes => 'byte size of second_segment' } + # ] + # put_static_obj_manifest('my_container', 'my_large_object', segments) + # + # @param container [String] Name for container where +object+ will be stored. + # Should be < 256 bytes and must not contain '/' + # @param object [String] Name for manifest object. + # @param segments [Array] Segment data for the object. + # @param options [Hash] Config headers for +object+. + # + # @raise [Fog::Storage::OpenStack::NotFound] HTTP 404 + # @raise [Excon::Errors::BadRequest] HTTP 400 + # @raise [Excon::Errors::Unauthorized] HTTP 401 + # @raise [Excon::Errors::HTTPStatusError] + # + # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html + def put_static_obj_manifest(container, object, segments, options = {}) + request( + :expects => 201, + :method => 'PUT', + :headers => options, + :body => Fog::JSON.encode(segments), + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}", + :query => { 'multipart-manifest' => 'put' } + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index 44badaadd..ec79e9ac5 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -22,6 +22,7 @@ module Fog request :delete_container request :delete_object request :delete_multiple_objects + request :delete_static_large_object request :get_container request :get_containers request :get_object @@ -32,6 +33,7 @@ module Fog request :put_container request :put_object request :put_object_manifest + request :put_static_obj_manifest class Mock diff --git a/tests/openstack/requests/storage/large_object_tests.rb b/tests/openstack/requests/storage/large_object_tests.rb index 577ee2bee..12c0a5649 100644 --- a/tests/openstack/requests/storage/large_object_tests.rb +++ b/tests/openstack/requests/storage/large_object_tests.rb @@ -1,106 +1,363 @@ -Shindo.tests('Fog::Storage[:openstack] | large object requests', ["openstack"]) do +Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) do unless Fog.mocking? - @directory = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests') + @directory = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests') @directory2 = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests2') + @segments = { + :a => { + :container => @directory.identity, + :name => 'fog_large_object/a', + :data => 'a' * (1024**2 + 10), + :size => 1024**2 + 10, + :etag => 'c2e97007d59f0c19b850debdcb80cca5' + }, + :b => { + :container => @directory.identity, + :name => 'fog_large_object/b', + :data => 'b' * (1024**2 + 20), + :size => 1024**2 + 20, + :etag => 'd35f50622a1259daad75ff7d5512c7ef' + }, + :c => { + :container => @directory.identity, + :name => 'fog_large_object2/a', + :data => 'c' * (1024**2 + 30), + :size => 1024**2 + 30, + :etag => '901d3531a87d188041d4d5b43cb464c1' + }, + :d => { + :container => @directory2.identity, + :name => 'fog_large_object2/b', + :data => 'd' * (1024**2 + 40), + :size => 1024**2 + 40, + :etag => '350c0e00525198813920a157df185c8d' + } + } end tests('success') do - tests("#put_object('foglargeobjecttests', 'fog_large_object/1', ('x' * 4 * 1024 * 1024))").succeeds do + tests('upload test segments').succeeds do pending if Fog.mocking? - Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/1', ('x' * 4 * 1024 * 1024)) + + @segments.each_value do |segment| + Fog::Storage[:openstack].put_object(segment[:container], segment[:name], segment[:data]) + end end - tests("#put_object('foglargeobjecttests', 'fog_large_object/2', ('x' * 2 * 1024 * 1024))").succeeds do + tests('dynamic large object requests') do pending if Fog.mocking? - Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/2', ('x' * 2 * 1024 * 1024)) + + tests('using default X-Object-Manifest header') do + + tests('#put_object_manifest').succeeds do + Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object') + end + + tests('#get_object streams all segments matching the default prefix').succeeds do + expected = @segments[:a][:data] + @segments[:b][:data] + @segments[:c][:data] + Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected + end + + # When the manifest object name is equal to the segment prefix, OpenStack treats it as if it's the first segment. + # So you must prepend the manifest object's Etag - Digest::MD5.hexdigest('') + tests('#head_object returns Etag that includes manifest object in calculation').succeeds do + etags = ['d41d8cd98f00b204e9800998ecf8427e', @segments[:a][:etag], @segments[:b][:etag], @segments[:c][:etag]] + expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"2577f38428e895c50de6ea78ccc7da2a"\" + Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected + end + + end + + tests('specifying X-Object-Manifest segment prefix') do + + tests('#put_object_manifest').succeeds do + options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" } + Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object', options) + end + + tests('#get_object streams segments only matching the specified prefix').succeeds do + expected = @segments[:a][:data] + @segments[:b][:data] + Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected + end + + tests('#head_object returns Etag that does not include manifest object in calculation').succeeds do + etags = [@segments[:a][:etag], @segments[:b][:etag]] + expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"0f035ed3cc38aa0ef46dda3478fad44d"\" + Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected + end + + end + + tests('storing manifest in a different container than the segments') do + + tests('#put_object_manifest').succeeds do + options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" } + Fog::Storage[:openstack].put_object_manifest(@directory2.identity, 'fog_large_object', options) + end + + tests('#get_object').succeeds do + expected = @segments[:a][:data] + @segments[:b][:data] + Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected + end + + end + end - tests("#put_object('foglargeobjecttests', 'fog_large_object2/1', ('x' * 1 * 1024 * 1024))").succeeds do + tests('static large object requests') do pending if Fog.mocking? - Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object2/1', ('x' * 1 * 1024 * 1024)) - end - tests("using default X-Object-Manifest header") do + tests('single container') do + + tests('#put_static_obj_manifest').succeeds do + segments = [ + { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }", + :etag => @segments[:a][:etag], + :size_bytes => @segments[:a][:size] }, + { :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }", + :etag => @segments[:c][:etag], + :size_bytes => @segments[:c][:size] } + ] + Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments) + end + + tests('#head_object') do + etags = [@segments[:a][:etag], @segments[:c][:etag]] + etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"ad7e633a12e8a4915b45e6dd1d4b0b4b\"" + content_length = (@segments[:a][:size] + @segments[:c][:size]).to_s + response = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object') + + returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] } + returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] } + returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] } + end + + tests('#get_object').succeeds do + expected = @segments[:a][:data] + @segments[:c][:data] + Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected + end + + tests('#delete_static_large_object') do + expected = { + 'Number Not Found' => 0, + 'Response Status' => '200 OK', + 'Errors' => [], + 'Number Deleted' => 3, + 'Response Body' => '' + } + returns(expected, 'deletes manifest and segments') do + Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body + end + end - tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object') end - tests("#get_object streams all segments matching the default prefix").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == ('x' * 7 * 1024 * 1024) + tests('multiple containers') do + + tests('#put_static_obj_manifest').succeeds do + segments = [ + { :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }", + :etag => @segments[:b][:etag], + :size_bytes => @segments[:b][:size] }, + { :path => "#{ @segments[:d][:container] }/#{ @segments[:d][:name] }", + :etag => @segments[:d][:etag], + :size_bytes => @segments[:d][:size] } + ] + Fog::Storage[:openstack].put_static_obj_manifest(@directory2.identity, 'fog_large_object', segments) + end + + tests('#head_object') do + etags = [@segments[:b][:etag], @segments[:d][:etag]] + etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"9801a4cc4472896a1e975d03f0d2c3f8\"" + content_length = (@segments[:b][:size] + @segments[:d][:size]).to_s + response = Fog::Storage[:openstack].head_object(@directory2.identity, 'fog_large_object') + + returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] } + returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] } + returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] } + end + + tests('#get_object').succeeds do + expected = @segments[:b][:data] + @segments[:d][:data] + Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected + end + + tests('#delete_static_large_object') do + expected = { + 'Number Not Found' => 0, + 'Response Status' => '200 OK', + 'Errors' => [], + 'Number Deleted' => 3, + 'Response Body' => '' + } + returns(expected, 'deletes manifest and segments') do + Fog::Storage[:openstack].delete_static_large_object(@directory2.identity, 'fog_large_object').body + end + end + end - tests("#head_object returns Etag that includes manifest object in calculation").succeeds do - pending if Fog.mocking? - - etags = [] - # When the manifest object name is equal to the prefix, OpenStack treats it as if it's the first segment. - etags << Digest::MD5.hexdigest('') # Etag for manifest object => "d41d8cd98f00b204e9800998ecf8427e" - etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" - etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" - etags << Digest::MD5.hexdigest('x' * 1 * 1024 * 1024) # => "b561f87202d04959e37588ee05cf5b10" - expected = Digest::MD5.hexdigest(etags.join) # => "42e92048bd2c8085e7072b0b55fd76ab" - actual = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] - actual.gsub('"', '') == expected # actual is returned in quotes "\"42e92048bd2c8085e7072b0b55fd76abu"\" - end - - end - - tests("specifying X-Object-Manifest segment prefix") do - - tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/')").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"}) - end - - tests("#get_object streams segments only matching the specified prefix").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) - end - - tests("#head_object returns Etag that does not include manifest object in calculation").succeeds do - pending if Fog.mocking? - - etags = [] - etags << Digest::MD5.hexdigest('x' * 4 * 1024 * 1024) # => "44981362d3ba9b5bacaf017c2f29d355" - etags << Digest::MD5.hexdigest('x' * 2 * 1024 * 1024) # => "67b2f816a30e8956149b2d7beb479e51" - expected = Digest::MD5.hexdigest(etags.join) # => "0b348495a774eaa4d4c4bbf770820f84" - actual = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] - actual.gsub('"', '') == expected # actual is returned in quotes "\"0b348495a774eaa4d4c4bbf770820f84"\" - end - - end - - tests("storing manifest object in a different container than the segments") do - - tests("#put_object_manifest('foglargeobjecttests2', 'fog_large_object', {'X-Object-Manifest' => 'foglargeobjecttests/fog_large_object/'})").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].put_object_manifest(@directory2.identity, 'fog_large_object', {'X-Object-Manifest' => "#{@directory.identity}/fog_large_object/"}) - end - - tests("#get_object('foglargeobjecttests2', 'fog_large_object').body").succeeds do - pending if Fog.mocking? - Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == ('x' * 6 * 1024 * 1024) - end - - end - - unless Fog.mocking? - ['fog_large_object', 'fog_large_object/1', 'fog_large_object/2', 'fog_large_object2/1'].each do |key| - @directory.files.new(:key => key).destroy - end - @directory2.files.new(:key => 'fog_large_object').destroy end end tests('failure') do - tests("put_object_manifest") + tests('dynamic large object requests') do + pending if Fog.mocking? + + tests('#put_object_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do + Fog::Storage[:openstack].put_object_manifest('fognoncontainer', 'fog_large_object') + end + + end + + tests('static large object requests') do + pending if Fog.mocking? + + tests('upload test segments').succeeds do + Fog::Storage[:openstack].put_object(@segments[:a][:container], @segments[:a][:name], @segments[:a][:data]) + Fog::Storage[:openstack].put_object(@segments[:b][:container], @segments[:b][:name], @segments[:b][:data]) + end + + tests('#put_static_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do + Fog::Storage[:openstack].put_static_obj_manifest('fognoncontainer', 'fog_large_object', []) + end + + tests('#put_static_obj_manifest with missing object') do + segments = [ + { :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }", + :etag => @segments[:c][:etag], + :size_bytes => @segments[:c][:size] } + ] + expected = { 'Errors' => [[segments[0][:path], '404 Not Found']] } + + error = nil + begin + Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments) + rescue => err + error = err + end + + raises(Excon::Errors::BadRequest) do + raise error if error + end + + returns(expected, 'returns error information') do + Fog::JSON.decode(error.response.body) + end + end + + tests('#put_static_obj_manifest with invalid etag') do + segments = [ + { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }", + :etag => @segments[:b][:etag], + :size_bytes => @segments[:a][:size] } + ] + expected = { 'Errors' => [[segments[0][:path], 'Etag Mismatch']] } + + error = nil + begin + Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments) + rescue => err + error = err + end + + raises(Excon::Errors::BadRequest) do + raise error if error + end + + returns(expected, 'returns error information') do + Fog::JSON.decode(error.response.body) + end + end + + tests('#put_static_obj_manifest with invalid byte_size') do + segments = [ + { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }", + :etag => @segments[:a][:etag], + :size_bytes => @segments[:b][:size] } + ] + expected = { 'Errors' => [[segments[0][:path], 'Size Mismatch']] } + + error = nil + begin + Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments) + rescue => err + error = err + end + + raises(Excon::Errors::BadRequest) do + raise error if error + end + + returns(expected, 'returns error information') do + Fog::JSON.decode(error.response.body) + end + end + + tests('#delete_static_large_object with missing container') do + expected = { + 'Number Not Found' => 1, + 'Response Status' => '200 OK', + 'Errors' => [], + 'Number Deleted' => 0, + 'Response Body' => '' + } + + returns(expected, 'reports missing object') do + Fog::Storage[:openstack].delete_static_large_object('fognoncontainer', 'fog_large_object').body + end + end + + tests('#delete_static_large_object with missing manifest') do + expected = { + 'Number Not Found' => 1, + 'Response Status' => '200 OK', + 'Errors' => [], + 'Number Deleted' => 0, + 'Response Body' => '' + } + + returns(expected, 'reports missing manifest') do + Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_non_object').body + end + end + + tests('#delete_static_large_object with missing segment') do + + tests('#put_static_obj_manifest for segments :a and :b').succeeds do + segments = [ + { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }", + :etag => @segments[:a][:etag], + :size_bytes => @segments[:a][:size] }, + { :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }", + :etag => @segments[:b][:etag], + :size_bytes => @segments[:b][:size] } + ] + Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments) + end + + tests('#delete_object segment :b').succeeds do + Fog::Storage[:openstack].delete_object(@segments[:b][:container], @segments[:b][:name]) + end + + tests('#delete_static_large_object') do + expected = { + 'Number Not Found' => 1, + 'Response Status' => '200 OK', + 'Errors' => [], + 'Number Deleted' => 2, + 'Response Body' => '' + } + returns(expected, 'deletes manifest and segment :a, and reports missing segment :b') do + Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body + end + end + + end + end end From 56c28d2cb2ba1826ce7f932e8f6e259aef149316 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 21:49:44 -0400 Subject: [PATCH 5/6] [openstack|storage] add #put_dynamic_obj_manifest Renames the current #put_object_manifest method to better differentiate this from the new #put_static_obj_manifest method. #put_object_manifest has been retained for backward compatibility. --- .../storage/put_dynamic_obj_manifest.rb | 43 +++++++++++++++++++ .../requests/storage/put_object_manifest.rb | 28 ++---------- lib/fog/openstack/storage.rb | 1 + .../requests/storage/large_object_tests.rb | 20 +++++---- 4 files changed, 59 insertions(+), 33 deletions(-) create mode 100644 lib/fog/openstack/requests/storage/put_dynamic_obj_manifest.rb diff --git a/lib/fog/openstack/requests/storage/put_dynamic_obj_manifest.rb b/lib/fog/openstack/requests/storage/put_dynamic_obj_manifest.rb new file mode 100644 index 000000000..d185d5489 --- /dev/null +++ b/lib/fog/openstack/requests/storage/put_dynamic_obj_manifest.rb @@ -0,0 +1,43 @@ +module Fog + module Storage + class OpenStack + class Real + + # Create a new dynamic large object manifest + # + # Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("/") + # for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix. + # Segments must sort in the order they should be concatenated. Note that any future objects stored in the container + # along with the segments that match the prefix will be included when retrieving the manifest object. + # + # All segments must be stored in the same container, but may be in a different container than the manifest object. + # The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+ + # to specify the prefix and/or the container where segments were stored. + # If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::OpenStack.escape}). + # + # @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/' + # @param object [String] Name for manifest object. + # @param options [Hash] Config headers for +object+. + # @option options [String] 'X-Object-Manifest' ("container/object") "/" for segment objects. + # + # @raise [Fog::Storage::OpenStack::NotFound] HTTP 404 + # @raise [Excon::Errors::BadRequest] HTTP 400 + # @raise [Excon::Errors::Unauthorized] HTTP 401 + # @raise [Excon::Errors::HTTPStatusError] + # + # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/dynamic-large-object-creation.html + def put_dynamic_obj_manifest(container, object, options = {}) + path = "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + headers = {'X-Object-Manifest' => path}.merge(options) + request( + :expects => 201, + :headers => headers, + :method => 'PUT', + :path => path + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/put_object_manifest.rb b/lib/fog/openstack/requests/storage/put_object_manifest.rb index 3af4adf44..0037e9903 100644 --- a/lib/fog/openstack/requests/storage/put_object_manifest.rb +++ b/lib/fog/openstack/requests/storage/put_object_manifest.rb @@ -3,33 +3,11 @@ module Fog class OpenStack class Real - # Create a new manifest object + # Create a new dynamic large object manifest # - # Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("/") - # for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix. - # Segments must sort in the order they should be concatenated. Note that any future objects stored in the container - # along with the segments that match the prefix will be included when retrieving the manifest object. - # - # All segments must be stored in the same container, but may be in a different container than the manifest object. - # The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+ - # to specify the prefix and/or the container where segments were stored. - # If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::Rackspace.escape}). - # - # @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/' - # @param object [String] Name for manifest object. - # @param options [Hash] Config headers for +object+. - # @option options [String] 'X-Object-Manifest' ("container/object") "/" for segment objects. - # - # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/large-object-creation.html + # This is an alias for {#put_dynamic_obj_manifest} for backward compatibility. def put_object_manifest(container, object, options = {}) - path = "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" - headers = {'X-Object-Manifest' => path}.merge(options) - request( - :expects => 201, - :headers => headers, - :method => 'PUT', - :path => path - ) + put_dynamic_obj_manifest(container, object, options) end end diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index ec79e9ac5..0d3279c4d 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -33,6 +33,7 @@ module Fog request :put_container request :put_object request :put_object_manifest + request :put_dynamic_obj_manifest request :put_static_obj_manifest class Mock diff --git a/tests/openstack/requests/storage/large_object_tests.rb b/tests/openstack/requests/storage/large_object_tests.rb index 12c0a5649..ea142a3d9 100644 --- a/tests/openstack/requests/storage/large_object_tests.rb +++ b/tests/openstack/requests/storage/large_object_tests.rb @@ -48,10 +48,14 @@ Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) tests('dynamic large object requests') do pending if Fog.mocking? + tests('#put_object_manifest alias').succeeds do + Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object') + end + tests('using default X-Object-Manifest header') do - tests('#put_object_manifest').succeeds do - Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object') + tests('#put_dynamic_obj_manifest').succeeds do + Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object') end tests('#get_object streams all segments matching the default prefix').succeeds do @@ -71,9 +75,9 @@ Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) tests('specifying X-Object-Manifest segment prefix') do - tests('#put_object_manifest').succeeds do + tests('#put_dynamic_obj_manifest').succeeds do options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" } - Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object', options) + Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object', options) end tests('#get_object streams segments only matching the specified prefix').succeeds do @@ -91,9 +95,9 @@ Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) tests('storing manifest in a different container than the segments') do - tests('#put_object_manifest').succeeds do + tests('#put_dynamic_obj_manifest').succeeds do options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" } - Fog::Storage[:openstack].put_object_manifest(@directory2.identity, 'fog_large_object', options) + Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory2.identity, 'fog_large_object', options) end tests('#get_object').succeeds do @@ -207,8 +211,8 @@ Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) tests('dynamic large object requests') do pending if Fog.mocking? - tests('#put_object_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do - Fog::Storage[:openstack].put_object_manifest('fognoncontainer', 'fog_large_object') + tests('#put_dynamic_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do + Fog::Storage[:openstack].put_dynamic_obj_manifest('fognoncontainer', 'fog_large_object') end end From 65a88e4af00816fc39323f62053c293bc72aaf37 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Sun, 28 Jul 2013 21:49:47 -0400 Subject: [PATCH 6/6] [openstack|storage] patch #delete_static_large_object for Swift v1.8 This commit addresses an issue where the Content-Type header is not being set correctly in the response. This has been fixed in Swift v1.9. openstack/swift@ad24cde120efe73fe5a9b866f74c7d3c766aa4cb --- .../requests/storage/delete_static_large_object.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/fog/openstack/requests/storage/delete_static_large_object.rb b/lib/fog/openstack/requests/storage/delete_static_large_object.rb index 07c737b0b..82600f09d 100644 --- a/lib/fog/openstack/requests/storage/delete_static_large_object.rb +++ b/lib/fog/openstack/requests/storage/delete_static_large_object.rb @@ -25,13 +25,16 @@ module Fog # # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html def delete_static_large_object(container, object, options = {}) - request( + response = request({ :expects => 200, :method => 'DELETE', - :headers => options.merge('Content-Type' => 'text/plain'), + :headers => options.merge('Content-Type' => 'text/plain', + 'Accept' => 'application/json'), :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}", :query => { 'multipart-manifest' => 'delete' } - ) + }, false) + response.body = Fog::JSON.decode(response.body) + response end end