From 17c8e44a31d54cc89201228aec17bef712b704f0 Mon Sep 17 00:00:00 2001 From: Frederick Cheung Date: Sat, 1 Sep 2012 23:09:58 +0100 Subject: [PATCH] [AWS|Glacier] multipart uploads --- lib/fog/aws/glacier.rb | 20 ++++++-- .../glacier/abort_multipart_upload.rb | 35 ++++++++++++++ .../glacier/complete_multipart_upload.rb | 42 +++++++++++++++++ .../glacier/initiate_multipart_upload.rb | 38 +++++++++++++++ .../glacier/list_multipart_uploads.rb | 37 +++++++++++++++ lib/fog/aws/requests/glacier/list_parts.rb | 37 +++++++++++++++ lib/fog/aws/requests/glacier/upload_part.rb | 46 +++++++++++++++++++ .../glacier/multipart_upload_tests.rb | 30 ++++++++++++ tests/aws/requests/glacier/tree_hash_tests.rb | 8 ++-- 9 files changed, 285 insertions(+), 8 deletions(-) create mode 100644 lib/fog/aws/requests/glacier/abort_multipart_upload.rb create mode 100644 lib/fog/aws/requests/glacier/complete_multipart_upload.rb create mode 100644 lib/fog/aws/requests/glacier/initiate_multipart_upload.rb create mode 100644 lib/fog/aws/requests/glacier/list_multipart_uploads.rb create mode 100644 lib/fog/aws/requests/glacier/list_parts.rb create mode 100644 lib/fog/aws/requests/glacier/upload_part.rb create mode 100644 tests/aws/requests/glacier/multipart_upload_tests.rb diff --git a/lib/fog/aws/glacier.rb b/lib/fog/aws/glacier.rb index 663bbf474..c53c2c7a1 100644 --- a/lib/fog/aws/glacier.rb +++ b/lib/fog/aws/glacier.rb @@ -10,6 +10,8 @@ module Fog request_path 'fog/aws/requests/glacier' + request :abort_multipart_upload + request :complete_multipart_upload request :create_archive request :create_vault request :delete_archive @@ -17,16 +19,19 @@ module Fog request :delete_vault_notification_configuration request :describe_vault request :get_vault_notification_configuration + request :initiate_multipart_upload + request :list_multipart_uploads + request :list_parts request :list_vaults request :set_vault_notification_configuration - + request :upload_part MEGABYTE = 1024*1024 class TreeHash def self.digest(body) - new.add_part(body).digest + new.add_part(body) end def reduce_digests(digests) @@ -47,8 +52,9 @@ module Fog end def add_part(bytes) - @digests << self.digest_for_part(bytes) - self + part = self.digest_for_part(bytes) + @digests << part + part.unpack('H*').first end def digest_for_part(body) @@ -57,8 +63,12 @@ module Fog reduce_digests(digests_for_part) end + def hexdigest + digest.unpack('H*').first + end + def digest - reduce_digests(@digests).unpack('H*').first + reduce_digests(@digests) end end diff --git a/lib/fog/aws/requests/glacier/abort_multipart_upload.rb b/lib/fog/aws/requests/glacier/abort_multipart_upload.rb new file mode 100644 index 000000000..79406f57c --- /dev/null +++ b/lib/fog/aws/requests/glacier/abort_multipart_upload.rb @@ -0,0 +1,35 @@ +module Fog + module AWS + class Glacier + class Real + + # Abort an upload + # + # ==== Parameters + # * name<~String> Name of the vault to upload to + # * upload_id<~String> The id of the upload to complete + # * options<~Hash> + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-abort-upload.html + # + def abort_multipart_upload(vault_name, upload_id, options={}) + account_id = options['account_id'] || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}" + + request( + :expects => 204, + :idempotent => true, + :headers => {}, + :method => :delete, + :path => path, + ) + end + end + + end + end +end diff --git a/lib/fog/aws/requests/glacier/complete_multipart_upload.rb b/lib/fog/aws/requests/glacier/complete_multipart_upload.rb new file mode 100644 index 000000000..56f27b6b7 --- /dev/null +++ b/lib/fog/aws/requests/glacier/complete_multipart_upload.rb @@ -0,0 +1,42 @@ +module Fog + module AWS + class Glacier + class Real + + # Complete an upload + # + # ==== Parameters + # * name<~String> Name of the vault to upload to + # * upload_id<~String> The id of the upload to complete + # * total_size<~Integer> The total archive size + # * tree_hash<~String> the treehash for the archive + # * options<~Hash> + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-complete-upload.html + # + def complete_multipart_upload(vault_name, upload_id, total_size, tree_hash, options={}) + account_id = options['account_id'] || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}" + + headers = { + 'x-amz-archive-size' => total_size.to_s, + 'x-amz-sha256-tree-hash' => tree_hash + } + + request( + :expects => 201, + :idempotent => true, + :headers => headers, + :method => :post, + :path => path + ) + end + end + + end + end +end diff --git a/lib/fog/aws/requests/glacier/initiate_multipart_upload.rb b/lib/fog/aws/requests/glacier/initiate_multipart_upload.rb new file mode 100644 index 000000000..362cdfb46 --- /dev/null +++ b/lib/fog/aws/requests/glacier/initiate_multipart_upload.rb @@ -0,0 +1,38 @@ +module Fog + module AWS + class Glacier + class Real + + # This operation initates a multipart upload of an archive to a vault + # + # ==== Parameters + # * name<~String> The vault name + # * part_size<~Integer> The part size to use. Must be a power of 2 multiple of 1MB (1,2,4,8,16,...) + # * options<~Hash> + # * description<~String> - The archive description + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # * body<~Hash>: + # + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-initiate-upload.html + # + def initiate_multipart_upload(name, part_size, options={}) + account_id = options['account_id'] || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/multipart-uploads" + + headers = {'x-amz-part-size' => part_size.to_s} + headers['x-amz-description'] = options['description'] if options['description'] + request( + :expects => 201, + :headers => headers, + :method => 'POST', + :path => path, + ) + end + end + + end + end +end diff --git a/lib/fog/aws/requests/glacier/list_multipart_uploads.rb b/lib/fog/aws/requests/glacier/list_multipart_uploads.rb new file mode 100644 index 000000000..07512a427 --- /dev/null +++ b/lib/fog/aws/requests/glacier/list_multipart_uploads.rb @@ -0,0 +1,37 @@ +module Fog + module AWS + class Glacier + class Real + + # lists in-progress multipart uploads for the specified vault + # + # ==== Parameters + # * name<~String> Name of the vault + # * options<~Hash> + # * limit<~Integer> - The maximum number of items returned in the response. (default 1000) + # * marker<~String> - marker used for pagination + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-uploads.html + # + def list_multipart_uploads(vault_name, options={}) + account_id = options.delete('account_id') || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads" + + + request( + :expects => 200, + :idempotent => true, + :headers => {}, + :method => :get, + :path => path, + :query => options + ) + end + end + + end + end +end diff --git a/lib/fog/aws/requests/glacier/list_parts.rb b/lib/fog/aws/requests/glacier/list_parts.rb new file mode 100644 index 000000000..ae92c536c --- /dev/null +++ b/lib/fog/aws/requests/glacier/list_parts.rb @@ -0,0 +1,37 @@ +module Fog + module AWS + class Glacier + class Real + + # lists the parts of an archive that have been uploaded in a specific multipart upload + # + # ==== Parameters + # * name<~String> Name of the vault + # * upload_id<~String> The id of the upload + # * options<~Hash> + # * limit<~Integer> - The maximum number of items returned in the response. (default 1000) + # * marker<~String> - marker used for pagination + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-parts.html + # + def list_parts(vault_name, upload_id, options={}) + account_id = options.delete('account_id') || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}" + + request( + :expects => 200, + :idempotent => true, + :headers => {}, + :method => :get, + :path => path, + :query => options + ) + end + end + + end + end +end diff --git a/lib/fog/aws/requests/glacier/upload_part.rb b/lib/fog/aws/requests/glacier/upload_part.rb new file mode 100644 index 000000000..c8e26715c --- /dev/null +++ b/lib/fog/aws/requests/glacier/upload_part.rb @@ -0,0 +1,46 @@ +module Fog + module AWS + class Glacier + class Real + + # Upload an archive + # + # ==== Parameters + # * name<~String> Name of the vault to upload to + # * uploadId<~String> Id of the upload + # * body<~String> The data to upload + # * offset<~Integer> The offset of the data within the archive + # * hash<~String> The tree hash for this part + # * options<~Hash> + # * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request + # ==== Returns + # * response<~Excon::Response>: + # + # ==== See Also + # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-upload-part.html + # + def upload_part(vault_name, upload_id, body, offset, hash, options={}) + account_id = options['account_id'] || '-' + path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}" + + headers = { + 'Content-Length' => body.length.to_s, + 'Content-Range' => "bytes #{offset}-#{offset+body.length-1}/*", + 'x-amz-content-sha256' => Digest::SHA256.hexdigest(body), + 'x-amz-sha256-tree-hash' => hash + } + + request( + :expects => 204, + :idempotent => true, + :headers => headers, + :method => :put, + :path => path, + :body => body + ) + end + end + + end + end +end diff --git a/tests/aws/requests/glacier/multipart_upload_tests.rb b/tests/aws/requests/glacier/multipart_upload_tests.rb new file mode 100644 index 000000000..10699147b --- /dev/null +++ b/tests/aws/requests/glacier/multipart_upload_tests.rb @@ -0,0 +1,30 @@ +Shindo.tests('AWS::Glacier | glacier archive tests', ['aws']) do + pending if Fog.mocking? + + Fog::AWS[:glacier].create_vault('Fog-Test-Vault-upload') + + tests('initiate and abort') do + id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id'] + returns(true){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)} + Fog::AWS[:glacier].abort_multipart_upload('Fog-Test-Vault-upload', id) + returns(false){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)} + end + + + tests('do multipart upload') do + hash = Fog::AWS::Glacier::TreeHash.new + id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id'] + part = 't'*1024*1024 + hash_for_part = hash.add_part(part) + Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part, 0, hash_for_part) + + part_2 = 'u'*1024*1024 + hash_for_part_2 = hash.add_part(part_2) + Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part_2, 1024*1024, hash_for_part_2) + + archive = Fog::AWS[:glacier].complete_multipart_upload('Fog-Test-Vault-upload', id, 2*1024*1024, hash.hexdigest).headers['x-amz-archive-id'] + + Fog::AWS[:glacier].delete_archive('Fog-Test-Vault-upload', archive) + #amazon won't let us delete the vault because it has been written to in the past day + end +end \ No newline at end of file diff --git a/tests/aws/requests/glacier/tree_hash_tests.rb b/tests/aws/requests/glacier/tree_hash_tests.rb index c052764ce..e00b9f2fb 100644 --- a/tests/aws/requests/glacier/tree_hash_tests.rb +++ b/tests/aws/requests/glacier/tree_hash_tests.rb @@ -32,8 +32,10 @@ Shindo.tests('AWS::Glacier | glacier tree hash calcuation', ['aws']) do tests('multipart') do tree_hash = Fog::AWS::Glacier::TreeHash.new - tree_hash.add_part ('x' * 1024*1024) + ('y'*1024*1024) - tree_hash.add_part ('z'* 1024*1024) + ('t'*1024*1024) + part = ('x' * 1024*1024) + ('y'*1024*1024) + returns(Fog::AWS::Glacier::TreeHash.digest(part)) { tree_hash.add_part part } + + tree_hash.add_part('z'* 1024*1024 + 't'*1024*1024) expected = Digest::SHA256.hexdigest( Digest::SHA256.digest( @@ -43,7 +45,7 @@ Shindo.tests('AWS::Glacier | glacier tree hash calcuation', ['aws']) do Digest::SHA256.digest('z' * 1024*1024) + Digest::SHA256.digest('t' * 1024*1024) ) ) - returns(expected) { tree_hash.digest} + returns(expected) { tree_hash.hexdigest} end