[AWS|Glacier] multipart uploads

lib/fog/aws/glacier.rb

@@ -10,6 +10,8 @@ module Fog
 
       request_path 'fog/aws/requests/glacier'
 
+      request :abort_multipart_upload
+      request :complete_multipart_upload
       request :create_archive
       request :create_vault
       request :delete_archive
@@ -17,16 +19,19 @@ module Fog
       request :delete_vault_notification_configuration
       request :describe_vault
       request :get_vault_notification_configuration
+      request :initiate_multipart_upload
+      request :list_multipart_uploads
+      request :list_parts
       request :list_vaults
       request :set_vault_notification_configuration
-
+      request :upload_part
 
       MEGABYTE = 1024*1024
 
       class TreeHash
 
         def self.digest(body)
-          new.add_part(body).digest
+          new.add_part(body)
         end
 
         def reduce_digests(digests)
@@ -47,8 +52,9 @@ module Fog
         end
 
         def add_part(bytes)
-          @digests << self.digest_for_part(bytes)
+          part = self.digest_for_part(bytes)
-          self
+          @digests << part
+          part.unpack('H*').first
         end
 
         def digest_for_part(body)
@@ -57,8 +63,12 @@ module Fog
           reduce_digests(digests_for_part)
         end
 
+        def hexdigest
+          digest.unpack('H*').first
+        end
+
         def digest
-          reduce_digests(@digests).unpack('H*').first
+          reduce_digests(@digests)
         end
       end
 

lib/fog/aws/requests/glacier/abort_multipart_upload.rb

@@ -0,0 +1,35 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        # Abort an upload
+        #
+        # ==== Parameters
+        # * name<~String> Name of the vault to upload to
+        # * upload_id<~String> The id of the upload to complete
+        # * options<~Hash>
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-abort-upload.html
+        #
+        def abort_multipart_upload(vault_name, upload_id, options={})
+          account_id = options['account_id'] || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}"
+
+          request(
+            :expects  => 204,
+            :idempotent => true,
+            :headers => {},
+            :method   => :delete,
+            :path     => path,
+          )
+        end
+      end
+
+    end
+  end
+end

lib/fog/aws/requests/glacier/complete_multipart_upload.rb

@@ -0,0 +1,42 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        # Complete an upload
+        #
+        # ==== Parameters
+        # * name<~String> Name of the vault to upload to
+        # * upload_id<~String> The id of the upload to complete
+        # * total_size<~Integer> The total archive size
+        # * tree_hash<~String> the treehash for the archive
+        # * options<~Hash>
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-complete-upload.html
+        #
+        def complete_multipart_upload(vault_name, upload_id, total_size, tree_hash, options={})
+          account_id = options['account_id'] || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}"
+
+          headers = {
+            'x-amz-archive-size' => total_size.to_s,
+            'x-amz-sha256-tree-hash' => tree_hash
+          }
+
+          request(
+            :expects  => 201,
+            :idempotent => true,
+            :headers => headers,
+            :method   => :post,
+            :path     => path
+          )
+        end
+      end
+
+    end
+  end
+end

lib/fog/aws/requests/glacier/initiate_multipart_upload.rb

@@ -0,0 +1,38 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        # This operation initates a multipart upload of an archive to a vault
+        #
+        # ==== Parameters
+        # * name<~String> The vault name
+        # * part_size<~Integer> The part size to use. Must be a power of 2 multiple of 1MB (1,2,4,8,16,...)
+        # * options<~Hash>
+        #   * description<~String> - The archive description
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~Hash>:
+        #
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-initiate-upload.html
+        #
+        def initiate_multipart_upload(name, part_size, options={})
+          account_id = options['account_id'] || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/multipart-uploads"
+
+          headers = {'x-amz-part-size' => part_size.to_s}
+          headers['x-amz-description'] = options['description'] if options['description']
+          request(
+            :expects  => 201,
+            :headers => headers,
+            :method   => 'POST',
+            :path     => path,
+          )
+        end
+      end
+
+    end
+  end
+end

lib/fog/aws/requests/glacier/list_multipart_uploads.rb

@@ -0,0 +1,37 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        #  lists in-progress multipart uploads for the specified vault
+        #
+        # ==== Parameters
+        # * name<~String> Name of the vault
+        # * options<~Hash>
+        #   * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
+        #   * marker<~String> - marker used for pagination
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-uploads.html
+        #
+        def list_multipart_uploads(vault_name, options={})
+          account_id = options.delete('account_id') || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads"
+
+
+          request(
+            :expects  => 200,
+            :idempotent => true,
+            :headers => {},
+            :method   => :get,
+            :path     => path,
+            :query => options
+          )
+        end
+      end
+
+    end
+  end
+end

lib/fog/aws/requests/glacier/list_parts.rb

@@ -0,0 +1,37 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        #  lists the parts of an archive that have been uploaded in a specific multipart upload
+        #
+        # ==== Parameters
+        # * name<~String> Name of the vault
+        # * upload_id<~String> The id of the upload
+        # * options<~Hash>
+        #   * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
+        #   * marker<~String> - marker used for pagination
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-parts.html
+        #
+        def list_parts(vault_name, upload_id, options={})
+          account_id = options.delete('account_id') || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}"
+
+          request(
+            :expects  => 200,
+            :idempotent => true,
+            :headers => {},
+            :method   => :get,
+            :path     => path,
+            :query    => options
+          )
+        end
+      end
+
+    end
+  end
+end

lib/fog/aws/requests/glacier/upload_part.rb

@@ -0,0 +1,46 @@
+module Fog
+  module AWS
+    class Glacier
+      class Real
+
+        # Upload an archive
+        #
+        # ==== Parameters
+        # * name<~String> Name of the vault to upload to
+        # * uploadId<~String> Id of the upload
+        # * body<~String> The data to upload
+        # * offset<~Integer> The offset of the data within the archive
+        # * hash<~String> The tree hash for this part
+        # * options<~Hash>
+        #   * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-upload-part.html
+        #
+        def upload_part(vault_name, upload_id, body, offset, hash, options={})
+          account_id = options['account_id'] || '-'
+          path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}"
+
+          headers = {
+            'Content-Length' => body.length.to_s,
+            'Content-Range' => "bytes #{offset}-#{offset+body.length-1}/*",
+            'x-amz-content-sha256' => Digest::SHA256.hexdigest(body),
+            'x-amz-sha256-tree-hash' => hash
+          }
+
+          request(
+            :expects  => 204,
+            :idempotent => true,
+            :headers => headers,
+            :method   => :put,
+            :path     => path,
+            :body     => body
+          )
+        end
+      end
+
+    end
+  end
+end

tests/aws/requests/glacier/multipart_upload_tests.rb

@@ -0,0 +1,30 @@
+Shindo.tests('AWS::Glacier | glacier archive tests', ['aws']) do
+  pending if Fog.mocking?
+
+  Fog::AWS[:glacier].create_vault('Fog-Test-Vault-upload')
+
+  tests('initiate and abort') do
+    id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id']
+    returns(true){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)}
+    Fog::AWS[:glacier].abort_multipart_upload('Fog-Test-Vault-upload', id)
+    returns(false){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)}
+  end
+
+
+  tests('do multipart upload') do
+    hash = Fog::AWS::Glacier::TreeHash.new
+    id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id']
+    part = 't'*1024*1024
+    hash_for_part = hash.add_part(part)
+    Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part, 0, hash_for_part)
+
+    part_2 = 'u'*1024*1024
+    hash_for_part_2 = hash.add_part(part_2)
+    Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part_2, 1024*1024, hash_for_part_2)
+
+    archive = Fog::AWS[:glacier].complete_multipart_upload('Fog-Test-Vault-upload', id, 2*1024*1024, hash.hexdigest).headers['x-amz-archive-id']
+
+    Fog::AWS[:glacier].delete_archive('Fog-Test-Vault-upload', archive)    
+  #amazon won't let us delete the vault because it has been written to in the past day
+  end
+end

tests/aws/requests/glacier/tree_hash_tests.rb

@@ -32,8 +32,10 @@ Shindo.tests('AWS::Glacier | glacier tree hash calcuation', ['aws']) do
 
   tests('multipart') do
     tree_hash = Fog::AWS::Glacier::TreeHash.new
-    tree_hash.add_part ('x' * 1024*1024) + ('y'*1024*1024)
+    part = ('x' * 1024*1024) + ('y'*1024*1024)
-    tree_hash.add_part ('z'* 1024*1024) + ('t'*1024*1024)
+    returns(Fog::AWS::Glacier::TreeHash.digest(part)) { tree_hash.add_part part }
+
+    tree_hash.add_part('z'* 1024*1024 + 't'*1024*1024)
 
     expected = Digest::SHA256.hexdigest(
                  Digest::SHA256.digest(
@@ -43,7 +45,7 @@ Shindo.tests('AWS::Glacier | glacier tree hash calcuation', ['aws']) do
                    Digest::SHA256.digest('z' * 1024*1024) + Digest::SHA256.digest('t' * 1024*1024)
                  )
                )
-    returns(expected) { tree_hash.digest}
+    returns(expected) { tree_hash.hexdigest}
 
   end