kotovalexarian-likes-github/fog--fog
1
0
Fork 0
mirror of https://github.com/fog/fog.git synced 2022-11-09 13:51:43 -05:00
Code Releases Activity

Merge pull request #1124 from fcheung/glacier

Browse source 
[AWS|Glacier] Initial support for glacier
This commit is contained in:
Frederick Cheung 2012-09-04 09:00:26 -07:00
commit b48fe05d35
37 changed files with 1466 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/fog/aws.rb
View file

@ -1,5 +1,6 @@
require 'fog/core'
require 'fog/aws/credential_fetcher'
require 'fog/aws/signaturev4'
module Fog
module AWS
@ -16,6 +17,7 @@ module Fog
service(:elasticache, 'aws/elasticache', 'Elasticache')
service(:elb, 'aws/elb', 'ELB')
service(:emr, 'aws/emr', 'EMR')
service(:glacier, 'aws/glacier', 'Glacier')
service(:iam, 'aws/iam', 'IAM')
service(:rds, 'aws/rds', 'RDS')
service(:ses, 'aws/ses', 'SES')

172
lib/fog/aws/glacier.rb Normal file
View file

@ -0,0 +1,172 @@
require 'fog/aws'
module Fog
module AWS
class Glacier < Fog::Service
extend Fog::AWS::CredentialFetcher::ServiceMethods
requires :aws_access_key_id, :aws_secret_access_key
recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at
request_path 'fog/aws/requests/glacier'
request :abort_multipart_upload
request :complete_multipart_upload
request :create_archive
request :create_vault
request :delete_archive
request :delete_vault
request :delete_vault_notification_configuration
request :describe_job
request :describe_vault
request :get_job_output
request :get_vault_notification_configuration
request :initiate_job
request :initiate_multipart_upload
request :list_jobs
request :list_multipart_uploads
request :list_parts
request :list_vaults
request :set_vault_notification_configuration
request :upload_part
model_path 'fog/aws/models/glacier'
model :vault
collection :vaults
MEGABYTE = 1024*1024
class TreeHash
def self.digest(body)
new.add_part(body)
end
def reduce_digests(digests)
while digests.length > 1
digests = digests.each_slice(2).collect do |pair|
if pair.length == 2
Digest::SHA256.digest(pair[0]+pair[1])
else
pair.first
end
end
end
digests.first
end
def initialize
@digests = []
end
def add_part(bytes)
part = self.digest_for_part(bytes)
@digests << part
part.unpack('H*').first
end
def digest_for_part(body)
chunk_count = [body.bytesize / MEGABYTE + (body.bytesize % MEGABYTE > 0 ? 1 : 0), 1].max
if body.respond_to? :byteslice
digests_for_part = chunk_count.times.collect {|chunk_index| Digest::SHA256.digest(body.byteslice(chunk_index * MEGABYTE, MEGABYTE))}
else
if body.respond_to? :encoding
old_encoding = body.encoding
body.force_encoding('BINARY')
end
digests_for_part = chunk_count.times.collect {|chunk_index| Digest::SHA256.digest(body.slice(chunk_index * MEGABYTE, MEGABYTE))}
if body.respond_to? :encoding
body.force_encoding(old_encoding)
end
end
reduce_digests(digests_for_part)
end
def hexdigest
digest.unpack('H*').first
end
def digest
reduce_digests(@digests)
end
end
class Mock
def initialize(options={})
Fog::Mock.not_implemented
end
end
class Real
include Fog::AWS::CredentialFetcher::ConnectionMethods
# Initialize connection to Glacier
#
# ==== Notes
# options parameter must include values for :aws_access_key_id and
# :aws_secret_access_key in order to create a connection
#
# ==== Examples
# ses = SES.new(
# :aws_access_key_id => your_aws_access_key_id,
# :aws_secret_access_key => your_aws_secret_access_key
# )
#
# ==== Parameters
# * options<~Hash> - config arguments for connection. Defaults to {}.
# * region<~String> - optional region to use. For instance, 'us-east-1' and etc.
#
# ==== Returns
# * Glacier object with connection to AWS.
def initialize(options={})
@use_iam_profile = options[:use_iam_profile]
@region = options[:region] || 'us-east-1'
setup_credentials(options)
@connection_options = options[:connection_options] || {}
@host = options[:host] || "glacier.#{@region}.amazonaws.com"
@version = '2012-06-01'
@path = options[:path] || '/'
@persistent = options[:persistent] || false
@port = options[:port] || 443
@scheme = options[:scheme] || 'https'
@connection = Fog::Connection.new("#{@scheme}://#{@host}:#{@port}#{@path}", @persistent, @connection_options)
end
private
def setup_credentials(options)
@aws_access_key_id = options[:aws_access_key_id]
@aws_secret_access_key = options[:aws_secret_access_key]
@aws_session_token = options[:aws_session_token]
@aws_credentials_expire_at = options[:aws_credentials_expire_at]
@signer = Fog::AWS::SignatureV4.new( @aws_access_key_id, @aws_secret_access_key,@region,'glacier')
end
def request(params, &block)
refresh_credentials_if_expired
date = Fog::Time.now
params[:headers]['Date'] = date.to_date_header
params[:headers]['x-amz-date'] = date.to_iso8601_basic
params[:headers]['Host'] = @host
params[:headers]['x-amz-glacier-version'] = @version
params[:headers]['x-amz-security-token'] = @aws_session_token if @aws_session_token
params[:headers]['Authorization'] = @signer.sign params, date
response = @connection.request(params, &block)
if response.headers['Content-Type'] == 'application/json'
response.body = Fog::JSON.decode(response.body)
end
response
end
end
end
end
end

70
lib/fog/aws/models/glacier/archive.rb Normal file
View file

@ -0,0 +1,70 @@
require 'fog/core/model'
module Fog
module AWS
class Glacier
class Archive < Fog::Model
identity :id
attribute :description
attribute :body
attr_accessor :multipart_chunk_size #must be a power of 2 multiple of 1MB
def vault
@vault
end
def save
requires :body, :vault
if multipart_chunk_size && body.respond_to?(:read)
self.id = multipart_save
else
data = connection.create_archive(vault.id, body, 'description' => description)
self.id = data.headers['x-amz-archive-id']
end
true
end
def destroy
requires :id
connection.delete_archive(vault.id,id)
end
private
def vault=(new_vault)
@vault = new_vault
end
def multipart_save
# Initiate the upload
res = connection.initiate_multipart_upload vault.id, multipart_chunk_size, 'description' => description
upload_id = res.headers["x-amz-multipart-upload-id"]
hash = Fog::AWS::Glacier::TreeHash.new
body.rewind if body.respond_to?(:rewind)
offset = 0
while (chunk = body.read(multipart_chunk_size)) do
part_hash = hash.add_part(chunk)
part_upload = connection.upload_part(vault.id, upload_id, chunk, offset, part_hash )
offset += chunk.bytesize
end
rescue
# Abort the upload & reraise
connection.abort_multipart_upload(vault.id, upload_id) if upload_id
raise
else
# Complete the upload
connection.complete_multipart_upload(vault.id, upload_id, offset, hash.hexdigest).headers['x-amz-archive-id']
end
end
end
end
end

30
lib/fog/aws/models/glacier/archives.rb Normal file
View file

@ -0,0 +1,30 @@
require 'fog/core/collection'
require 'fog/aws/models/glacier/archive'
module Fog
module AWS
class Glacier
class Archives < Fog::Collection
model Fog::AWS::Glacier::Archive
attribute :vault
#you can't list a vault's archives
def all
nil
end
def get(key)
new(:id => key)
end
def new(attributes = {})
requires :vault
super({ :vault => vault }.merge!(attributes))
end
end
end
end
end

65
lib/fog/aws/models/glacier/job.rb Normal file
View file

@ -0,0 +1,65 @@
require 'fog/core/model'
module Fog
module AWS
class Glacier
class Job < Fog::Model
ARCHIVE = 'archive-retrieval'
INVENTORY = 'inventory-retrieval'
identity :id, :aliases => "JobId"
attribute :action, :aliases => "Action"
attribute :archive_id, :aliases => "ArchiveId"
attribute :archive_size, :aliases => "ArchiveSizeInBytes", :type => :integer
attribute :completed, :aliases => "Completed", :type => :boolean
attribute :completed_at, :aliases => "CompletionDate", :type => :time
attribute :created_at, :aliases => "CreationDate", :type => :time
attribute :inventory_size, :aliases => "InventorySizeInBytes", :type => :integer
attribute :description, :aliases=> "JobDescription"
attribute :tree_hash, :aliases=> "SHA256TreeHash"
attribute :sns_topic, :aliases => "SNSTopic"
attribute :status_code, :aliases=> "StatusCode"
attribute :status_message, :aliases=> "StatusMessage"
attribute :vault_arn, :aliases=> "VaultARN"
attribute :format
attribute :type
def ready?
completed
end
def save
requires :vault, :type
specification = {'Type' => type, 'ArchiveId' => archive_id, 'Format' => format, 'Description' => description, 'SNSTopic' => sns_topic}.reject{|k,v| v.nil?}
data = connection.initiate_job(vault.id, specification)
self.id = data.headers['x-amz-job-id']
reload
end
def vault
@vault
end
#pass :range => 1..1234 to only retrieve those bytes
#pass :io => f to stream the response to that tio
def get_output(options={})
if io = options.delete(:io)
options = options.merge :response_block => lambda {|chunk, remaining_bytes, total_bytes| io.write chunk}
end
options['Range'] = options.delete :range
connection.get_job_output(vault.id, id, options)
end
private
def vault=(new_vault)
@vault = new_vault
end
end
end
end
end

35
lib/fog/aws/models/glacier/jobs.rb Normal file
View file

@ -0,0 +1,35 @@
require 'fog/core/collection'
require 'fog/aws/models/glacier/job'
module Fog
module AWS
class Glacier
class Jobs < Fog::Collection
model Fog::AWS::Glacier::Job
attribute :vault
def all
data = connection.list_jobs(vault.id).body['JobList']
load(data)
end
def get(key)
data = connection.describe_job(vault.id, key).body
new(data)
rescue Excon::Errors::NotFound
nil
end
def new(attributes = {})
requires :vault
super({ :vault => vault }.merge!(attributes))
end
end
end
end
end

53
lib/fog/aws/models/glacier/vault.rb Normal file
View file

@ -0,0 +1,53 @@
require 'fog/core/model'
require 'fog/aws/models/glacier/archives'
require 'fog/aws/models/glacier/jobs'
module Fog
module AWS
class Glacier
class Vault < Fog::Model
identity :id, :aliases => 'VaultName'
attribute :created_at, :aliases => 'CreationDate', :type => :time
attribute :last_inventory_at, :aliases => 'LastInventoryDate', :type => :time
attribute :number_of_archives, :aliases => 'NumberOfArchives', :type => :integer
attribute :size_in_bytes, :aliases => 'SizeInBytes', :type => :integer
attribute :arn, :aliases => 'VaultARN'
def ready?
# Glacier requests are synchronous
true
end
def archives
@archives ||= Fog::AWS::Glacier::Archives.new(:vault => self, :connection => connection)
end
def jobs
@jobs ||= Fog::AWS::Glacier::Jobs.new(:vault => self, :connection => connection)
end
def set_notification_configuration(topic, events)
connection.set_vault_notification_configuration(id, topic, events)
end
def delete_notification_configuration
connection.delete_vault_notification_configuration
end
def save
requires :id
connection.create_vault(id)
reload
end
def destroy
requires :id
connection.delete_vault(id)
end
end
end
end
end

28
lib/fog/aws/models/glacier/vaults.rb Normal file
View file

@ -0,0 +1,28 @@
require 'fog/core/collection'
require 'fog/aws/models/glacier/vault'
module Fog
module AWS
class Glacier
class Vaults < Fog::Collection
model Fog::AWS::Glacier::Vault
def all
data = connection.list_vaults.body['VaultList']
load(data)
end
def get(key)
data = connection.describe_vault(key).body
new(data)
rescue Excon::Errors::NotFound
nil
end
end
end
end
end

35
lib/fog/aws/requests/glacier/abort_multipart_upload.rb Normal file
View file

@ -0,0 +1,35 @@
module Fog
module AWS
class Glacier
class Real
# Abort an upload
#
# ==== Parameters
# * name<~String> Name of the vault to upload to
# * upload_id<~String> The id of the upload to complete
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-abort-upload.html
#
def abort_multipart_upload(vault_name, upload_id, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}"
request(
:expects => 204,
:idempotent => true,
:headers => {},
:method => :delete,
:path => path
)
end
end
end
end
end

42
lib/fog/aws/requests/glacier/complete_multipart_upload.rb Normal file
View file

@ -0,0 +1,42 @@
module Fog
module AWS
class Glacier
class Real
# Complete an upload
#
# ==== Parameters
# * name<~String> Name of the vault to upload to
# * upload_id<~String> The id of the upload to complete
# * total_size<~Integer> The total archive size
# * tree_hash<~String> the treehash for the archive
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-complete-upload.html
#
def complete_multipart_upload(vault_name, upload_id, total_size, tree_hash, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{upload_id}"
headers = {
'x-amz-archive-size' => total_size.to_s,
'x-amz-sha256-tree-hash' => tree_hash
}
request(
:expects => 201,
:idempotent => true,
:headers => headers,
:method => :post,
:path => path
)
end
end
end
end
end

43
lib/fog/aws/requests/glacier/create_archive.rb Normal file
View file

@ -0,0 +1,43 @@
module Fog
module AWS
class Glacier
class Real
# Upload an archive
#
# ==== Parameters
# * name<~String> Name of the vault to upload to
# * body<~String> The data to upload
# * options<~Hash>
# * description<~String> - The archive description
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-archive-post.html
#
def create_archive(vault_name, body, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/archives"
headers = {
'Content-Length' => body.bytesize.to_s,
'x-amz-content-sha256' => Digest::SHA256.hexdigest(body),
'x-amz-sha256-tree-hash' => Fog::AWS::Glacier::TreeHash.digest(body)
}
headers['x-amz-archive-description'] = options['description'] if options['description']
request(
:expects => 201,
:headers => headers,
:method => :post,
:path => path,
:body => body
)
end
end
end
end
end

34
lib/fog/aws/requests/glacier/create_vault.rb Normal file
View file

@ -0,0 +1,34 @@
module Fog
module AWS
class Glacier
class Real
# This operation creates a new vault with the specified name. .
#
# ==== Parameters
# * name<~String> 1-255 characters. must be unique within a region for an AWS account
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-put.html
#
def create_vault(name,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}"
request(options.merge({
:expects => 201,
:idempotent => true,
:headers => {},
:method => :put,
:path => path,
}))
end
end
end
end
end

35
lib/fog/aws/requests/glacier/delete_archive.rb Normal file
View file

@ -0,0 +1,35 @@
module Fog
module AWS
class Glacier
class Real
# Delete an archive
#
# ==== Parameters
# * name<~String> Name of the vault to delete
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
# * 'RequestId'<~String> - Id of the request
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-delete.html
#
def delete_archive(name,archive_id,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/archives/#{archive_id}"
request(
:expects => 204,
:idempotent => true,
:headers => {},
:method => :delete,
:path => path
)
end
end
end
end
end

34
lib/fog/aws/requests/glacier/delete_vault.rb Normal file
View file

@ -0,0 +1,34 @@
module Fog
module AWS
class Glacier
class Real
# Delete a vault. Amazon Glacier will delete a vault only if there are no archives in the vault as per the last inventory
# and there have been no writes to the vault since the last inventory
#
# ==== Parameters
# * name<~String> Name of the vault to delete
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-delete.html
#
def delete_vault(name,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}"
request(
:expects => 204,
:idempotent => true,
:headers => {},
:method => :delete,
:path => path
)
end
end
end
end
end

33
lib/fog/aws/requests/glacier/delete_vault_notification_configuration.rb Normal file
View file

@ -0,0 +1,33 @@
module Fog
module AWS
class Glacier
class Real
# Delete vault's notification configuration
#
# ==== Parameters
# * name<~String> Name of the vault
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-notifications-delete.html
#
def delete_vault_notification_configuration(name,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/notification-configuration"
request(
:expects => 204,
:idempotent => true,
:headers => {},
:method => :delete,
:path => path
)
end
end
end
end
end

35
lib/fog/aws/requests/glacier/describe_job.rb Normal file
View file

@ -0,0 +1,35 @@
module Fog
module AWS
class Glacier
class Real
# Complete an upload
#
# ==== Parameters
# * name<~String> Name of the vault
# * job_id<~String> The id of the job
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-describe-job-get.html
#
def describe_job(vault_name, job_id, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/jobs/#{job_id}"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path
)
end
end
end
end
end

34
lib/fog/aws/requests/glacier/describe_vault.rb Normal file
View file

@ -0,0 +1,34 @@
module Fog
module AWS
class Glacier
class Real
# This operation returns information about a vault
#
# ==== Parameters
# * name<~String> Vault name
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-get.html
#
def describe_vault(name,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path
)
end
end
end
end
end

41
lib/fog/aws/requests/glacier/get_job_output.rb Normal file
View file

@ -0,0 +1,41 @@
module Fog
module AWS
class Glacier
class Real
# Get the output from a job
#
# ==== Parameters
# * name<~String> Name of the vault
# * job_id<~String> The id of the job
# * options<~Hash>
# * Range<~Range> The range to retrieve
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# * response_block<~Proc> Proc to use for streaming the response
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-job-output-get.html
#
def get_job_output(vault_name, job_id, options={})
account_id = options.delete('account_id') || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/jobs/#{job_id}/output"
headers = {}
if range = options.delete('Range')
headers['Range'] = "bytes=#{range.begin}-#{range.end}"
end
request(
options.merge(
:expects => [200,206],
:idempotent => true,
:headers => headers,
:method => :get,
:path => path
))
end
end
end
end
end

34
lib/fog/aws/requests/glacier/get_vault_notification_configuration.rb Normal file
View file

@ -0,0 +1,34 @@
module Fog
module AWS
class Glacier
class Real
# Get a vault's notification configuration
#
# ==== Parameters
# * name<~String> Name of the vault
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-notifications-get.html
#
def get_vault_notification_configuration(name,options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/notification-configuration"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path
)
end
end
end
end
end

41
lib/fog/aws/requests/glacier/initiate_job.rb Normal file
View file

@ -0,0 +1,41 @@
module Fog
module AWS
class Glacier
class Real
# This operation initates a multipart upload of an archive to a vault
#
# ==== Parameters
# * name<~String> The vault name
# * job_specification<~Hash> A specification of the job
# * Type<~String> The job type. Mandatory. Values: archive-retrieval, inventory-retrieval
# * Description<~String> The job description
# * ArchiveId<~String> The id of the archive to retrieve (only for Type==archive-retrieval)
# * Format<~String> The format to return (only for inventory retrieval). Values: CSV, JSON
# * SNSTopic<String> ARN of a topic to publish to when the job is complete
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-initiate-job-post.html
#
def initiate_job(name, job_specification, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/jobs"
request({
:expects => 202,
:headers => {},
:method => 'POST',
:path => path,
:body => Fog::JSON.encode(job_specification)
})
end
end
end
end
end

38
lib/fog/aws/requests/glacier/initiate_multipart_upload.rb Normal file
View file

@ -0,0 +1,38 @@
module Fog
module AWS
class Glacier
class Real
# This operation initates a multipart upload of an archive to a vault
#
# ==== Parameters
# * name<~String> The vault name
# * part_size<~Integer> The part size to use. Must be a power of 2 multiple of 1MB (1,2,4,8,16,...)
# * options<~Hash>
# * description<~String> - The archive description
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-initiate-upload.html
#
def initiate_multipart_upload(name, part_size, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/multipart-uploads"
headers = {'x-amz-part-size' => part_size.to_s}
headers['x-amz-archive-description'] = options['description'] if options['description']
request(
:expects => 201,
:headers => headers,
:method => 'POST',
:path => path
)
end
end
end
end
end

39
lib/fog/aws/requests/glacier/list_jobs.rb Normal file
View file

@ -0,0 +1,39 @@
module Fog
module AWS
class Glacier
class Real
# lists in-progress and recently jobs for the specified vault
#
# ==== Parameters
# * name<~String> Name of the vault
# * options<~Hash>
# * completed<~Boolean>Specifies the state of the jobs to return. You can specify true or false
# * statuscode<~String> Filter returned jobs by status (InProgress, Succeeded, or Failed)
# * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
# * marker<~String> - marker used for pagination
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# ==== See Also
#http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-jobs-get.html
#
def list_jobs(vault_name, options={})
account_id = options.delete('account_id') || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/jobs"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path,
:query => options
)
end
end
end
end
end

37
lib/fog/aws/requests/glacier/list_multipart_uploads.rb Normal file
View file

@ -0,0 +1,37 @@
module Fog
module AWS
class Glacier
class Real
# lists in-progress multipart uploads for the specified vault
#
# ==== Parameters
# * name<~String> Name of the vault
# * options<~Hash>
# * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
# * marker<~String> - marker used for pagination
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-uploads.html
#
def list_multipart_uploads(vault_name, options={})
account_id = options.delete('account_id') || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path,
:query => options
)
end
end
end
end
end

37
lib/fog/aws/requests/glacier/list_parts.rb Normal file
View file

@ -0,0 +1,37 @@
module Fog
module AWS
class Glacier
class Real
# lists the parts of an archive that have been uploaded in a specific multipart upload
#
# ==== Parameters
# * name<~String> Name of the vault
# * upload_id<~String> The id of the upload
# * options<~Hash>
# * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
# * marker<~String> - marker used for pagination
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-list-parts.html
#
def list_parts(vault_name, upload_id, options={})
account_id = options.delete('account_id') || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => :get,
:path => path,
:query => options
)
end
end
end
end
end

36
lib/fog/aws/requests/glacier/list_vaults.rb Normal file
View file

@ -0,0 +1,36 @@
module Fog
module AWS
class Glacier
class Real
# This operation lists all vaults owned by the calling user’s account.
#
# ==== Parameters
# * options<~Hash>
# * limit<~Integer> - The maximum number of items returned in the response. (default 1000)
# * marker<~String> - marker used for pagination
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vaults-get.html
#
def list_vaults(options={})
account_id = options.delete('account_id') || '-'
path = "/#{account_id}/vaults"
request(
:expects => 200,
:idempotent => true,
:headers => {},
:method => 'GET',
:path => path,
:query => options
)
end
end
end
end
end

37
lib/fog/aws/requests/glacier/set_vault_notification_configuration.rb Normal file
View file

@ -0,0 +1,37 @@
module Fog
module AWS
class Glacier
class Real
# Set a vault's notification configuration
#
# ==== Parameters
# * name<~String> Name of the vault
# * SnsTopic<~String> ARN of the topic to notify
# * events<~Array> Events you wish to receive. Valid events are ArchiveRetrievalCompleted, InventoryRetrievalCompleted
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
# * body<~Hash>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-vault-notifications-put.html
#
def set_vault_notification_configuration(name,sns_topic, events, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(name)}/notification-configuration"
request(
:expects => 204,
:idempotent => true,
:headers => {},
:method => :put,
:path => path,
:body => Fog::JSON.encode('SNSTopic' => sns_topic, 'Events' => events)
)
end
end
end
end
end

46
lib/fog/aws/requests/glacier/upload_part.rb Normal file
View file

@ -0,0 +1,46 @@
module Fog
module AWS
class Glacier
class Real
# Upload an archive
#
# ==== Parameters
# * name<~String> Name of the vault to upload to
# * uploadId<~String> Id of the upload
# * body<~String> The data to upload
# * offset<~Integer> The offset of the data within the archive
# * hash<~String> The tree hash for this part
# * options<~Hash>
# * account_id<~String> - The AWS account id. Defaults to the account owning the credentials making the request
# ==== Returns
# * response<~Excon::Response>:
#
# ==== See Also
# http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-upload-part.html
#
def upload_part(vault_name, upload_id, body, offset, hash, options={})
account_id = options['account_id'] || '-'
path = "/#{account_id}/vaults/#{Fog::AWS.escape(vault_name)}/multipart-uploads/#{Fog::AWS.escape(upload_id)}"
headers = {
'Content-Length' => body.bytesize.to_s,
'Content-Range' => "bytes #{offset}-#{offset+body.bytesize-1}/*",
'x-amz-content-sha256' => Digest::SHA256.hexdigest(body),
'x-amz-sha256-tree-hash' => hash
}
request(
:expects => 204,
:idempotent => true,
:headers => headers,
:method => :put,
:path => path,
:body => body
)
end
end
end
end
end

73
lib/fog/aws/signaturev4.rb Normal file
View file

@ -0,0 +1,73 @@
# See http://docs.amazonwebservices.com/general/latest/gr/signature-version-4.html
#
module Fog
module AWS
class SignatureV4
def initialize(aws_access_key_id, secret_key, region,service)
@region = region
@service = service
@aws_access_key_id = aws_access_key_id
@hmac = Fog::HMAC.new('sha256', 'AWS4' + secret_key)
end
def sign(params, date)
canonical_request = <<-DATA
#{params[:method].to_s.upcase}
#{params[:path]}
#{canonical_query_string(params[:query])}
#{canonical_headers(params[:headers])}
#{signed_headers(params[:headers])}
#{Digest::SHA256.hexdigest(params[:body] || '')}
DATA
canonical_request.chop!
credential_scope = "#{date.utc.strftime('%Y%m%d')}/#{@region}/#{@service}/aws4_request"
string_to_sign = <<-DATA
AWS4-HMAC-SHA256
#{date.to_iso8601_basic}
#{credential_scope}
#{Digest::SHA256.hexdigest(canonical_request)}
DATA
string_to_sign.chop!
signature = derived_hmac(date).sign(string_to_sign)
"AWS4-HMAC-SHA256 Credential=#{@aws_access_key_id}/#{credential_scope}, SignedHeaders=#{signed_headers(params[:headers])}, Signature=#{signature.unpack('H*').first}"
end
protected
def canonical_query_string(query)
canonical_query_string = []
for key in (query || {}).keys.sort
component = "#{Fog::AWS.escape(key)}=#{Fog::AWS.escape(query[key].to_s)}"
canonical_query_string << component
end
canonical_query_string.join("&")
end
def canonical_headers(headers)
canonical_headers = ''
for key in headers.keys.sort
canonical_headers << "#{key.downcase}:#{headers[key].to_s.strip}\n"
end
canonical_headers
end
def signed_headers(headers)
headers.keys.sort.collect {|key| key.downcase}.join(';')
end
def derived_hmac(date)
kDate = @hmac.sign(date.utc.strftime('%Y%m%d'))
kRegion = Fog::HMAC.new('sha256', kDate).sign(@region)
kService = Fog::HMAC.new('sha256', kRegion).sign(@service)
kSigning = Fog::HMAC.new('sha256', kService).sign('aws4_request')
Fog::HMAC.new('sha256', kSigning)
end
end
end
end

2
lib/fog/bin/aws.rb
View file

@ -25,6 +25,8 @@ class AWS < Fog::Bin
Fog::AWS::ELB
when :emr
Fog::AWS::EMR
when :glacier
Fog::AWS::Glacier
when :iam
Fog::AWS::IAM
when :sdb, :simpledb

4
lib/fog/core/time.rb
View file

@ -22,5 +22,9 @@ module Fog
self.utc.strftime("#{DAYS[self.utc.wday]}, %d #{MONTHS[self.utc.month - 1]} %Y %H:%M:%S +0000")
end
def to_iso8601_basic
self.utc.strftime('%Y%m%dT%H%M%SZ')
end
end
end

47
tests/aws/models/glacier/model_tests.rb Normal file
View file

@ -0,0 +1,47 @@
Shindo.tests('AWS::Glacier | models', ['aws', 'glacier']) do
pending if Fog.mocking?
tests('success') do
tests('vaults') do
tests('getting a missing vault') do
returns(nil) { Fog::AWS[:glacier].vaults.get('no-such-vault') }
end
vault = nil
tests('creating a vault') do
vault = Fog::AWS[:glacier].vaults.create :id => 'Fog-Test-Vault'
tests("id is Fog-Test-Vault").returns('Fog-Test-Vault') {vault.id}
end
tests('all') do
tests('contains vault').returns(true) { Fog::AWS[:glacier].vaults.collect {|vault| vault.id}.include?(vault.id)}
end
tests('destroy') do
vault.destroy
tests('removes vault').returns(nil) {Fog::AWS[:glacier].vaults.get(vault.id)}
end
end
tests("archives") do
vault = Fog::AWS[:glacier].vaults.create :id => 'Fog-Test-Vault-upload'
tests('create') do
archive = vault.archives.create(:body => 'data')
tests('sets id').returns(true) {!archive.id.nil?}
archive.destroy
end
tests('create multipart') do
body = StringIO.new('x'*1024*1024*2)
body.rewind
archive = vault.archives.create(:body => body, :multipart_chunk_size => 1024*1024)
tests('sets id').returns(true) {!archive.id.nil?}
archive.destroy
end
end
vault = Fog::AWS[:glacier].vaults.create :id => 'Fog-Test-Vault'
tests("jobs") do
tests('all').returns([]) {vault.jobs}
end
vault.destroy
end
end

1
tests/aws/requests/compute/tag_tests.rb
View file

@ -74,4 +74,5 @@ Shindo.tests('Fog::Compute[:aws] | tag requests', ['aws']) do
end
end
Fog::Compute::AWS::Mock.reset if Fog.mocking?
end

13
tests/aws/requests/glacier/archive_tests.rb Normal file
View file

@ -0,0 +1,13 @@
Shindo.tests('AWS::Glacier | glacier archive tests', ['aws']) do
pending if Fog.mocking?
Fog::AWS[:glacier].create_vault('Fog-Test-Vault-upload')
tests('single part upload') do
id = Fog::AWS[:glacier].create_archive('Fog-Test-Vault-upload', 'data body').headers['x-amz-archive-id']
Fog::AWS[:glacier].delete_archive('Fog-Test-Vault-upload', id)
end
#amazon won't let us delete the vault because it has been written to in the past day
end

30
tests/aws/requests/glacier/multipart_upload_tests.rb Normal file
View file

@ -0,0 +1,30 @@
Shindo.tests('AWS::Glacier | glacier archive tests', ['aws']) do
pending if Fog.mocking?
Fog::AWS[:glacier].create_vault('Fog-Test-Vault-upload')
tests('initiate and abort') do
id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id']
returns(true){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)}
Fog::AWS[:glacier].abort_multipart_upload('Fog-Test-Vault-upload', id)
returns(false){ Fog::AWS[:glacier].list_multipart_uploads('Fog-Test-Vault-upload').body['UploadsList'].collect {|item| item['MultipartUploadId']}.include?(id)}
end
tests('do multipart upload') do
hash = Fog::AWS::Glacier::TreeHash.new
id = Fog::AWS[:glacier].initiate_multipart_upload('Fog-Test-Vault-upload', 1024*1024).headers['x-amz-multipart-upload-id']
part = 't'*1024*1024
hash_for_part = hash.add_part(part)
Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part, 0, hash_for_part)
part_2 = 'u'*1024*1024
hash_for_part_2 = hash.add_part(part_2)
Fog::AWS[:glacier].upload_part('Fog-Test-Vault-upload', id, part_2, 1024*1024, hash_for_part_2)
archive = Fog::AWS[:glacier].complete_multipart_upload('Fog-Test-Vault-upload', id, 2*1024*1024, hash.hexdigest).headers['x-amz-archive-id']
Fog::AWS[:glacier].delete_archive('Fog-Test-Vault-upload', archive)
#amazon won't let us delete the vault because it has been written to in the past day
end
end

63
tests/aws/requests/glacier/tree_hash_tests.rb Normal file
View file

@ -0,0 +1,63 @@
Shindo.tests('AWS::Glacier | glacier tree hash calcuation', ['aws']) do
tests('tree_hash(single part < 1MB)') do
returns(Digest::SHA256.hexdigest('')) { Fog::AWS::Glacier::TreeHash.digest('')}
end
tests('tree_hash(multibyte characters)') do
body = ("\xC2\xA1" * 1024*1024)
body.force_encoding('UTF-8') if body.respond_to? :encoding
expected = Digest::SHA256.hexdigest(
Digest::SHA256.digest("\xC2\xA1" * 1024*512) + Digest::SHA256.digest("\xC2\xA1" * 1024*512)
)
returns(expected) { Fog::AWS::Glacier::TreeHash.digest(body)}
end
tests('tree_hash(power of 2 number of parts)') do
body = ('x' * 1024*1024) + ('y'*1024*1024) + ('z'*1024*1024) + ('t'*1024*1024)
expected = Digest::SHA256.hexdigest(
Digest::SHA256.digest(
Digest::SHA256.digest('x' * 1024*1024) + Digest::SHA256.digest('y' * 1024*1024)
) +
Digest::SHA256.digest(
Digest::SHA256.digest('z' * 1024*1024) + Digest::SHA256.digest('t' * 1024*1024)
)
)
returns(expected) { Fog::AWS::Glacier::TreeHash.digest(body)}
end
tests('tree_hash(non power of 2 number of parts)') do
body = ('x' * 1024*1024) + ('y'*1024*1024) + ('z'*1024*1024)
expected = Digest::SHA256.hexdigest(
Digest::SHA256.digest(
Digest::SHA256.digest('x' * 1024*1024) + Digest::SHA256.digest('y' * 1024*1024)
) +
Digest::SHA256.digest('z' * 1024*1024)
)
returns(expected) { Fog::AWS::Glacier::TreeHash.digest(body)}
end
tests('multipart') do
tree_hash = Fog::AWS::Glacier::TreeHash.new
part = ('x' * 1024*1024) + ('y'*1024*1024)
returns(Fog::AWS::Glacier::TreeHash.digest(part)) { tree_hash.add_part part }
tree_hash.add_part('z'* 1024*1024 + 't'*1024*1024)
expected = Digest::SHA256.hexdigest(
Digest::SHA256.digest(
Digest::SHA256.digest('x' * 1024*1024) + Digest::SHA256.digest('y' * 1024*1024)
) +
Digest::SHA256.digest(
Digest::SHA256.digest('z' * 1024*1024) + Digest::SHA256.digest('t' * 1024*1024)
)
)
returns(expected) { tree_hash.hexdigest}
end
end

35
tests/aws/requests/glacier/vault_tests.rb Normal file
View file

@ -0,0 +1,35 @@
Shindo.tests('AWS::Glacier | glacier vault requests', ['aws']) do
pending if Fog.mocking?
topic_arn = Fog::AWS[:sns].create_topic( 'fog_test_glacier_topic').body['TopicArn']
Fog::AWS[:glacier].create_vault('Fog-Test-Vault')
tests('list_vaults') do
returns(true){Fog::AWS[:glacier].list_vaults.body['VaultList'].collect {|data| data['VaultName']}.include?('Fog-Test-Vault')}
end
tests('describe_vault') do
returns('Fog-Test-Vault'){Fog::AWS[:glacier].describe_vault('Fog-Test-Vault').body['VaultName']}
end
tests('set_vault_notification_configuration') do
Fog::AWS[:glacier].set_vault_notification_configuration 'Fog-Test-Vault', topic_arn, ['ArchiveRetrievalCompleted']
end
tests('get_vault_notification_configuration') do
returns('SNSTopic' => topic_arn, 'Events' => ['ArchiveRetrievalCompleted']){ Fog::AWS[:glacier].get_vault_notification_configuration( 'Fog-Test-Vault').body}
end
tests('delete_vault_notification_configuration') do
Fog::AWS[:glacier].delete_vault_notification_configuration( 'Fog-Test-Vault')
raises(Excon::Errors::NotFound){Fog::AWS[:glacier].get_vault_notification_configuration( 'Fog-Test-Vault')}
end
tests('delete_vault') do
Fog::AWS[:glacier].delete_vault( 'Fog-Test-Vault')
raises(Excon::Errors::NotFound){Fog::AWS[:glacier].describe_vault( 'Fog-Test-Vault')}
end
Fog::AWS[:sns].delete_topic topic_arn
end

32
tests/aws/signaturev4_tests.rb Normal file
View file

@ -0,0 +1,32 @@
# encoding: utf-8
Shindo.tests('AWS | signaturev4', ['aws']) do
# These testcases are from http://docs.amazonwebservices.com/general/latest/gr/signature-v4-test-suite.html
@signer = Fog::AWS::SignatureV4.new('AKIDEXAMPLE', 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY', 'us-east-1','host')
Fog::Time.now = ::Time.utc(2011,9,9,23,36,0)
#get-vanilla
returns(@signer.sign({:headers => {'Host' => 'host.foo.com', 'Date' => 'Mon, 09 Sep 2011 23:36:00 GMT'}, :method => :get, :path => '/'}, Fog::Time.now)) do
'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470'
end
#get-vanilla-query-order-key
returns(@signer.sign({:query => {'a' => 'foo', 'b' => 'foo'}, :headers => {'Host' => 'host.foo.com', 'Date' => 'Mon, 09 Sep 2011 23:36:00 GMT'}, :method => :get, :path => '/'}, Fog::Time.now)) do
'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b'
end
#get-unreserved
returns(@signer.sign({:headers => {'Host' => 'host.foo.com', 'Date' => 'Mon, 09 Sep 2011 23:36:00 GMT'}, :method => :get, :path => '/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'}, Fog::Time.now)) do
'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e'
end
#post-x-www-form-urlencoded-parameter
returns(@signer.sign({:headers => {'Content-type' => 'application/x-www-form-urlencoded; charset=utf8', 'Host' => 'host.foo.com', 'Date' => 'Mon, 09 Sep 2011 23:36:00 GMT'}, :method => :post, :path => '/',
:body => 'foo=bar'}, Fog::Time.now)) do
'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71'
end
Fog::Time.now = ::Time.now
end