2014-12-30 17:25:09 -05:00
|
|
|
module Fog
|
|
|
|
module AWS
|
|
|
|
class Glacier < Fog::Service
|
|
|
|
extend Fog::AWS::CredentialFetcher::ServiceMethods
|
|
|
|
|
|
|
|
requires :aws_access_key_id, :aws_secret_access_key
|
|
|
|
recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
|
|
|
|
|
|
|
|
request_path 'fog/aws/requests/glacier'
|
|
|
|
|
|
|
|
request :abort_multipart_upload
|
|
|
|
request :complete_multipart_upload
|
|
|
|
request :create_archive
|
|
|
|
request :create_vault
|
|
|
|
request :delete_archive
|
|
|
|
request :delete_vault
|
|
|
|
request :delete_vault_notification_configuration
|
|
|
|
request :describe_job
|
|
|
|
request :describe_vault
|
|
|
|
request :get_job_output
|
|
|
|
request :get_vault_notification_configuration
|
|
|
|
request :initiate_job
|
|
|
|
request :initiate_multipart_upload
|
|
|
|
request :list_jobs
|
|
|
|
request :list_multipart_uploads
|
|
|
|
request :list_parts
|
|
|
|
request :list_vaults
|
|
|
|
request :set_vault_notification_configuration
|
|
|
|
request :upload_part
|
|
|
|
|
|
|
|
model_path 'fog/aws/models/glacier'
|
|
|
|
model :vault
|
|
|
|
collection :vaults
|
|
|
|
|
|
|
|
MEGABYTE = 1024*1024
|
|
|
|
|
|
|
|
class TreeHash
|
|
|
|
def self.digest(body)
|
|
|
|
new.add_part(body)
|
|
|
|
end
|
|
|
|
|
2019-05-21 03:47:02 -04:00
|
|
|
def initialize
|
2019-05-29 15:05:55 -04:00
|
|
|
@last_chunk_digest = nil # Digest OBJECT for last chunk (Digest::SHA256)
|
|
|
|
@last_chunk_digest_temp = nil # Digest VALUE for last chunk
|
|
|
|
@last_chunk_length = 0 # Length of last chunk, always smaller than 1MB.
|
2019-05-21 03:47:02 -04:00
|
|
|
@digest_stack = []
|
2019-05-29 15:05:55 -04:00
|
|
|
# First position on stack corresponds to 1MB, second 2MB, third 4MB, fourt 8MB and so on.
|
|
|
|
# In any time, the size of all already added parts is equal to sum of all existing (non-nil)
|
|
|
|
# positions multiplied by that number, plus last_chunk_length for the remainder smaller than
|
|
|
|
# one megabyte. So, if last_chunk_length is half megabyte, stack[0] is filled, stack[1] and
|
|
|
|
# stack[2] empty and stack[3] filled, the size is 0.5MB + 1x1MB + 0x2MB + 0x4MB + 1x8MB = 9.5MB.
|
2019-05-21 03:47:02 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def update_digest_stack(digest, stack)
|
|
|
|
stack.each_with_index{|s,i|
|
|
|
|
if s
|
|
|
|
digest = Digest::SHA256.digest(s + digest)
|
|
|
|
stack[i] = nil
|
|
|
|
else
|
2019-05-29 15:05:55 -04:00
|
|
|
stack[i] = digest # Update this position with value obtained in previous run of cycle.
|
2019-05-21 03:47:02 -04:00
|
|
|
digest = nil
|
|
|
|
break
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
2019-05-21 03:47:02 -04:00
|
|
|
}
|
|
|
|
stack << digest if digest
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
|
|
|
|
2019-05-21 03:47:02 -04:00
|
|
|
def reduce_digest_stack(digest, stack)
|
|
|
|
stack.each_with_index{|s,i|
|
|
|
|
unless digest
|
|
|
|
digest = stack[i]
|
|
|
|
next
|
|
|
|
end
|
|
|
|
if stack[i]
|
|
|
|
digest = Digest::SHA256.digest(stack[i] + digest)
|
|
|
|
end
|
|
|
|
}
|
|
|
|
digest
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def add_part(bytes)
|
|
|
|
part = self.digest_for_part(bytes)
|
|
|
|
part.unpack('H*').first
|
|
|
|
end
|
|
|
|
|
2019-05-21 03:47:02 -04:00
|
|
|
def prepare_body_for_slice(body)
|
2014-12-30 17:25:09 -05:00
|
|
|
if body.respond_to? :byteslice
|
2019-05-21 03:47:02 -04:00
|
|
|
r = yield(body, :byteslice)
|
2014-12-30 17:25:09 -05:00
|
|
|
else
|
|
|
|
if body.respond_to? :encoding
|
|
|
|
old_encoding = body.encoding
|
|
|
|
body.force_encoding('BINARY')
|
|
|
|
end
|
2019-05-21 03:47:02 -04:00
|
|
|
r = yield(body, :slice)
|
2014-12-30 17:25:09 -05:00
|
|
|
if body.respond_to? :encoding
|
|
|
|
body.force_encoding(old_encoding)
|
|
|
|
end
|
|
|
|
end
|
2019-05-21 03:47:02 -04:00
|
|
|
r
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
|
|
|
|
2019-05-21 03:47:02 -04:00
|
|
|
def digest_for_part(body)
|
|
|
|
part_stack = []
|
|
|
|
part_temp = nil
|
|
|
|
body_size = body.bytesize
|
|
|
|
prepare_body_for_slice(body) {|body, slice|
|
|
|
|
start_offset = 0
|
|
|
|
if @last_chunk_length != 0
|
|
|
|
start_offset = MEGABYTE - @last_chunk_length
|
|
|
|
@last_chunk_hash.update(body.send(slice, 0, start_offset))
|
|
|
|
hash = @last_chunk_hash.digest
|
|
|
|
@last_chunk_digest_temp = hash
|
|
|
|
if body_size > start_offset
|
|
|
|
@last_chunk_length = 0
|
|
|
|
@last_chunk_hash = nil
|
|
|
|
@last_chunk_digest_temp = nil
|
|
|
|
update_digest_stack(hash, @digest_stack)
|
|
|
|
else
|
|
|
|
part_temp = hash
|
|
|
|
@last_chunk_digest_temp = hash
|
|
|
|
@last_chunk_length += body_size
|
|
|
|
next
|
|
|
|
end
|
|
|
|
end
|
|
|
|
whole_chunk_count = (body_size - start_offset) / MEGABYTE
|
|
|
|
whole_chunk_count.times.each {|chunk_index|
|
|
|
|
hash = Digest::SHA256.digest(body.send(slice, start_offset + chunk_index * MEGABYTE, MEGABYTE))
|
|
|
|
update_digest_stack(hash, part_stack)
|
|
|
|
update_digest_stack(hash, @digest_stack)
|
|
|
|
}
|
|
|
|
rest_size = body_size - start_offset - whole_chunk_count * MEGABYTE
|
|
|
|
if rest_size > 0 || whole_chunk_count == 0
|
|
|
|
@last_chunk_hash = Digest::SHA256.new
|
|
|
|
@last_chunk_length = rest_size
|
|
|
|
@last_chunk_hash.update(body.send(slice, start_offset + whole_chunk_count * MEGABYTE, rest_size))
|
|
|
|
hash = @last_chunk_hash.digest
|
|
|
|
@last_chunk_digest_temp = hash
|
|
|
|
part_temp = hash
|
|
|
|
end
|
|
|
|
}
|
|
|
|
reduce_digest_stack(part_temp, part_stack)
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def digest
|
2019-05-21 03:47:02 -04:00
|
|
|
reduce_digest_stack(@last_chunk_digest_temp, @digest_stack)
|
|
|
|
end
|
|
|
|
|
|
|
|
def hexdigest
|
|
|
|
digest.unpack('H*').first
|
2014-12-30 17:25:09 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
class Mock
|
|
|
|
def initialize(options={})
|
|
|
|
Fog::Mock.not_implemented
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
class Real
|
|
|
|
include Fog::AWS::CredentialFetcher::ConnectionMethods
|
|
|
|
# Initialize connection to Glacier
|
|
|
|
#
|
|
|
|
# ==== Notes
|
|
|
|
# options parameter must include values for :aws_access_key_id and
|
|
|
|
# :aws_secret_access_key in order to create a connection
|
|
|
|
#
|
|
|
|
# ==== Examples
|
|
|
|
# ses = SES.new(
|
|
|
|
# :aws_access_key_id => your_aws_access_key_id,
|
|
|
|
# :aws_secret_access_key => your_aws_secret_access_key
|
|
|
|
# )
|
|
|
|
#
|
|
|
|
# ==== Parameters
|
|
|
|
# * options<~Hash> - config arguments for connection. Defaults to {}.
|
|
|
|
# * region<~String> - optional region to use. For instance, 'us-east-1' and etc.
|
|
|
|
#
|
|
|
|
# ==== Returns
|
2015-01-02 12:34:40 -05:00
|
|
|
# * Glacier object with connection to AWS.
|
2014-12-30 17:25:09 -05:00
|
|
|
def initialize(options={})
|
|
|
|
@use_iam_profile = options[:use_iam_profile]
|
|
|
|
@region = options[:region] || 'us-east-1'
|
|
|
|
|
|
|
|
setup_credentials(options)
|
|
|
|
|
|
|
|
@instrumentor = options[:instrumentor]
|
|
|
|
@instrumentor_name = options[:instrumentor_name] || 'fog.aws.glacier'
|
|
|
|
@connection_options = options[:connection_options] || {}
|
|
|
|
@host = options[:host] || "glacier.#{@region}.amazonaws.com"
|
|
|
|
@version = '2012-06-01'
|
|
|
|
@path = options[:path] || '/'
|
|
|
|
@persistent = options[:persistent] || false
|
|
|
|
@port = options[:port] || 443
|
|
|
|
@scheme = options[:scheme] || 'https'
|
|
|
|
|
|
|
|
@connection = Fog::XML::Connection.new("#{@scheme}://#{@host}:#{@port}#{@path}", @persistent, @connection_options)
|
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
def setup_credentials(options)
|
|
|
|
@aws_access_key_id = options[:aws_access_key_id]
|
|
|
|
@aws_secret_access_key = options[:aws_secret_access_key]
|
|
|
|
@aws_session_token = options[:aws_session_token]
|
|
|
|
@aws_credentials_expire_at = options[:aws_credentials_expire_at]
|
|
|
|
|
|
|
|
@signer = Fog::AWS::SignatureV4.new( @aws_access_key_id, @aws_secret_access_key,@region,'glacier')
|
|
|
|
end
|
|
|
|
|
|
|
|
def request(params, &block)
|
|
|
|
refresh_credentials_if_expired
|
|
|
|
|
|
|
|
date = Fog::Time.now
|
|
|
|
params[:headers]['Date'] = date.to_date_header
|
|
|
|
params[:headers]['x-amz-date'] = date.to_iso8601_basic
|
|
|
|
|
|
|
|
params[:headers]['Host'] = @host
|
|
|
|
params[:headers]['x-amz-glacier-version'] = @version
|
|
|
|
params[:headers]['x-amz-security-token'] = @aws_session_token if @aws_session_token
|
|
|
|
params[:headers]['Authorization'] = @signer.sign params, date
|
|
|
|
|
|
|
|
if @instrumentor
|
|
|
|
@instrumentor.instrument("#{@instrumentor_name}.request", params) do
|
|
|
|
_request(params, &block)
|
|
|
|
end
|
|
|
|
else
|
|
|
|
_request(params, &block)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def _request(params, &block)
|
|
|
|
response = @connection.request(params, &block)
|
|
|
|
if response.headers['Content-Type'] == 'application/json' && response.body.size > 0 #body will be empty if the streaming form has been used
|
|
|
|
response.body = Fog::JSON.decode(response.body)
|
|
|
|
end
|
|
|
|
response
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|