kotovalexarian-likes-github/fog--fog
1
0
Fork 0
mirror of https://github.com/fog/fog.git synced 2022-11-09 13:51:43 -05:00
Code Releases Activity

Add generic support for EMC Atmos.

Browse source 
Refactor ninefold to provide generic Atmos support. Add an additional
argument, when compared to ninefold, which is the endpoint. The
endpoint should be a full URL, e.g.
https://storage.provider.com:1337/atmos. The API path and port are
optional. If the port is not specified, it is inferred from the
protocol.
This commit is contained in:
Timur Alperovich 2012-08-12 16:35:35 -07:00 committed by Timur Alperovich
parent 2981ae8075
commit cc69d10c89
19 changed files with 660 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
lib/fog/atmos.rb Normal file
View file

@ -0,0 +1,11 @@
require 'fog/core'
module Fog
module Atmos
extend Fog::Provider
service(:storage, 'atmos/storage', 'Storage')
end
end

48
lib/fog/atmos/models/storage/directories.rb Normal file
View file

@ -0,0 +1,48 @@
require 'fog/core/collection'
require 'fog/atmos/models/storage/directory'
module Fog
module Storage
class Atmos
class Directories < Fog::Collection
attribute :directory
model Fog::Storage::Atmos::Directory
def all
directory ? ns = directory.key : ns = ''
ns = ns + '/' unless ns =~ /\/$/
data = connection.get_namespace(ns).body[:DirectoryList]
data = {:DirectoryEntry => []} if data.kind_of? String
data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash
dirs = data[:DirectoryEntry].select {|de| de[:FileType] == 'directory'}
dirs.each do |d|
d[:Filename] = ns + d[:Filename] if directory
d[:Filename] += '/' unless d[:Filename] =~ /\/$/
end
load(dirs)
end
def get(key, options = {})
return nil if key == '' # Root dir shouldn't be retrieved like this.
key =~ /\/$/ ? ns = key : ns = key + '/'
res = connection.get_namespace ns
emc_meta = res.headers['x-emc-meta']
obj_id = emc_meta.scan(/objectid=(\w+),/).flatten[0]
new(:objectid => obj_id, :key => ns)
rescue Fog::Storage::Atmos::NotFound
nil
end
def new(attributes ={})
attributes = {:directory => directory}.merge(attributes) if directory
super(attributes)
end
end
end
end
end

53
lib/fog/atmos/models/storage/directory.rb Normal file
View file

@ -0,0 +1,53 @@
require 'fog/core/model'
module Fog
module Storage
class Atmos
class Directory < Fog::Model
identity :key, :aliases => :Filename
attribute :objectid, :aliases => :ObjectID
def files
@files ||= begin
Fog::Storage::Atmos::Files.new(
:directory => self,
:connection => connection
)
end
end
def directories
@directories ||= begin
Fog::Storage::Atmos::Directories.new(
:directory => self,
:connection => connection
)
end
end
def save
self.key = attributes[:directory].key + key if attributes[:directory]
self.key = key + '/' unless key =~ /\/$/
res = connection.post_namespace key
reload
end
def destroy(opts={})
if opts[:recursive]
files.each {|f| f.destroy }
directories.each do |d|
d.files.each {|f| f.destroy }
d.destroy(opts)
end
end
connection.delete_namespace key
end
end
end
end
end

107
lib/fog/atmos/models/storage/file.rb Normal file
View file

@ -0,0 +1,107 @@
require 'fog/core/model'
module Fog
module Storage
class Atmos
class File < Fog::Model
identity :key, :aliases => :Filename
attribute :content_length, :aliases => ['bytes', 'Content-Length'], :type => :integer
attribute :content_type, :aliases => ['content_type', 'Content-Type']
attribute :objectid, :aliases => :ObjectID
def body
attributes[:body] ||= if objectid
collection.get(identity).body
else
''
end
end
def body=(new_body)
attributes[:body] = new_body
end
def directory
@directory
end
def copy(target_directory_key, target_file_key, options={})
target_directory = connection.directories.new(:key => target_directory_key)
target_directory.files.create(
:key => target_file_key,
:body => body
)
end
def destroy
requires :directory, :key
connection.delete_namespace([directory.key, key].join('/'))
true
end
# def owner=(new_owner)
# if new_owner
# attributes[:owner] = {
# :display_name => new_owner['DisplayName'],
# :id => new_owner['ID']
# }
# end
# end
def public=(new_public)
# NOOP - we don't need to flag files as public, getting the public URL for a file handles it.
end
# By default, expire in 5 years
def public_url(expires = (Time.now + 5 * 365 * 24 * 60 * 60))
requires :objectid
# TODO - more efficient method to get this?
storage = Fog::Storage.new(:provider => 'Atmos')
uri = URI::HTTP.build(:scheme => @prefix, :host => @storage_host, :port => @storage_port.to_i, :path => "/rest/objects/#{objectid}" )
Fog::Storage.new(:provider => 'Atmos').uid
sb = "GET\n"
sb += uri.path.downcase + "\n"
sb += storage.uid + "\n"
sb += String(expires.to_i())
signature = storage.sign( sb )
uri.query = "uid=#{CGI::escape(storage.uid)}&expires=#{expires.to_i()}&signature=#{CGI::escape(signature)}"
uri.to_s
end
def save(options = {})
requires :body, :directory, :key
directory.kind_of?(Directory) ? ns = directory.key : ns = directory
ns += key
options[:headers] ||= {}
options[:headers]['Content-Type'] = content_type if content_type
options[:body] = body
begin
data = connection.post_namespace(ns, options)
self.objectid = data.headers['location'].split('/')[-1]
rescue => error
if error.message =~ /The resource you are trying to create already exists./
data = connection.put_namespace(ns, options)
else
raise error
end
end
# merge_attributes(data.headers)
true
end
private
def directory=(new_directory)
@directory = new_directory
end
end
end
end
end

73
lib/fog/atmos/models/storage/files.rb Normal file
View file

@ -0,0 +1,73 @@
require 'fog/core/collection'
require 'fog/atmos/models/storage/file'
module Fog
module Storage
class Atmos
class Files < Fog::Collection
attribute :directory
attribute :limit
attribute :marker
attribute :path
attribute :prefix
model Fog::Storage::Atmos::File
def all(options = {})
requires :directory
directory ? ns = directory.key : ns = ''
ns = ns + '/' unless ns =~ /\/$/
data = connection.get_namespace(ns).body[:DirectoryList]
data = {:DirectoryEntry => []} if data.kind_of? String
data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash
files = data[:DirectoryEntry].select {|de| de[:FileType] == 'regular'}
files.each do |s|
s[:directory] = directory
end
# TODO - Load additional file meta?
load(files)
end
def get(key, &block)
requires :directory
data = connection.get_namespace(directory.key + key, :parse => false)#, &block)
file_data = data.headers.merge({
:body => data.body,
:key => key
})
new(file_data)
rescue Fog::Storage::Atmos::NotFound
nil
end
def get_url(key)
requires :directory
if self.directory.public_url
"#{self.directory.public_url}/#{key}"
end
end
def head(key, options = {})
requires :directory
data = connection.head_namespace(directory.key + key, :parse => false)
file_data = data.headers.merge({
:body => data.body,
:key => key
})
new(file_data)
rescue Fog::Storage::Atmos::NotFound
nil
end
def new(attributes = {})
requires :directory
super({ :directory => directory }.merge!(attributes))
end
end
end
end
end

19
lib/fog/atmos/requests/storage/delete_namespace.rb Normal file
View file

@ -0,0 +1,19 @@
module Fog
module Storage
class Atmos
class Real
def delete_namespace(namespace = '', options = {})
options = options.reject {|key, value| value.nil?}
request({
:expects => 204,
:method => 'DELETE',
:path => "namespace/" + namespace,
:query => options
}.merge(options))
end
end
end
end
end

20
lib/fog/atmos/requests/storage/get_namespace.rb Normal file
View file

@ -0,0 +1,20 @@
module Fog
module Storage
class Atmos
class Real
def get_namespace(namespace = '', options = {})
options = options.reject {|key, value| value.nil?}
request({
:expects => 200,
:method => 'GET',
:path => "namespace/" + namespace,
:query => {},
:parse => true
}.merge(options))
end
end
end
end
end

20
lib/fog/atmos/requests/storage/head_namespace.rb Normal file
View file

@ -0,0 +1,20 @@
module Fog
module Storage
class Atmos
class Real
def head_namespace(namespace = '', options = {})
options = options.reject {|key, value| value.nil?}
request({
:expects => 200,
:method => 'HEAD',
:path => "namespace/" + namespace,
:query => {},
:parse => true
}.merge(options))
end
end
end
end
end

20
lib/fog/atmos/requests/storage/post_namespace.rb Normal file
View file

@ -0,0 +1,20 @@
module Fog
module Storage
class Atmos
class Real
def post_namespace(namespace = '', options = {})
options = options.reject {|key, value| value.nil?}
request({
:expects => 201,
:method => 'POST',
:path => "namespace/" + namespace,
:query => {},
:parse => true
}.merge(options))
end
end
end
end
end

20
lib/fog/atmos/requests/storage/put_namespace.rb Normal file
View file

@ -0,0 +1,20 @@
module Fog
module Storage
class Atmos
class Real
def put_namespace(namespace = '', options = {})
options = options.reject {|key, value| value.nil?}
request({
:expects => 200,
:method => 'PUT',
:path => "namespace/" + namespace,
:query => {},
:parse => true
}.merge(options))
end
end
end
end
end

186
lib/fog/atmos/storage.rb Normal file
View file

@ -0,0 +1,186 @@
require 'fog/atmos'
require 'fog/storage'
module Fog
module Storage
class Atmos < Fog::Service
requires :atmos_storage_endpoint,
:atmos_storage_secret,
:atmos_storage_token
recognizes :persistent
model_path 'fog/atmos/models/storage'
model :directory
collection :directories
model :file
collection :files
request_path 'fog/atmos/requests/storage'
# request :delete_container
request :get_namespace
request :head_namespace
request :post_namespace
request :put_namespace
request :delete_namespace
module Utils
ENDPOINT_REGEX = /(https*):\/\/([a-zA-Z0-9\.\-]+)(:[0-9]+)?(\/.*)?/
def ssl?
protocol = @endpoint.match(ENDPOINT_REGEX)[1]
raise ArgumentError, 'Invalid endpoint URL' if protocol.nil?
return true if protocol == 'https'
return false if protocol == 'http'
raise ArgumentError, "Unknown protocol #{protocol}"
end
def port
port = @endpoint.match(ENDPOINT_REGEX)[3]
return ssl? ? 443 : 80 if port.nil?
port.split(':')[1].to_i
end
def host
@endpoint.match(ENDPOINT_REGEX)[2]
end
def api_path
@endpoint.match(ENDPOINT_REGEX)[4]
end
def setup_credentials(options)
@storage_token = options[:atmos_storage_token]
@storage_secret = options[:atmos_storage_secret]
@storage_secret_decoded = Base64.decode64(@storage_secret)
@endpoint = options[:atmos_storage_endpoint]
@prefix = self.ssl? ? 'https' : 'http'
@storage_host = self.host
@storage_port = self.port
@api_path = self.api_path
end
end
class Mock
include Utils
def initialize(options={})
require 'mime/types'
setup_credentials(options)
end
def request(options)
raise "Atmos Storage mocks not implemented"
end
end
class Real
include Utils
def initialize(options={})
require 'mime/types'
setup_credentials(options)
@connection_options = options[:connection_options] || {}
@hmac = Fog::HMAC.new('sha1', @storage_secret_decoded)
@persistent = options.fetch(:persistent, false)
@connection = Fog::Connection.new("#{@prefix}://#{@storage_host}:#{@storage_port}",
@persistent, @connection_options)
end
def uid
@storage_token#.split('/')[-1]
end
def sign(string)
value = @hmac.sign(string)
Base64.encode64( value ).chomp()
end
def reload
@connection.reset
end
def request(params, &block)
req_path = params[:path]
# Force set host and port
params.merge!({
:host => @storage_host,
:path => "#{@api_path}/rest/#{params[:path]}",
})
# Set default method and headers
params = {:method => 'GET', :headers => {}}.merge params
params[:headers]["Content-Type"] ||= "application/octet-stream"
# Add request date
params[:headers]["date"] = Time.now().httpdate()
params[:headers]["x-emc-uid"] = @storage_token
# Build signature string
signstring = ""
signstring += params[:method]
signstring += "\n"
signstring += params[:headers]["Content-Type"]
signstring += "\n"
if( params[:headers]["range"] )
signstring += params[:headers]["range"]
end
signstring += "\n"
signstring += params[:headers]["date"]
signstring += "\n"
signstring += "/rest/" + URI.unescape( req_path ).downcase
query_str = params[:query].map{|k,v| "#{k}=#{v}"}.join('&')
signstring += '?' + query_str unless query_str.empty?
signstring += "\n"
customheaders = {}
params[:headers].each { |key,value|
case key
when 'x-emc-date', 'x-emc-signature'
#skip
when /^x-emc-/
customheaders[ key.downcase ] = value
end
}
header_arr = customheaders.sort()
header_arr.each { |key,value|
# Values are lowercase and whitespace-normalized
signstring += key + ":" + value.strip.chomp.squeeze( " " ) + "\n"
}
digest = @hmac.sign(signstring.chomp())
signature = Base64.encode64( digest ).chomp()
params[:headers]["x-emc-signature"] = signature
begin
response = @connection.request(params, &block)
rescue Excon::Errors::HTTPStatusError => error
raise case error
when Excon::Errors::NotFound
Fog::Storage::Atmos::NotFound.slurp(error)
else
error
end
end
unless response.body.empty?
if params[:parse]
document = Fog::ToHashDocument.new
parser = Nokogiri::XML::SAX::PushParser.new(document)
parser << response.body
parser.finish
response.body = document.body
end
end
response
end
end
end
end
end

2
lib/fog/bin.rb
View file

@ -4,6 +4,7 @@ module Fog
class << self
def available_providers
Kernel.const_get('Ninefold')
@available_providers ||= Fog.providers.values.select {|provider| Kernel.const_get(provider).available?}.sort
end
@ -56,6 +57,7 @@ module Fog
end
require 'fog/bin/atmos'
require 'fog/bin/aws'
require 'fog/bin/bluebox'
require 'fog/bin/brightbox'

31
lib/fog/bin/atmos.rb Normal file
View file

@ -0,0 +1,31 @@
class Atmos < Fog::Bin
class << self
def class_for(key)
case key
when :storage
Fog::Storage::Atmos
else
raise ArgumentError, "Unsupported #{self} service: #{key}"
end
end
def [](service)
@@connections ||= Hash.new do |hash, key|
hash[key] = case key
when :storage
Fog::Logger.warning("Atmos[:storage] is not recommended, use Storage[:atmos] for portability")
Fog::Storage.new(:provider => 'Atmos')
else
raise ArgumentError, "Unrecognized service: #{service}"
end
end
@@connections[service]
end
def services
Fog::Atmos.services
end
end
end

1
lib/fog/hp/storage.rb
View file

@ -108,6 +108,7 @@ module Fog
def initialize(options={})
require 'mime/types'
puts "Called with #{options}"
@hp_secret_key = options[:hp_secret_key]
@hp_account_id = options[:hp_account_id]
end

1
lib/fog/providers.rb
View file

@ -1,3 +1,4 @@
require 'fog/atmos'
require 'fog/aws'
require 'fog/bluebox'
require 'fog/brightbox'

3
lib/fog/storage.rb
View file

@ -8,6 +8,9 @@ module Fog
def self.new(attributes)
attributes = attributes.dup # prevent delete from having side effects
case provider = attributes.delete(:provider).to_s.downcase.to_sym
when :atmos
require 'fog/atmos/storage'
Fog::Storage::Atmos.new(attributes)
when :aws
require 'fog/aws/storage'
Fog::Storage::AWS.new(attributes)

19
tests/atmos/models/storage/file_update_tests.rb Normal file
View file

@ -0,0 +1,19 @@
Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do
unless Fog.mocking?
@directory = Fog::Storage[:atmos].directories.create(:key => 'updatefiletests')
end
atmos = Fog::Storage[:atmos]
tests("update a file").succeeds do
pending if Fog.mocking?
file = @directory.files.create(:key => 'lorem.txt', :body => lorem_file)
file.body = "xxxxxx"
file.save
end
unless Fog.mocking?
@directory.destroy(:recursive => true)
end
end

23
tests/atmos/models/storage/nested_directories_tests.rb Normal file
View file

@ -0,0 +1,23 @@
Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do
atmos = Fog::Storage[:atmos]
tests("create a directory with a / character").succeeds do
pending if Fog.mocking?
atmos.directories.create(:key => 'sub/path')
end
tests("List of top directory returns sub dir").returns(1) do
pending if Fog.mocking?
atmos.directories.get('sub').directories.count
end
tests("create a directory in a sub dir").returns('sub/path/newdir/') do
pending if Fog.mocking?
atmos.directories.get('sub/path').directories.create(:key => 'newdir').identity
end
tests("Recursively destroy parent dir").succeeds do
pending if Fog.mocking?
atmos.directories.get('sub').destroy(:recursive => true)
end
end

3
tests/helpers/mock_helper.rb
View file

@ -11,6 +11,9 @@ if Fog.mock?
Fog.credentials = {
:aws_access_key_id => 'aws_access_key_id',
:aws_secret_access_key => 'aws_secret_access_key',
:atmos_storage_token => 'atmos_token',
:atmos_storage_secret => 'atmos_secret',
:atmos_storage_endpoint => 'http://atmos.is.cool:1000/test1.0',
:bluebox_api_key => 'bluebox_api_key',
:bluebox_customer_id => 'bluebox_customer_id',
:brightbox_client_id => 'brightbox_client_id',