diff --git a/lib/fog/atmos.rb b/lib/fog/atmos.rb new file mode 100644 index 000000000..5bd43b51b --- /dev/null +++ b/lib/fog/atmos.rb @@ -0,0 +1,11 @@ +require 'fog/core' + +module Fog + module Atmos + + extend Fog::Provider + + service(:storage, 'atmos/storage', 'Storage') + + end +end diff --git a/lib/fog/atmos/models/storage/directories.rb b/lib/fog/atmos/models/storage/directories.rb new file mode 100644 index 000000000..568fbe9c8 --- /dev/null +++ b/lib/fog/atmos/models/storage/directories.rb @@ -0,0 +1,48 @@ +require 'fog/core/collection' +require 'fog/atmos/models/storage/directory' + +module Fog + module Storage + class Atmos + + class Directories < Fog::Collection + + attribute :directory + + model Fog::Storage::Atmos::Directory + + def all + directory ? ns = directory.key : ns = '' + ns = ns + '/' unless ns =~ /\/$/ + data = connection.get_namespace(ns).body[:DirectoryList] + data = {:DirectoryEntry => []} if data.kind_of? String + data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash + dirs = data[:DirectoryEntry].select {|de| de[:FileType] == 'directory'} + dirs.each do |d| + d[:Filename] = ns + d[:Filename] if directory + d[:Filename] += '/' unless d[:Filename] =~ /\/$/ + end + load(dirs) + end + + def get(key, options = {}) + return nil if key == '' # Root dir shouldn't be retrieved like this. + key =~ /\/$/ ? ns = key : ns = key + '/' + res = connection.get_namespace ns + emc_meta = res.headers['x-emc-meta'] + obj_id = emc_meta.scan(/objectid=(\w+),/).flatten[0] + new(:objectid => obj_id, :key => ns) + rescue Fog::Storage::Atmos::NotFound + nil + end + + def new(attributes ={}) + attributes = {:directory => directory}.merge(attributes) if directory + super(attributes) + end + + end + + end + end +end diff --git a/lib/fog/atmos/models/storage/directory.rb b/lib/fog/atmos/models/storage/directory.rb new file mode 100644 index 000000000..5a30a2cfd --- /dev/null +++ b/lib/fog/atmos/models/storage/directory.rb @@ -0,0 +1,53 @@ +require 'fog/core/model' + +module Fog + module Storage + class Atmos + + class Directory < Fog::Model + + identity :key, :aliases => :Filename + attribute :objectid, :aliases => :ObjectID + + def files + @files ||= begin + Fog::Storage::Atmos::Files.new( + :directory => self, + :connection => connection + ) + end + end + + def directories + @directories ||= begin + Fog::Storage::Atmos::Directories.new( + :directory => self, + :connection => connection + ) + end + end + + def save + self.key = attributes[:directory].key + key if attributes[:directory] + self.key = key + '/' unless key =~ /\/$/ + res = connection.post_namespace key + reload + end + + def destroy(opts={}) + if opts[:recursive] + files.each {|f| f.destroy } + directories.each do |d| + d.files.each {|f| f.destroy } + d.destroy(opts) + end + end + connection.delete_namespace key + end + + + end + + end + end +end diff --git a/lib/fog/atmos/models/storage/file.rb b/lib/fog/atmos/models/storage/file.rb new file mode 100644 index 000000000..7b74fc1b9 --- /dev/null +++ b/lib/fog/atmos/models/storage/file.rb @@ -0,0 +1,107 @@ +require 'fog/core/model' + +module Fog + module Storage + class Atmos + + class File < Fog::Model + + identity :key, :aliases => :Filename + + attribute :content_length, :aliases => ['bytes', 'Content-Length'], :type => :integer + attribute :content_type, :aliases => ['content_type', 'Content-Type'] + attribute :objectid, :aliases => :ObjectID + + def body + attributes[:body] ||= if objectid + collection.get(identity).body + else + '' + end + end + + def body=(new_body) + attributes[:body] = new_body + end + + def directory + @directory + end + + def copy(target_directory_key, target_file_key, options={}) + target_directory = connection.directories.new(:key => target_directory_key) + target_directory.files.create( + :key => target_file_key, + :body => body + ) + end + + def destroy + requires :directory, :key + connection.delete_namespace([directory.key, key].join('/')) + true + end + + # def owner=(new_owner) + # if new_owner + # attributes[:owner] = { + # :display_name => new_owner['DisplayName'], + # :id => new_owner['ID'] + # } + # end + # end + + def public=(new_public) + # NOOP - we don't need to flag files as public, getting the public URL for a file handles it. + end + + # By default, expire in 5 years + def public_url(expires = (Time.now + 5 * 365 * 24 * 60 * 60)) + requires :objectid + # TODO - more efficient method to get this? + storage = Fog::Storage.new(:provider => 'Atmos') + uri = URI::HTTP.build(:scheme => @prefix, :host => @storage_host, :port => @storage_port.to_i, :path => "/rest/objects/#{objectid}" ) + Fog::Storage.new(:provider => 'Atmos').uid + + sb = "GET\n" + sb += uri.path.downcase + "\n" + sb += storage.uid + "\n" + sb += String(expires.to_i()) + + signature = storage.sign( sb ) + uri.query = "uid=#{CGI::escape(storage.uid)}&expires=#{expires.to_i()}&signature=#{CGI::escape(signature)}" + uri.to_s + end + + def save(options = {}) + requires :body, :directory, :key + directory.kind_of?(Directory) ? ns = directory.key : ns = directory + ns += key + options[:headers] ||= {} + options[:headers]['Content-Type'] = content_type if content_type + options[:body] = body + begin + data = connection.post_namespace(ns, options) + self.objectid = data.headers['location'].split('/')[-1] + rescue => error + if error.message =~ /The resource you are trying to create already exists./ + data = connection.put_namespace(ns, options) + else + raise error + end + end + # merge_attributes(data.headers) + true + end + + private + + def directory=(new_directory) + @directory = new_directory + end + + end + + end + end +end diff --git a/lib/fog/atmos/models/storage/files.rb b/lib/fog/atmos/models/storage/files.rb new file mode 100644 index 000000000..40c234148 --- /dev/null +++ b/lib/fog/atmos/models/storage/files.rb @@ -0,0 +1,73 @@ +require 'fog/core/collection' +require 'fog/atmos/models/storage/file' + +module Fog + module Storage + class Atmos + + class Files < Fog::Collection + + attribute :directory + attribute :limit + attribute :marker + attribute :path + attribute :prefix + + model Fog::Storage::Atmos::File + + def all(options = {}) + requires :directory + directory ? ns = directory.key : ns = '' + ns = ns + '/' unless ns =~ /\/$/ + data = connection.get_namespace(ns).body[:DirectoryList] + data = {:DirectoryEntry => []} if data.kind_of? String + data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash + files = data[:DirectoryEntry].select {|de| de[:FileType] == 'regular'} + files.each do |s| + s[:directory] = directory + end + # TODO - Load additional file meta? + load(files) + end + + def get(key, &block) + requires :directory + data = connection.get_namespace(directory.key + key, :parse => false)#, &block) + file_data = data.headers.merge({ + :body => data.body, + :key => key + }) + new(file_data) + rescue Fog::Storage::Atmos::NotFound + nil + end + + def get_url(key) + requires :directory + if self.directory.public_url + "#{self.directory.public_url}/#{key}" + end + end + + def head(key, options = {}) + requires :directory + data = connection.head_namespace(directory.key + key, :parse => false) + file_data = data.headers.merge({ + :body => data.body, + :key => key + }) + new(file_data) + rescue Fog::Storage::Atmos::NotFound + nil + end + + def new(attributes = {}) + requires :directory + super({ :directory => directory }.merge!(attributes)) + end + + end + + end + end +end diff --git a/lib/fog/atmos/requests/storage/delete_namespace.rb b/lib/fog/atmos/requests/storage/delete_namespace.rb new file mode 100644 index 000000000..623538025 --- /dev/null +++ b/lib/fog/atmos/requests/storage/delete_namespace.rb @@ -0,0 +1,19 @@ +module Fog + module Storage + class Atmos + class Real + + def delete_namespace(namespace = '', options = {}) + options = options.reject {|key, value| value.nil?} + request({ + :expects => 204, + :method => 'DELETE', + :path => "namespace/" + namespace, + :query => options + }.merge(options)) + end + + end + end + end +end diff --git a/lib/fog/atmos/requests/storage/get_namespace.rb b/lib/fog/atmos/requests/storage/get_namespace.rb new file mode 100644 index 000000000..eb988c4b1 --- /dev/null +++ b/lib/fog/atmos/requests/storage/get_namespace.rb @@ -0,0 +1,20 @@ +module Fog + module Storage + class Atmos + class Real + + def get_namespace(namespace = '', options = {}) + options = options.reject {|key, value| value.nil?} + request({ + :expects => 200, + :method => 'GET', + :path => "namespace/" + namespace, + :query => {}, + :parse => true + }.merge(options)) + end + + end + end + end +end diff --git a/lib/fog/atmos/requests/storage/head_namespace.rb b/lib/fog/atmos/requests/storage/head_namespace.rb new file mode 100644 index 000000000..342ba2e14 --- /dev/null +++ b/lib/fog/atmos/requests/storage/head_namespace.rb @@ -0,0 +1,20 @@ +module Fog + module Storage + class Atmos + class Real + + def head_namespace(namespace = '', options = {}) + options = options.reject {|key, value| value.nil?} + request({ + :expects => 200, + :method => 'HEAD', + :path => "namespace/" + namespace, + :query => {}, + :parse => true + }.merge(options)) + end + + end + end + end +end diff --git a/lib/fog/atmos/requests/storage/post_namespace.rb b/lib/fog/atmos/requests/storage/post_namespace.rb new file mode 100644 index 000000000..f8d611767 --- /dev/null +++ b/lib/fog/atmos/requests/storage/post_namespace.rb @@ -0,0 +1,20 @@ +module Fog + module Storage + class Atmos + class Real + + def post_namespace(namespace = '', options = {}) + options = options.reject {|key, value| value.nil?} + request({ + :expects => 201, + :method => 'POST', + :path => "namespace/" + namespace, + :query => {}, + :parse => true + }.merge(options)) + end + + end + end + end +end diff --git a/lib/fog/atmos/requests/storage/put_namespace.rb b/lib/fog/atmos/requests/storage/put_namespace.rb new file mode 100644 index 000000000..cc7a42ee6 --- /dev/null +++ b/lib/fog/atmos/requests/storage/put_namespace.rb @@ -0,0 +1,20 @@ +module Fog + module Storage + class Atmos + class Real + + def put_namespace(namespace = '', options = {}) + options = options.reject {|key, value| value.nil?} + request({ + :expects => 200, + :method => 'PUT', + :path => "namespace/" + namespace, + :query => {}, + :parse => true + }.merge(options)) + end + + end + end + end +end diff --git a/lib/fog/atmos/storage.rb b/lib/fog/atmos/storage.rb new file mode 100644 index 000000000..a4533ef62 --- /dev/null +++ b/lib/fog/atmos/storage.rb @@ -0,0 +1,186 @@ +require 'fog/atmos' +require 'fog/storage' + +module Fog + module Storage + class Atmos < Fog::Service + requires :atmos_storage_endpoint, + :atmos_storage_secret, + :atmos_storage_token + recognizes :persistent + + model_path 'fog/atmos/models/storage' + model :directory + collection :directories + model :file + collection :files + + request_path 'fog/atmos/requests/storage' + # request :delete_container + request :get_namespace + request :head_namespace + request :post_namespace + request :put_namespace + request :delete_namespace + + module Utils + ENDPOINT_REGEX = /(https*):\/\/([a-zA-Z0-9\.\-]+)(:[0-9]+)?(\/.*)?/ + + def ssl? + protocol = @endpoint.match(ENDPOINT_REGEX)[1] + raise ArgumentError, 'Invalid endpoint URL' if protocol.nil? + + return true if protocol == 'https' + return false if protocol == 'http' + + raise ArgumentError, "Unknown protocol #{protocol}" + end + + def port + port = @endpoint.match(ENDPOINT_REGEX)[3] + return ssl? ? 443 : 80 if port.nil? + port.split(':')[1].to_i + end + + def host + @endpoint.match(ENDPOINT_REGEX)[2] + end + + def api_path + @endpoint.match(ENDPOINT_REGEX)[4] + end + + def setup_credentials(options) + @storage_token = options[:atmos_storage_token] + @storage_secret = options[:atmos_storage_secret] + @storage_secret_decoded = Base64.decode64(@storage_secret) + @endpoint = options[:atmos_storage_endpoint] + @prefix = self.ssl? ? 'https' : 'http' + @storage_host = self.host + @storage_port = self.port + @api_path = self.api_path + end + end + + class Mock + include Utils + + def initialize(options={}) + require 'mime/types' + setup_credentials(options) + end + + def request(options) + raise "Atmos Storage mocks not implemented" + end + + end + + class Real + include Utils + + def initialize(options={}) + require 'mime/types' + + setup_credentials(options) + @connection_options = options[:connection_options] || {} + @hmac = Fog::HMAC.new('sha1', @storage_secret_decoded) + @persistent = options.fetch(:persistent, false) + + @connection = Fog::Connection.new("#{@prefix}://#{@storage_host}:#{@storage_port}", + @persistent, @connection_options) + end + + def uid + @storage_token#.split('/')[-1] + end + + def sign(string) + value = @hmac.sign(string) + Base64.encode64( value ).chomp() + end + + def reload + @connection.reset + end + + def request(params, &block) + req_path = params[:path] + # Force set host and port + params.merge!({ + :host => @storage_host, + :path => "#{@api_path}/rest/#{params[:path]}", + }) + # Set default method and headers + params = {:method => 'GET', :headers => {}}.merge params + + params[:headers]["Content-Type"] ||= "application/octet-stream" + + # Add request date + params[:headers]["date"] = Time.now().httpdate() + params[:headers]["x-emc-uid"] = @storage_token + + # Build signature string + signstring = "" + signstring += params[:method] + signstring += "\n" + signstring += params[:headers]["Content-Type"] + signstring += "\n" + if( params[:headers]["range"] ) + signstring += params[:headers]["range"] + end + signstring += "\n" + signstring += params[:headers]["date"] + signstring += "\n" + + signstring += "/rest/" + URI.unescape( req_path ).downcase + query_str = params[:query].map{|k,v| "#{k}=#{v}"}.join('&') + signstring += '?' + query_str unless query_str.empty? + signstring += "\n" + + customheaders = {} + params[:headers].each { |key,value| + case key + when 'x-emc-date', 'x-emc-signature' + #skip + when /^x-emc-/ + customheaders[ key.downcase ] = value + end + } + header_arr = customheaders.sort() + + header_arr.each { |key,value| + # Values are lowercase and whitespace-normalized + signstring += key + ":" + value.strip.chomp.squeeze( " " ) + "\n" + } + + digest = @hmac.sign(signstring.chomp()) + signature = Base64.encode64( digest ).chomp() + params[:headers]["x-emc-signature"] = signature + + begin + response = @connection.request(params, &block) + rescue Excon::Errors::HTTPStatusError => error + raise case error + when Excon::Errors::NotFound + Fog::Storage::Atmos::NotFound.slurp(error) + else + error + end + end + unless response.body.empty? + if params[:parse] + document = Fog::ToHashDocument.new + parser = Nokogiri::XML::SAX::PushParser.new(document) + parser << response.body + parser.finish + response.body = document.body + end + end + response + end + + end + end + end +end diff --git a/lib/fog/bin.rb b/lib/fog/bin.rb index 9f08745be..c8c370422 100644 --- a/lib/fog/bin.rb +++ b/lib/fog/bin.rb @@ -4,6 +4,7 @@ module Fog class << self def available_providers + Kernel.const_get('Ninefold') @available_providers ||= Fog.providers.values.select {|provider| Kernel.const_get(provider).available?}.sort end @@ -56,6 +57,7 @@ module Fog end +require 'fog/bin/atmos' require 'fog/bin/aws' require 'fog/bin/bluebox' require 'fog/bin/brightbox' diff --git a/lib/fog/bin/atmos.rb b/lib/fog/bin/atmos.rb new file mode 100644 index 000000000..59da2f226 --- /dev/null +++ b/lib/fog/bin/atmos.rb @@ -0,0 +1,31 @@ +class Atmos < Fog::Bin + class << self + + def class_for(key) + case key + when :storage + Fog::Storage::Atmos + else + raise ArgumentError, "Unsupported #{self} service: #{key}" + end + end + + def [](service) + @@connections ||= Hash.new do |hash, key| + hash[key] = case key + when :storage + Fog::Logger.warning("Atmos[:storage] is not recommended, use Storage[:atmos] for portability") + Fog::Storage.new(:provider => 'Atmos') + else + raise ArgumentError, "Unrecognized service: #{service}" + end + end + @@connections[service] + end + + def services + Fog::Atmos.services + end + + end +end diff --git a/lib/fog/hp/storage.rb b/lib/fog/hp/storage.rb index a3162949a..b0b50d956 100644 --- a/lib/fog/hp/storage.rb +++ b/lib/fog/hp/storage.rb @@ -108,6 +108,7 @@ module Fog def initialize(options={}) require 'mime/types' + puts "Called with #{options}" @hp_secret_key = options[:hp_secret_key] @hp_account_id = options[:hp_account_id] end diff --git a/lib/fog/providers.rb b/lib/fog/providers.rb index b8867c452..6154a5a38 100644 --- a/lib/fog/providers.rb +++ b/lib/fog/providers.rb @@ -1,3 +1,4 @@ +require 'fog/atmos' require 'fog/aws' require 'fog/bluebox' require 'fog/brightbox' diff --git a/lib/fog/storage.rb b/lib/fog/storage.rb index 752b5d78a..044014ee5 100644 --- a/lib/fog/storage.rb +++ b/lib/fog/storage.rb @@ -8,6 +8,9 @@ module Fog def self.new(attributes) attributes = attributes.dup # prevent delete from having side effects case provider = attributes.delete(:provider).to_s.downcase.to_sym + when :atmos + require 'fog/atmos/storage' + Fog::Storage::Atmos.new(attributes) when :aws require 'fog/aws/storage' Fog::Storage::AWS.new(attributes) diff --git a/tests/atmos/models/storage/file_update_tests.rb b/tests/atmos/models/storage/file_update_tests.rb new file mode 100644 index 000000000..e9bc34f1c --- /dev/null +++ b/tests/atmos/models/storage/file_update_tests.rb @@ -0,0 +1,19 @@ +Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do + + unless Fog.mocking? + @directory = Fog::Storage[:atmos].directories.create(:key => 'updatefiletests') + end + + atmos = Fog::Storage[:atmos] + tests("update a file").succeeds do + pending if Fog.mocking? + file = @directory.files.create(:key => 'lorem.txt', :body => lorem_file) + file.body = "xxxxxx" + file.save + end + + unless Fog.mocking? + @directory.destroy(:recursive => true) + end + +end diff --git a/tests/atmos/models/storage/nested_directories_tests.rb b/tests/atmos/models/storage/nested_directories_tests.rb new file mode 100644 index 000000000..3481fdd4f --- /dev/null +++ b/tests/atmos/models/storage/nested_directories_tests.rb @@ -0,0 +1,23 @@ +Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do + atmos = Fog::Storage[:atmos] + tests("create a directory with a / character").succeeds do + pending if Fog.mocking? + atmos.directories.create(:key => 'sub/path') + end + + tests("List of top directory returns sub dir").returns(1) do + pending if Fog.mocking? + atmos.directories.get('sub').directories.count + end + + tests("create a directory in a sub dir").returns('sub/path/newdir/') do + pending if Fog.mocking? + atmos.directories.get('sub/path').directories.create(:key => 'newdir').identity + end + + tests("Recursively destroy parent dir").succeeds do + pending if Fog.mocking? + atmos.directories.get('sub').destroy(:recursive => true) + end + +end diff --git a/tests/helpers/mock_helper.rb b/tests/helpers/mock_helper.rb index 57a8bf0ae..6f2c31023 100644 --- a/tests/helpers/mock_helper.rb +++ b/tests/helpers/mock_helper.rb @@ -11,6 +11,9 @@ if Fog.mock? Fog.credentials = { :aws_access_key_id => 'aws_access_key_id', :aws_secret_access_key => 'aws_secret_access_key', + :atmos_storage_token => 'atmos_token', + :atmos_storage_secret => 'atmos_secret', + :atmos_storage_endpoint => 'http://atmos.is.cool:1000/test1.0', :bluebox_api_key => 'bluebox_api_key', :bluebox_customer_id => 'bluebox_customer_id', :brightbox_client_id => 'brightbox_client_id',