From eb1468c4ffe29d9bf83b97a6c82fe2a5cc05f1da Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Sat, 8 Sep 2012 21:07:32 -0700 Subject: [PATCH] [Ninefold|Storage] Use Atmos in Ninefold storage. Currently, code is duplicated between ninefold and atmos. We should remove the code duplication and extend the atmos module in Ninefold. --- .../ninefold/models/storage/directories.rb | 48 ------- lib/fog/ninefold/models/storage/directory.rb | 53 -------- lib/fog/ninefold/models/storage/file.rb | 108 --------------- lib/fog/ninefold/models/storage/files.rb | 73 ---------- .../requests/storage/delete_namespace.rb | 19 --- .../requests/storage/get_namespace.rb | 20 --- .../requests/storage/head_namespace.rb | 20 --- .../requests/storage/post_namespace.rb | 20 --- .../requests/storage/put_namespace.rb | 20 --- lib/fog/ninefold/storage.rb | 126 ++---------------- 10 files changed, 14 insertions(+), 493 deletions(-) delete mode 100644 lib/fog/ninefold/models/storage/directories.rb delete mode 100644 lib/fog/ninefold/models/storage/directory.rb delete mode 100644 lib/fog/ninefold/models/storage/file.rb delete mode 100644 lib/fog/ninefold/models/storage/files.rb delete mode 100644 lib/fog/ninefold/requests/storage/delete_namespace.rb delete mode 100644 lib/fog/ninefold/requests/storage/get_namespace.rb delete mode 100644 lib/fog/ninefold/requests/storage/head_namespace.rb delete mode 100644 lib/fog/ninefold/requests/storage/post_namespace.rb delete mode 100644 lib/fog/ninefold/requests/storage/put_namespace.rb diff --git a/lib/fog/ninefold/models/storage/directories.rb b/lib/fog/ninefold/models/storage/directories.rb deleted file mode 100644 index 516d429da..000000000 --- a/lib/fog/ninefold/models/storage/directories.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'fog/core/collection' -require 'fog/ninefold/models/storage/directory' - -module Fog - module Storage - class Ninefold - - class Directories < Fog::Collection - - attribute :directory - - model Fog::Storage::Ninefold::Directory - - def all - directory ? ns = directory.key : ns = '' - ns = ns + '/' unless ns =~ /\/$/ - data = connection.get_namespace(ns).body[:DirectoryList] - data = {:DirectoryEntry => []} if data.kind_of? String - data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash - dirs = data[:DirectoryEntry].select {|de| de[:FileType] == 'directory'} - dirs.each do |d| - d[:Filename] = ns + d[:Filename] if directory - d[:Filename] += '/' unless d[:Filename] =~ /\/$/ - end - load(dirs) - end - - def get(key, options = {}) - return nil if key == '' # Root dir shouldn't be retrieved like this. - key =~ /\/$/ ? ns = key : ns = key + '/' - res = connection.get_namespace ns - emc_meta = res.headers['x-emc-meta'] - obj_id = emc_meta.scan(/objectid=(\w+),/).flatten[0] - new(:objectid => obj_id, :key => ns) - rescue Fog::Storage::Ninefold::NotFound - nil - end - - def new(attributes ={}) - attributes = {:directory => directory}.merge(attributes) if directory - super(attributes) - end - - end - - end - end -end diff --git a/lib/fog/ninefold/models/storage/directory.rb b/lib/fog/ninefold/models/storage/directory.rb deleted file mode 100644 index a71ee9aea..000000000 --- a/lib/fog/ninefold/models/storage/directory.rb +++ /dev/null @@ -1,53 +0,0 @@ -require 'fog/core/model' - -module Fog - module Storage - class Ninefold - - class Directory < Fog::Model - - identity :key, :aliases => :Filename - attribute :objectid, :aliases => :ObjectID - - def files - @files ||= begin - Fog::Storage::Ninefold::Files.new( - :directory => self, - :connection => connection - ) - end - end - - def directories - @directories ||= begin - Fog::Storage::Ninefold::Directories.new( - :directory => self, - :connection => connection - ) - end - end - - def save - self.key = attributes[:directory].key + key if attributes[:directory] - self.key = key + '/' unless key =~ /\/$/ - res = connection.post_namespace key - reload - end - - def destroy(opts={}) - if opts[:recursive] - files.each {|f| f.destroy } - directories.each do |d| - d.files.each {|f| f.destroy } - d.destroy(opts) - end - end - connection.delete_namespace key - end - - - end - - end - end -end diff --git a/lib/fog/ninefold/models/storage/file.rb b/lib/fog/ninefold/models/storage/file.rb deleted file mode 100644 index 050a3c40c..000000000 --- a/lib/fog/ninefold/models/storage/file.rb +++ /dev/null @@ -1,108 +0,0 @@ -require 'fog/core/model' - -module Fog - module Storage - class Ninefold - - class File < Fog::Model - - identity :key, :aliases => :Filename - - attribute :content_length, :aliases => ['bytes', 'Content-Length'], :type => :integer - attribute :content_type, :aliases => ['content_type', 'Content-Type'] - attribute :objectid, :aliases => :ObjectID - - def body - attributes[:body] ||= if objectid - collection.get(identity).body - else - '' - end - end - - def body=(new_body) - attributes[:body] = new_body - end - - def directory - @directory - end - - def copy(target_directory_key, target_file_key, options={}) - target_directory = connection.directories.new(:key => target_directory_key) - target_directory.files.create( - :key => target_file_key, - :body => body - ) - end - - def destroy - requires :directory, :key - connection.delete_namespace([directory.key, key].join('/')) - true - end - - # def owner=(new_owner) - # if new_owner - # attributes[:owner] = { - # :display_name => new_owner['DisplayName'], - # :id => new_owner['ID'] - # } - # end - # end - - def public=(new_public) - # NOOP - we don't need to flag files as public, getting the public URL for a file handles it. - end - - # By default, expire in 5 years - def public_url(expires = (Time.now + 5 * 365 * 24 * 60 * 60)) - requires :objectid - # TODO - more efficient method to get this? - storage = Fog::Storage.new(:provider => 'Ninefold') - uri = URI::HTTP.build(:scheme => Fog::Storage::Ninefold::STORAGE_SCHEME, :host => Fog::Storage::Ninefold::STORAGE_HOST, :port => Fog::Storage::Ninefold::STORAGE_PORT.to_i, :path => "/rest/objects/#{objectid}" ) - Fog::Storage.new(:provider => 'Ninefold').uid - - - sb = "GET\n" - sb += uri.path.downcase + "\n" - sb += storage.uid + "\n" - sb += String(expires.to_i()) - - signature = storage.sign( sb ) - uri.query = "uid=#{CGI::escape(storage.uid)}&expires=#{expires.to_i()}&signature=#{CGI::escape(signature)}" - uri.to_s - end - - def save(options = {}) - requires :body, :directory, :key - directory.kind_of?(Directory) ? ns = directory.key : ns = directory - ns += key - options[:headers] ||= {} - options[:headers]['Content-Type'] = content_type if content_type - options[:body] = body - begin - data = connection.post_namespace(ns, options) - self.objectid = data.headers['location'].split('/')[-1] - rescue => error - if error.message =~ /The resource you are trying to create already exists./ - data = connection.put_namespace(ns, options) - else - raise error - end - end - # merge_attributes(data.headers) - true - end - - private - - def directory=(new_directory) - @directory = new_directory - end - - end - - end - end -end diff --git a/lib/fog/ninefold/models/storage/files.rb b/lib/fog/ninefold/models/storage/files.rb deleted file mode 100644 index 99d8ea8a3..000000000 --- a/lib/fog/ninefold/models/storage/files.rb +++ /dev/null @@ -1,73 +0,0 @@ -require 'fog/core/collection' -require 'fog/ninefold/models/storage/file' - -module Fog - module Storage - class Ninefold - - class Files < Fog::Collection - - attribute :directory - attribute :limit - attribute :marker - attribute :path - attribute :prefix - - model Fog::Storage::Ninefold::File - - def all(options = {}) - requires :directory - directory ? ns = directory.key : ns = '' - ns = ns + '/' unless ns =~ /\/$/ - data = connection.get_namespace(ns).body[:DirectoryList] - data = {:DirectoryEntry => []} if data.kind_of? String - data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash - files = data[:DirectoryEntry].select {|de| de[:FileType] == 'regular'} - files.each do |s| - s[:directory] = directory - end - # TODO - Load additional file meta? - load(files) - end - - def get(key, &block) - requires :directory - data = connection.get_namespace(directory.key + key, :parse => false)#, &block) - file_data = data.headers.merge({ - :body => data.body, - :key => key - }) - new(file_data) - rescue Fog::Storage::Ninefold::NotFound - nil - end - - def get_url(key) - requires :directory - if self.directory.public_url - "#{self.directory.public_url}/#{key}" - end - end - - def head(key, options = {}) - requires :directory - data = connection.head_namespace(directory.key + key, :parse => false) - file_data = data.headers.merge({ - :body => data.body, - :key => key - }) - new(file_data) - rescue Fog::Storage::Ninefold::NotFound - nil - end - - def new(attributes = {}) - requires :directory - super({ :directory => directory }.merge!(attributes)) - end - - end - - end - end -end diff --git a/lib/fog/ninefold/requests/storage/delete_namespace.rb b/lib/fog/ninefold/requests/storage/delete_namespace.rb deleted file mode 100644 index d9ad7b8c6..000000000 --- a/lib/fog/ninefold/requests/storage/delete_namespace.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Fog - module Storage - class Ninefold - class Real - - def delete_namespace(namespace = '', options = {}) - options = options.reject {|key, value| value.nil?} - request({ - :expects => 204, - :method => 'DELETE', - :path => "namespace/" + namespace, - :query => options - }.merge(options)) - end - - end - end - end -end diff --git a/lib/fog/ninefold/requests/storage/get_namespace.rb b/lib/fog/ninefold/requests/storage/get_namespace.rb deleted file mode 100644 index 28b6b13b6..000000000 --- a/lib/fog/ninefold/requests/storage/get_namespace.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Fog - module Storage - class Ninefold - class Real - - def get_namespace(namespace = '', options = {}) - options = options.reject {|key, value| value.nil?} - request({ - :expects => 200, - :method => 'GET', - :path => "namespace/" + namespace, - :query => {}, - :parse => true - }.merge(options)) - end - - end - end - end -end diff --git a/lib/fog/ninefold/requests/storage/head_namespace.rb b/lib/fog/ninefold/requests/storage/head_namespace.rb deleted file mode 100644 index 436d65717..000000000 --- a/lib/fog/ninefold/requests/storage/head_namespace.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Fog - module Storage - class Ninefold - class Real - - def head_namespace(namespace = '', options = {}) - options = options.reject {|key, value| value.nil?} - request({ - :expects => 200, - :method => 'HEAD', - :path => "namespace/" + namespace, - :query => {}, - :parse => true - }.merge(options)) - end - - end - end - end -end diff --git a/lib/fog/ninefold/requests/storage/post_namespace.rb b/lib/fog/ninefold/requests/storage/post_namespace.rb deleted file mode 100644 index 6a314a46a..000000000 --- a/lib/fog/ninefold/requests/storage/post_namespace.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Fog - module Storage - class Ninefold - class Real - - def post_namespace(namespace = '', options = {}) - options = options.reject {|key, value| value.nil?} - request({ - :expects => 201, - :method => 'POST', - :path => "namespace/" + namespace, - :query => {}, - :parse => true - }.merge(options)) - end - - end - end - end -end diff --git a/lib/fog/ninefold/requests/storage/put_namespace.rb b/lib/fog/ninefold/requests/storage/put_namespace.rb deleted file mode 100644 index 19a820c69..000000000 --- a/lib/fog/ninefold/requests/storage/put_namespace.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Fog - module Storage - class Ninefold - class Real - - def put_namespace(namespace = '', options = {}) - options = options.reject {|key, value| value.nil?} - request({ - :expects => 200, - :method => 'PUT', - :path => "namespace/" + namespace, - :query => {}, - :parse => true - }.merge(options)) - end - - end - end - end -end diff --git a/lib/fog/ninefold/storage.rb b/lib/fog/ninefold/storage.rb index 5c440f526..0990eca32 100644 --- a/lib/fog/ninefold/storage.rb +++ b/lib/fog/ninefold/storage.rb @@ -1,34 +1,28 @@ require 'fog/ninefold' require 'fog/storage' +require 'fog/atmos' module Fog module Storage - class Ninefold < Fog::Service + class Ninefold < Fog::Storage::Atmos STORAGE_HOST = "onlinestorage.ninefold.com" #"api.ninefold.com" STORAGE_PATH = "" #"/storage/v1.0" STORAGE_PORT = "80" # "443" STORAGE_SCHEME = "http" # "https" requires :ninefold_storage_token, :ninefold_storage_secret + recognizes :persistent - model_path 'fog/ninefold/models/storage' + model_path 'fog/atmos/models/storage' model :directory collection :directories model :file collection :files - request_path 'fog/ninefold/requests/storage' - # request :delete_container - request :get_namespace - request :head_namespace - request :post_namespace - request :put_namespace - request :delete_namespace - module Utils end - class Mock + class Mock < Fog::Storage::Atmos::Mock include Utils def initialize(options={}) @@ -43,111 +37,19 @@ module Fog end - class Real + class Real < Fog::Storage::Atmos::Real include Utils def initialize(options={}) - require 'mime/types' - @ninefold_storage_token = options[:ninefold_storage_token] - @ninefold_storage_secret = options[:ninefold_storage_secret] - @ninefold_storage_secret_decoded = Base64.decode64( @ninefold_storage_secret ) - - @connection_options = options[:connection_options] || {} - @hmac = Fog::HMAC.new('sha1', @ninefold_storage_secret_decoded) - @persistent = options.fetch(:persistent, true) - - @connection = Fog::Connection.new("#{Fog::Storage::Ninefold::STORAGE_SCHEME}://#{Fog::Storage::Ninefold::STORAGE_HOST}:#{Fog::Storage::Ninefold::STORAGE_PORT}", @persistent, @connection_options) + endpoint = "#{STORAGE_SCHEME}://"\ + "#{STORAGE_HOST}:"\ + "#{STORAGE_PORT}"\ + "#{STORAGE_PATH}" + options[:atmos_storage_endpoint] = endpoint + options[:atmos_storage_token] = options[:ninefold_storage_token] + options[:atmos_storage_secret] = options[:ninefold_storage_secret] + super(options) end - - def uid - @ninefold_storage_token#.split('/')[-1] - end - - def sign(string) - value = @hmac.sign(string) - Base64.encode64( value ).chomp() - end - - def reload - @connection.reset - end - - def request(params, &block) - req_path = params[:path] - # Force set host and port - params.merge!({ - :host => Fog::Storage::Ninefold::STORAGE_HOST, - :path => "#{Fog::Storage::Ninefold::STORAGE_PATH}/rest/#{params[:path]}", - }) - # Set default method and headers - params = {:method => 'GET', :headers => {}}.merge params - - params[:headers]["Content-Type"] ||= "application/octet-stream" - - # Add request date - params[:headers]["date"] = Time.now().httpdate() - params[:headers]["x-emc-uid"] = @ninefold_storage_token - - # Build signature string - signstring = "" - signstring += params[:method] - signstring += "\n" - signstring += params[:headers]["Content-Type"] - signstring += "\n" - if( params[:headers]["range"] ) - signstring += params[:headers]["range"] - end - signstring += "\n" - signstring += params[:headers]["date"] - signstring += "\n" - - signstring += "/rest/" + URI.unescape( req_path ).downcase - query_str = params[:query].map{|k,v| "#{k}=#{v}"}.join('&') - signstring += '?' + query_str unless query_str.empty? - signstring += "\n" - - customheaders = {} - params[:headers].each { |key,value| - case key - when 'x-emc-date', 'x-emc-signature' - #skip - when /^x-emc-/ - customheaders[ key.downcase ] = value - end - } - header_arr = customheaders.sort() - - header_arr.each { |key,value| - # Values are lowercase and whitespace-normalized - signstring += key + ":" + value.strip.chomp.squeeze( " " ) + "\n" - } - - digest = @hmac.sign(signstring.chomp()) - signature = Base64.encode64( digest ).chomp() - params[:headers]["x-emc-signature"] = signature - - begin - response = @connection.request(params, &block) - rescue Excon::Errors::HTTPStatusError => error - raise case error - when Excon::Errors::NotFound - Fog::Storage::Ninefold::NotFound.slurp(error) - else - error - end - end - unless response.body.empty? - if params[:parse] - document = Fog::ToHashDocument.new - parser = Nokogiri::XML::SAX::PushParser.new(document) - parser << response.body - parser.finish - response.body = document.body - end - end - response - end - end end end