Add generic support for EMC Atmos.

Refactor ninefold to provide generic Atmos support. Add an additional
argument, when compared to ninefold, which is the endpoint. The
endpoint should be a full URL, e.g.
https://storage.provider.com:1337/atmos. The API path and port are
optional. If the port is not specified, it is inferred from the
protocol.

lib/fog/atmos.rb

@@ -0,0 +1,11 @@
+require 'fog/core'
+
+module Fog
+  module Atmos
+
+    extend Fog::Provider
+
+    service(:storage, 'atmos/storage', 'Storage')
+
+  end
+end

lib/fog/atmos/models/storage/directories.rb

@@ -0,0 +1,48 @@
+require 'fog/core/collection'
+require 'fog/atmos/models/storage/directory'
+
+module Fog
+  module Storage
+    class Atmos
+
+      class Directories < Fog::Collection
+
+        attribute :directory
+
+        model Fog::Storage::Atmos::Directory
+
+        def all
+          directory ? ns = directory.key : ns = ''
+          ns = ns + '/' unless ns =~ /\/$/
+          data = connection.get_namespace(ns).body[:DirectoryList]
+          data = {:DirectoryEntry => []} if data.kind_of? String
+          data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash
+          dirs = data[:DirectoryEntry].select {|de| de[:FileType] == 'directory'}
+          dirs.each do |d|
+            d[:Filename] = ns + d[:Filename] if directory
+            d[:Filename] += '/' unless d[:Filename] =~ /\/$/
+          end
+          load(dirs)
+        end
+
+        def get(key, options = {})
+          return nil if key == '' # Root dir shouldn't be retrieved like this.
+          key =~ /\/$/ ? ns = key : ns = key + '/'
+          res = connection.get_namespace ns
+          emc_meta = res.headers['x-emc-meta']
+          obj_id = emc_meta.scan(/objectid=(\w+),/).flatten[0]
+          new(:objectid => obj_id, :key => ns)
+        rescue Fog::Storage::Atmos::NotFound
+          nil
+        end
+
+        def new(attributes ={})
+          attributes = {:directory => directory}.merge(attributes) if directory
+          super(attributes)
+        end
+
+      end
+
+    end
+  end
+end

lib/fog/atmos/models/storage/directory.rb

@@ -0,0 +1,53 @@
+require 'fog/core/model'
+
+module Fog
+  module Storage
+    class Atmos
+
+      class Directory < Fog::Model
+
+        identity :key, :aliases => :Filename
+        attribute :objectid, :aliases => :ObjectID
+
+        def files
+          @files ||= begin
+                       Fog::Storage::Atmos::Files.new(
+                                                          :directory    => self,
+                                                          :connection   => connection
+                                                          )
+          end
+        end
+
+        def directories
+          @directories ||= begin
+                       Fog::Storage::Atmos::Directories.new(
+                                                               :directory    => self,
+                                                               :connection   => connection
+                                                               )
+          end
+        end
+
+        def save
+          self.key = attributes[:directory].key + key if attributes[:directory]
+          self.key = key + '/' unless key =~ /\/$/
+          res = connection.post_namespace key
+          reload
+        end
+
+        def destroy(opts={})
+          if opts[:recursive]
+            files.each {|f| f.destroy }
+            directories.each do |d|
+              d.files.each {|f| f.destroy }
+              d.destroy(opts)
+            end
+          end
+          connection.delete_namespace key
+        end
+
+
+      end
+
+    end
+  end
+end

lib/fog/atmos/models/storage/file.rb

@@ -0,0 +1,107 @@
+require 'fog/core/model'
+
+module Fog
+  module Storage
+    class Atmos
+
+      class File < Fog::Model
+
+        identity  :key,             :aliases => :Filename
+
+        attribute :content_length,  :aliases => ['bytes', 'Content-Length'], :type => :integer
+        attribute :content_type,    :aliases => ['content_type', 'Content-Type']
+        attribute :objectid,        :aliases => :ObjectID
+
+        def body
+          attributes[:body] ||= if objectid
+            collection.get(identity).body
+          else
+            ''
+          end
+        end
+
+        def body=(new_body)
+          attributes[:body] = new_body
+        end
+
+        def directory
+          @directory
+        end
+
+        def copy(target_directory_key, target_file_key, options={})
+          target_directory = connection.directories.new(:key => target_directory_key)
+          target_directory.files.create(
+            :key => target_file_key,
+            :body => body
+          )
+        end
+
+        def destroy
+          requires :directory, :key
+          connection.delete_namespace([directory.key, key].join('/'))
+          true
+        end
+
+        # def owner=(new_owner)
+        #   if new_owner
+        #     attributes[:owner] = {
+        #       :display_name => new_owner['DisplayName'],
+        #       :id           => new_owner['ID']
+        #     }
+        #   end
+        # end
+
+        def public=(new_public)
+          # NOOP - we don't need to flag files as public, getting the public URL for a file handles it.
+        end
+
+        # By default, expire in 5 years
+        def public_url(expires = (Time.now + 5 * 365 * 24 * 60 * 60))
+          requires :objectid
+          # TODO - more efficient method to get this?
+          storage = Fog::Storage.new(:provider => 'Atmos')
+          uri = URI::HTTP.build(:scheme => @prefix, :host => @storage_host, :port => @storage_port.to_i, :path => "/rest/objects/#{objectid}" )
+          Fog::Storage.new(:provider => 'Atmos').uid
+
+          sb = "GET\n"
+          sb += uri.path.downcase + "\n"
+          sb += storage.uid + "\n"
+          sb += String(expires.to_i())
+
+          signature = storage.sign( sb )
+          uri.query = "uid=#{CGI::escape(storage.uid)}&expires=#{expires.to_i()}&signature=#{CGI::escape(signature)}"
+          uri.to_s
+        end
+
+        def save(options = {})
+          requires :body, :directory, :key
+          directory.kind_of?(Directory) ? ns = directory.key : ns = directory
+          ns += key
+          options[:headers] ||= {}
+          options[:headers]['Content-Type'] = content_type if content_type
+          options[:body] = body
+          begin
+            data = connection.post_namespace(ns, options)
+            self.objectid = data.headers['location'].split('/')[-1]
+          rescue => error
+            if error.message =~ /The resource you are trying to create already exists./
+              data = connection.put_namespace(ns, options)
+            else
+              raise error
+            end
+          end
+          # merge_attributes(data.headers)
+          true
+        end
+
+        private
+
+        def directory=(new_directory)
+          @directory = new_directory
+        end
+
+      end
+
+    end
+  end
+end

lib/fog/atmos/models/storage/files.rb

@@ -0,0 +1,73 @@
+require 'fog/core/collection'
+require 'fog/atmos/models/storage/file'
+
+module Fog
+  module Storage
+    class Atmos
+
+      class Files < Fog::Collection
+
+        attribute :directory
+        attribute :limit
+        attribute :marker
+        attribute :path
+        attribute :prefix
+
+        model Fog::Storage::Atmos::File
+
+        def all(options = {})
+          requires :directory
+          directory ? ns = directory.key : ns = ''
+          ns = ns + '/' unless ns =~ /\/$/
+          data = connection.get_namespace(ns).body[:DirectoryList]
+          data = {:DirectoryEntry => []} if data.kind_of? String
+          data[:DirectoryEntry] = [data[:DirectoryEntry]] if data[:DirectoryEntry].kind_of? Hash
+          files = data[:DirectoryEntry].select {|de| de[:FileType] == 'regular'}
+          files.each do |s|
+            s[:directory] = directory
+          end
+          # TODO - Load additional file meta?
+          load(files)
+        end
+
+        def get(key, &block)
+          requires :directory
+          data = connection.get_namespace(directory.key + key, :parse => false)#, &block)
+          file_data = data.headers.merge({
+            :body => data.body,
+            :key  => key
+          })
+          new(file_data)
+        rescue Fog::Storage::Atmos::NotFound
+          nil
+        end
+
+        def get_url(key)
+          requires :directory
+          if self.directory.public_url
+            "#{self.directory.public_url}/#{key}"
+          end
+        end
+
+        def head(key, options = {})
+          requires :directory
+          data = connection.head_namespace(directory.key + key, :parse => false)
+          file_data = data.headers.merge({
+            :body => data.body,
+            :key => key
+          })
+          new(file_data)
+        rescue Fog::Storage::Atmos::NotFound
+          nil
+        end
+
+        def new(attributes = {})
+          requires :directory
+          super({ :directory => directory }.merge!(attributes))
+        end
+
+      end
+
+    end
+  end
+end

lib/fog/atmos/requests/storage/delete_namespace.rb

@@ -0,0 +1,19 @@
+module Fog
+  module Storage
+    class Atmos
+      class Real
+
+        def delete_namespace(namespace = '', options = {})
+          options = options.reject {|key, value| value.nil?}
+          request({
+                    :expects  => 204,
+                    :method   => 'DELETE',
+                    :path     => "namespace/" + namespace,
+                    :query    => options
+                  }.merge(options))
+        end
+
+      end
+    end
+  end
+end

lib/fog/atmos/requests/storage/get_namespace.rb

@@ -0,0 +1,20 @@
+module Fog
+  module Storage
+    class Atmos
+      class Real
+
+        def get_namespace(namespace = '', options = {})
+          options = options.reject {|key, value| value.nil?}
+          request({
+                    :expects  => 200,
+                    :method   => 'GET',
+                    :path     => "namespace/" + namespace,
+                    :query    => {},
+                    :parse => true
+                  }.merge(options))
+        end
+
+      end
+    end
+  end
+end

lib/fog/atmos/requests/storage/head_namespace.rb

@@ -0,0 +1,20 @@
+module Fog
+  module Storage
+    class Atmos
+      class Real
+
+        def head_namespace(namespace = '', options = {})
+          options = options.reject {|key, value| value.nil?}
+          request({
+                    :expects  => 200,
+                    :method   => 'HEAD',
+                    :path     => "namespace/" + namespace,
+                    :query    => {},
+                    :parse => true
+                  }.merge(options))
+        end
+
+      end
+    end
+  end
+end

lib/fog/atmos/requests/storage/post_namespace.rb

@@ -0,0 +1,20 @@
+module Fog
+  module Storage
+    class Atmos
+      class Real
+
+        def post_namespace(namespace = '', options = {})
+          options = options.reject {|key, value| value.nil?}
+          request({
+                    :expects  => 201,
+                    :method   => 'POST',
+                    :path     => "namespace/" + namespace,
+                    :query    => {},
+                    :parse => true
+                  }.merge(options))
+        end
+
+      end
+    end
+  end
+end

lib/fog/atmos/requests/storage/put_namespace.rb

@@ -0,0 +1,20 @@
+module Fog
+  module Storage
+    class Atmos
+      class Real
+
+        def put_namespace(namespace = '', options = {})
+          options = options.reject {|key, value| value.nil?}
+          request({
+                    :expects  => 200,
+                    :method   => 'PUT',
+                    :path     => "namespace/" + namespace,
+                    :query    => {},
+                    :parse => true
+                  }.merge(options))
+        end
+
+      end
+    end
+  end
+end

lib/fog/atmos/storage.rb

@@ -0,0 +1,186 @@
+require 'fog/atmos'
+require 'fog/storage'
+
+module Fog
+  module Storage
+    class Atmos < Fog::Service
+      requires :atmos_storage_endpoint,
+               :atmos_storage_secret,
+               :atmos_storage_token
+      recognizes :persistent
+
+      model_path 'fog/atmos/models/storage'
+      model       :directory
+      collection  :directories
+      model       :file
+      collection  :files
+
+      request_path 'fog/atmos/requests/storage'
+      # request :delete_container
+      request :get_namespace
+      request :head_namespace
+      request :post_namespace
+      request :put_namespace
+      request :delete_namespace
+
+      module Utils
+        ENDPOINT_REGEX = /(https*):\/\/([a-zA-Z0-9\.\-]+)(:[0-9]+)?(\/.*)?/
+
+        def ssl?
+          protocol = @endpoint.match(ENDPOINT_REGEX)[1]
+          raise ArgumentError, 'Invalid endpoint URL' if protocol.nil?
+
+          return true if protocol == 'https'
+          return false if protocol == 'http'
+
+          raise ArgumentError, "Unknown protocol #{protocol}"
+        end
+
+        def port
+          port = @endpoint.match(ENDPOINT_REGEX)[3]
+          return ssl? ? 443 : 80 if port.nil?
+          port.split(':')[1].to_i
+        end
+
+        def host
+          @endpoint.match(ENDPOINT_REGEX)[2]
+        end
+
+        def api_path
+          @endpoint.match(ENDPOINT_REGEX)[4]
+        end
+
+        def setup_credentials(options)
+          @storage_token = options[:atmos_storage_token]
+          @storage_secret = options[:atmos_storage_secret]
+          @storage_secret_decoded = Base64.decode64(@storage_secret)
+          @endpoint = options[:atmos_storage_endpoint]
+          @prefix = self.ssl? ? 'https' : 'http'
+          @storage_host = self.host
+          @storage_port = self.port
+          @api_path = self.api_path
+        end
+      end
+
+      class Mock
+        include Utils
+
+        def initialize(options={})
+          require 'mime/types'
+          setup_credentials(options)
+        end
+
+        def request(options)
+          raise "Atmos Storage mocks not implemented"
+        end
+
+      end
+
+      class Real
+        include Utils
+
+        def initialize(options={})
+          require 'mime/types'
+
+          setup_credentials(options)
+          @connection_options = options[:connection_options] || {}
+          @hmac               = Fog::HMAC.new('sha1', @storage_secret_decoded)
+          @persistent = options.fetch(:persistent, false)
+
+          @connection = Fog::Connection.new("#{@prefix}://#{@storage_host}:#{@storage_port}",
+              @persistent, @connection_options)
+        end
+
+        def uid
+          @storage_token#.split('/')[-1]
+        end
+
+        def sign(string)
+          value = @hmac.sign(string)
+          Base64.encode64( value ).chomp()
+        end
+
+        def reload
+          @connection.reset
+        end
+
+        def request(params, &block)
+          req_path = params[:path]
+          # Force set host and port
+          params.merge!({
+                          :host     => @storage_host,
+                          :path     => "#{@api_path}/rest/#{params[:path]}",
+                        })
+          # Set default method and headers
+          params = {:method => 'GET', :headers => {}}.merge params
+
+          params[:headers]["Content-Type"] ||= "application/octet-stream"
+
+          # Add request date
+          params[:headers]["date"] = Time.now().httpdate()
+          params[:headers]["x-emc-uid"] = @storage_token
+
+          # Build signature string
+          signstring = ""
+          signstring += params[:method]
+          signstring += "\n"
+          signstring += params[:headers]["Content-Type"]
+          signstring += "\n"
+          if( params[:headers]["range"] )
+            signstring += params[:headers]["range"]
+          end
+          signstring += "\n"
+          signstring += params[:headers]["date"]
+          signstring += "\n"
+
+          signstring += "/rest/" + URI.unescape( req_path ).downcase
+          query_str = params[:query].map{|k,v| "#{k}=#{v}"}.join('&')
+          signstring += '?' + query_str unless query_str.empty?
+          signstring += "\n"
+
+          customheaders = {}
+          params[:headers].each { |key,value|
+            case key
+            when 'x-emc-date', 'x-emc-signature'
+              #skip
+            when /^x-emc-/
+              customheaders[ key.downcase ] = value
+            end
+          }
+          header_arr = customheaders.sort()
+
+          header_arr.each { |key,value|
+            # Values are lowercase and whitespace-normalized
+            signstring += key + ":" + value.strip.chomp.squeeze( " " ) + "\n"
+          }
+
+          digest = @hmac.sign(signstring.chomp())
+          signature = Base64.encode64( digest ).chomp()
+          params[:headers]["x-emc-signature"] = signature
+
+          begin
+            response = @connection.request(params, &block)
+          rescue Excon::Errors::HTTPStatusError => error
+            raise case error
+            when Excon::Errors::NotFound
+              Fog::Storage::Atmos::NotFound.slurp(error)
+            else
+              error
+            end
+          end
+          unless response.body.empty?
+            if params[:parse]
+              document = Fog::ToHashDocument.new
+              parser = Nokogiri::XML::SAX::PushParser.new(document)
+              parser << response.body
+              parser.finish
+              response.body = document.body
+            end
+          end
+          response
+        end
+
+      end
+    end
+  end
+end

lib/fog/bin.rb

@@ -4,6 +4,7 @@ module Fog
   class << self
 
     def available_providers
+	  Kernel.const_get('Ninefold')
       @available_providers ||= Fog.providers.values.select {|provider| Kernel.const_get(provider).available?}.sort
     end
 
@@ -56,6 +57,7 @@ module Fog
 
 end
 
+require 'fog/bin/atmos'
 require 'fog/bin/aws'
 require 'fog/bin/bluebox'
 require 'fog/bin/brightbox'

lib/fog/bin/atmos.rb

@@ -0,0 +1,31 @@
+class Atmos < Fog::Bin
+  class << self
+
+    def class_for(key)
+      case key
+      when :storage
+        Fog::Storage::Atmos
+      else
+        raise ArgumentError, "Unsupported #{self} service: #{key}"
+      end
+    end
+
+    def [](service)
+      @@connections ||= Hash.new do |hash, key|
+        hash[key] = case key
+        when :storage
+          Fog::Logger.warning("Atmos[:storage] is not recommended, use Storage[:atmos] for portability")
+          Fog::Storage.new(:provider => 'Atmos')
+        else
+          raise ArgumentError, "Unrecognized service: #{service}"
+        end
+      end
+      @@connections[service]
+    end
+
+    def services
+      Fog::Atmos.services
+    end
+
+  end
+end

lib/fog/hp/storage.rb

@@ -108,6 +108,7 @@ module Fog
 
         def initialize(options={})
           require 'mime/types'
+          puts "Called with #{options}"
           @hp_secret_key = options[:hp_secret_key]
           @hp_account_id = options[:hp_account_id]
         end

lib/fog/providers.rb

@@ -1,3 +1,4 @@
+require 'fog/atmos'
 require 'fog/aws'
 require 'fog/bluebox'
 require 'fog/brightbox'

lib/fog/storage.rb

@@ -8,6 +8,9 @@ module Fog
     def self.new(attributes)
       attributes = attributes.dup # prevent delete from having side effects
       case provider = attributes.delete(:provider).to_s.downcase.to_sym
+      when :atmos
+        require 'fog/atmos/storage'
+        Fog::Storage::Atmos.new(attributes)
       when :aws
         require 'fog/aws/storage'
         Fog::Storage::AWS.new(attributes)

tests/atmos/models/storage/file_update_tests.rb

@@ -0,0 +1,19 @@
+Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do
+
+  unless Fog.mocking?
+    @directory = Fog::Storage[:atmos].directories.create(:key => 'updatefiletests')
+  end
+
+  atmos = Fog::Storage[:atmos]
+  tests("update a file").succeeds do
+    pending if Fog.mocking?
+    file = @directory.files.create(:key => 'lorem.txt', :body => lorem_file)
+    file.body = "xxxxxx"
+    file.save
+  end
+
+  unless Fog.mocking?
+    @directory.destroy(:recursive => true)
+  end
+
+end

tests/atmos/models/storage/nested_directories_tests.rb

@@ -0,0 +1,23 @@
+Shindo.tests("Storage[:atmos] | nested directories", ['atmos']) do
+  atmos = Fog::Storage[:atmos]
+  tests("create a directory with a / character").succeeds do
+    pending if Fog.mocking?
+    atmos.directories.create(:key => 'sub/path')
+  end
+
+  tests("List of top directory returns sub dir").returns(1) do
+    pending if Fog.mocking?
+    atmos.directories.get('sub').directories.count
+  end
+
+  tests("create a directory in a sub dir").returns('sub/path/newdir/') do
+    pending if Fog.mocking?
+    atmos.directories.get('sub/path').directories.create(:key => 'newdir').identity
+  end
+
+  tests("Recursively destroy parent dir").succeeds do
+    pending if Fog.mocking?
+    atmos.directories.get('sub').destroy(:recursive => true)
+  end
+
+end

tests/helpers/mock_helper.rb

@@ -11,6 +11,9 @@ if Fog.mock?
   Fog.credentials = {
     :aws_access_key_id                => 'aws_access_key_id',
     :aws_secret_access_key            => 'aws_secret_access_key',
+    :atmos_storage_token              => 'atmos_token',
+    :atmos_storage_secret             => 'atmos_secret',
+    :atmos_storage_endpoint           => 'http://atmos.is.cool:1000/test1.0',
     :bluebox_api_key                  => 'bluebox_api_key',
     :bluebox_customer_id              => 'bluebox_customer_id',
     :brightbox_client_id              => 'brightbox_client_id',