From 9ace35ce5f95f3441b68fd4de4c0430be95dccbf Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Wed, 23 Jan 2013 20:26:17 +0100 Subject: [PATCH 1/7] [openstack|storage] intial import Porting Fog Rackspace Storage service to OpenStack. Mostly replaced names, removed CDN stuff (perhaps Rackspace specific) and used authenticate_v2 (keystone). --- lib/fog/openstack.rb | 1 + .../openstack/models/storage/directories.rb | 39 +++++ lib/fog/openstack/models/storage/directory.rb | 50 ++++++ lib/fog/openstack/models/storage/file.rb | 150 +++++++++++++++++ lib/fog/openstack/models/storage/files.rb | 94 +++++++++++ .../openstack/requests/storage/copy_object.rb | 27 ++++ .../requests/storage/delete_container.rb | 22 +++ .../requests/storage/delete_object.rb | 23 +++ .../requests/storage/get_container.rb | 44 +++++ .../requests/storage/get_containers.rb | 33 ++++ .../openstack/requests/storage/get_object.rb | 30 ++++ .../requests/storage/get_object_https_url.rb | 51 ++++++ .../requests/storage/head_container.rb | 28 ++++ .../requests/storage/head_containers.rb | 25 +++ .../openstack/requests/storage/head_object.rb | 23 +++ .../storage/post_set_meta_temp_url_key.rb | 37 +++++ .../requests/storage/put_container.rb | 22 +++ .../openstack/requests/storage/put_object.rb | 30 ++++ .../requests/storage/put_object_manifest.rb | 25 +++ lib/fog/openstack/storage.rb | 151 ++++++++++++++++++ 20 files changed, 905 insertions(+) create mode 100644 lib/fog/openstack/models/storage/directories.rb create mode 100644 lib/fog/openstack/models/storage/directory.rb create mode 100644 lib/fog/openstack/models/storage/file.rb create mode 100644 lib/fog/openstack/models/storage/files.rb create mode 100644 lib/fog/openstack/requests/storage/copy_object.rb create mode 100644 lib/fog/openstack/requests/storage/delete_container.rb create mode 100644 lib/fog/openstack/requests/storage/delete_object.rb create mode 100644 lib/fog/openstack/requests/storage/get_container.rb create mode 100644 lib/fog/openstack/requests/storage/get_containers.rb create mode 100644 lib/fog/openstack/requests/storage/get_object.rb create mode 100644 lib/fog/openstack/requests/storage/get_object_https_url.rb create mode 100644 lib/fog/openstack/requests/storage/head_container.rb create mode 100644 lib/fog/openstack/requests/storage/head_containers.rb create mode 100644 lib/fog/openstack/requests/storage/head_object.rb create mode 100644 lib/fog/openstack/requests/storage/post_set_meta_temp_url_key.rb create mode 100644 lib/fog/openstack/requests/storage/put_container.rb create mode 100644 lib/fog/openstack/requests/storage/put_object.rb create mode 100644 lib/fog/openstack/requests/storage/put_object_manifest.rb create mode 100644 lib/fog/openstack/storage.rb diff --git a/lib/fog/openstack.rb b/lib/fog/openstack.rb index 90cbed36d..8dfcac803 100644 --- a/lib/fog/openstack.rb +++ b/lib/fog/openstack.rb @@ -44,6 +44,7 @@ module Fog service(:compute , 'openstack/compute' , 'Compute' ) service(:identity, 'openstack/identity', 'Identity') service(:network, 'openstack/network', 'Network') + service(:storage, 'openstack/storage', 'Storage') # legacy v1.0 style auth def self.authenticate_v1(options, connection_options = {}) diff --git a/lib/fog/openstack/models/storage/directories.rb b/lib/fog/openstack/models/storage/directories.rb new file mode 100644 index 000000000..ce3d8921a --- /dev/null +++ b/lib/fog/openstack/models/storage/directories.rb @@ -0,0 +1,39 @@ +require 'fog/core/collection' +require 'fog/openstack/models/storage/directory' + +module Fog + module Storage + class OpenStack + + class Directories < Fog::Collection + + model Fog::Storage::OpenStack::Directory + + def all + data = service.get_containers.body + load(data) + end + + def get(key, options = {}) + data = service.get_container(key, options) + directory = new(:key => key) + for key, value in data.headers + if ['X-Container-Bytes-Used', 'X-Container-Object-Count'].include?(key) + directory.merge_attributes(key => value) + end + end + directory.files.merge_attributes(options) + directory.files.instance_variable_set(:@loaded, true) + data.body.each do |file| + directory.files << directory.files.new(file) + end + directory + rescue Fog::Storage::OpenStack::NotFound + nil + end + + end + + end + end +end diff --git a/lib/fog/openstack/models/storage/directory.rb b/lib/fog/openstack/models/storage/directory.rb new file mode 100644 index 000000000..b04730a70 --- /dev/null +++ b/lib/fog/openstack/models/storage/directory.rb @@ -0,0 +1,50 @@ +require 'fog/core/model' +require 'fog/openstack/models/storage/files' + +module Fog + module Storage + class OpenStack + + class Directory < Fog::Model + + identity :key, :aliases => 'name' + + attribute :bytes, :aliases => 'X-Container-Bytes-Used' + attribute :count, :aliases => 'X-Container-Object-Count' + + def destroy + requires :key + service.delete_container(key) + true + rescue Excon::Errors::NotFound + false + end + + def files + @files ||= begin + Fog::Storage::OpenStack::Files.new( + :directory => self, + :service => service + ) + end + end + + def public=(new_public) + @public = new_public + end + + def public_url + raise NotImplementedError + end + + def save + requires :key + service.put_container(key) + true + end + + end + + end + end +end diff --git a/lib/fog/openstack/models/storage/file.rb b/lib/fog/openstack/models/storage/file.rb new file mode 100644 index 000000000..53fdffacb --- /dev/null +++ b/lib/fog/openstack/models/storage/file.rb @@ -0,0 +1,150 @@ +require 'fog/core/model' + +module Fog + module Storage + class OpenStack + + class File < Fog::Model + + identity :key, :aliases => 'name' + + attribute :content_length, :aliases => ['bytes', 'Content-Length'], :type => :integer + attribute :content_type, :aliases => ['content_type', 'Content-Type'] + attribute :etag, :aliases => ['hash', 'Etag'] + attribute :last_modified, :aliases => ['last_modified', 'Last-Modified'], :type => :time + attribute :access_control_allow_origin, :aliases => ['Access-Control-Allow-Origin'] + attribute :origin, :aliases => ['Origin'] + + def body + attributes[:body] ||= if last_modified + collection.get(identity).body + else + '' + end + end + + def body=(new_body) + attributes[:body] = new_body + end + + def directory + @directory + end + + def copy(target_directory_key, target_file_key, options={}) + requires :directory, :key + options['Content-Type'] ||= content_type if content_type + options['Access-Control-Allow-Origin'] ||= access_control_allow_origin if access_control_allow_origin + options['Origin'] ||= origin if origin + service.copy_object(directory.key, key, target_directory_key, target_file_key, options) + target_directory = service.directories.new(:key => target_directory_key) + target_directory.files.get(target_file_key) + end + + def destroy + requires :directory, :key + service.delete_object(directory.key, key) + true + end + + def metadata + @metadata ||= headers_to_metadata + end + + def owner=(new_owner) + if new_owner + attributes[:owner] = { + :display_name => new_owner['DisplayName'], + :id => new_owner['ID'] + } + end + end + + def public=(new_public) + new_public + end + + def public_url + requires :key + self.collection.get_url(self.key) + end + + def save(options = {}) + requires :body, :directory, :key + options['Content-Type'] = content_type if content_type + options['Access-Control-Allow-Origin'] = access_control_allow_origin if access_control_allow_origin + options['Origin'] = origin if origin + options.merge!(metadata_to_headers) + + data = service.put_object(directory.key, key, body, options) + update_attributes_from(data) + refresh_metadata + + self.content_length = Fog::Storage.get_body_size(body) + self.content_type ||= Fog::Storage.get_content_type(body) + true + end + + private + + def directory=(new_directory) + @directory = new_directory + end + + def refresh_metadata + metadata.reject! {|k, v| v.nil? } + end + + def headers_to_metadata + key_map = key_mapping + Hash[metadata_attributes.map {|k, v| [key_map[k], v] }] + end + + def key_mapping + key_map = metadata_attributes + key_map.each_pair {|k, v| key_map[k] = header_to_key(k)} + end + + def header_to_key(opt) + opt.gsub(metadata_prefix, '').split('-').map {|k| k[0, 1].downcase + k[1..-1]}.join('_').to_sym + end + + def metadata_to_headers + header_map = header_mapping + Hash[metadata.map {|k, v| [header_map[k], v] }] + end + + def header_mapping + header_map = metadata.dup + header_map.each_pair {|k, v| header_map[k] = key_to_header(k)} + end + + def key_to_header(key) + metadata_prefix + key.to_s.split(/[-_]/).map(&:capitalize).join('-') + end + + def metadata_attributes + if last_modified + headers = service.head_object(directory.key, self.key).headers + headers.reject! {|k, v| !metadata_attribute?(k)} + else + {} + end + end + + def metadata_attribute?(key) + key.to_s =~ /^#{metadata_prefix}/ + end + + def metadata_prefix + "X-Object-Meta-" + end + + def update_attributes_from(data) + merge_attributes(data.headers.reject {|key, value| ['Content-Length', 'Content-Type'].include?(key)}) + end + end + + end + end +end diff --git a/lib/fog/openstack/models/storage/files.rb b/lib/fog/openstack/models/storage/files.rb new file mode 100644 index 000000000..21d4f9589 --- /dev/null +++ b/lib/fog/openstack/models/storage/files.rb @@ -0,0 +1,94 @@ +require 'fog/core/collection' +require 'fog/openstack/models/storage/file' + +module Fog + module Storage + class OpenStack + + class Files < Fog::Collection + + attribute :directory + attribute :limit + attribute :marker + attribute :path + attribute :prefix + + model Fog::Storage::OpenStack::File + + def all(options = {}) + requires :directory + options = { + 'limit' => limit, + 'marker' => marker, + 'path' => path, + 'prefix' => prefix + }.merge!(options) + merge_attributes(options) + parent = directory.collection.get( + directory.key, + options + ) + if parent + load(parent.files.map {|file| file.attributes}) + else + nil + end + end + + alias :each_file_this_page :each + def each + if !block_given? + self + else + subset = dup.all + + subset.each_file_this_page {|f| yield f} + while subset.length == (subset.limit || 10000) + subset = subset.all(:marker => subset.last.key) + subset.each_file_this_page {|f| yield f} + end + + self + end + end + + def get(key, &block) + requires :directory + data = service.get_object(directory.key, key, &block) + file_data = data.headers.merge({ + :body => data.body, + :key => key + }) + new(file_data) + rescue Fog::Storage::OpenStack::NotFound + nil + end + + def get_url(key) + requires :directory + if self.directory.public_url + "#{self.directory.public_url}/#{Fog::OpenStack.escape(key, '/')}" + end + end + + def head(key, options = {}) + requires :directory + data = service.head_object(directory.key, key) + file_data = data.headers.merge({ + :key => key + }) + new(file_data) + rescue Fog::Storage::OpenStack::NotFound + nil + end + + def new(attributes = {}) + requires :directory + super({ :directory => directory }.merge!(attributes)) + end + + end + + end + end +end diff --git a/lib/fog/openstack/requests/storage/copy_object.rb b/lib/fog/openstack/requests/storage/copy_object.rb new file mode 100644 index 000000000..1a9de8959 --- /dev/null +++ b/lib/fog/openstack/requests/storage/copy_object.rb @@ -0,0 +1,27 @@ +module Fog + module Storage + class OpenStack + class Real + + # Copy object + # + # ==== Parameters + # * source_container_name<~String> - Name of source bucket + # * source_object_name<~String> - Name of source object + # * target_container_name<~String> - Name of bucket to create copy in + # * target_object_name<~String> - Name for new copy of object + # * options<~Hash> - Additional headers + def copy_object(source_container_name, source_object_name, target_container_name, target_object_name, options={}) + headers = { 'X-Copy-From' => "/#{source_container_name}/#{source_object_name}" }.merge(options) + request({ + :expects => 201, + :headers => headers, + :method => 'PUT', + :path => "#{Fog::OpenStack.escape(target_container_name)}/#{Fog::OpenStack.escape(target_object_name)}" + }) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/delete_container.rb b/lib/fog/openstack/requests/storage/delete_container.rb new file mode 100644 index 000000000..9cc083191 --- /dev/null +++ b/lib/fog/openstack/requests/storage/delete_container.rb @@ -0,0 +1,22 @@ +module Fog + module Storage + class OpenStack + class Real + + # Delete an existing container + # + # ==== Parameters + # * name<~String> - Name of container to delete + # + def delete_container(name) + request( + :expects => 204, + :method => 'DELETE', + :path => Fog::OpenStack.escape(name) + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/delete_object.rb b/lib/fog/openstack/requests/storage/delete_object.rb new file mode 100644 index 000000000..27a24fd4d --- /dev/null +++ b/lib/fog/openstack/requests/storage/delete_object.rb @@ -0,0 +1,23 @@ +module Fog + module Storage + class OpenStack + class Real + + # Delete an existing object + # + # ==== Parameters + # * container<~String> - Name of container to delete + # * object<~String> - Name of object to delete + # + def delete_object(container, object) + request( + :expects => 204, + :method => 'DELETE', + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/get_container.rb b/lib/fog/openstack/requests/storage/get_container.rb new file mode 100644 index 000000000..0b1a42bb4 --- /dev/null +++ b/lib/fog/openstack/requests/storage/get_container.rb @@ -0,0 +1,44 @@ +module Fog + module Storage + class OpenStack + class Real + + # Get details for container and total bytes stored + # + # ==== Parameters + # * container<~String> - Name of container to retrieve info for + # * options<~String>: + # * 'limit'<~String> - Maximum number of objects to return + # * 'marker'<~String> - Only return objects whose name is greater than marker + # * 'prefix'<~String> - Limits results to those starting with prefix + # * 'path'<~String> - Return objects nested in the pseudo path + # + # ==== Returns + # * response<~Excon::Response>: + # * headers<~Hash>: + # * 'X-Account-Container-Count'<~String> - Count of containers + # * 'X-Account-Bytes-Used'<~String> - Bytes used + # * body<~Array>: + # * 'bytes'<~Integer> - Number of bytes used by container + # * 'count'<~Integer> - Number of items in container + # * 'name'<~String> - Name of container + # * item<~Hash>: + # * 'bytes'<~String> - Size of object + # * 'content_type'<~String> Content-Type of object + # * 'hash'<~String> - Hash of object (etag?) + # * 'last_modified'<~String> - Last modified timestamp + # * 'name'<~String> - Name of object + def get_container(container, options = {}) + options = options.reject {|key, value| value.nil?} + request( + :expects => 200, + :method => 'GET', + :path => Fog::OpenStack.escape(container), + :query => {'format' => 'json'}.merge!(options) + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/get_containers.rb b/lib/fog/openstack/requests/storage/get_containers.rb new file mode 100644 index 000000000..f2a7ce252 --- /dev/null +++ b/lib/fog/openstack/requests/storage/get_containers.rb @@ -0,0 +1,33 @@ +module Fog + module Storage + class OpenStack + class Real + + # List existing storage containers + # + # ==== Parameters + # * options<~Hash>: + # * 'limit'<~Integer> - Upper limit to number of results returned + # * 'marker'<~String> - Only return objects with name greater than this value + # + # ==== Returns + # * response<~Excon::Response>: + # * body<~Array>: + # * container<~Hash>: + # * 'bytes'<~Integer>: - Number of bytes used by container + # * 'count'<~Integer>: - Number of items in container + # * 'name'<~String>: - Name of container + def get_containers(options = {}) + options = options.reject {|key, value| value.nil?} + request( + :expects => [200, 204], + :method => 'GET', + :path => '', + :query => {'format' => 'json'}.merge!(options) + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/get_object.rb b/lib/fog/openstack/requests/storage/get_object.rb new file mode 100644 index 000000000..676ca5022 --- /dev/null +++ b/lib/fog/openstack/requests/storage/get_object.rb @@ -0,0 +1,30 @@ +module Fog + module Storage + class OpenStack + class Real + + # Get details for object + # + # ==== Parameters + # * container<~String> - Name of container to look in + # * object<~String> - Name of object to look for + # + def get_object(container, object, &block) + params = {} + + if block_given? + params[:response_block] = Proc.new + end + + request(params.merge!({ + :block => block, + :expects => 200, + :method => 'GET', + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + }), false) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/get_object_https_url.rb b/lib/fog/openstack/requests/storage/get_object_https_url.rb new file mode 100644 index 000000000..5c512c253 --- /dev/null +++ b/lib/fog/openstack/requests/storage/get_object_https_url.rb @@ -0,0 +1,51 @@ +module Fog + module Storage + class OpenStack + + class Real + + # Get an expiring object https url from Cloud Files + # + # ==== Parameters + # * container<~String> - Name of container containing object + # * object<~String> - Name of object to get expiring url for + # * expires<~Time> - An expiry time for this url + # + # ==== Returns + # * response<~Excon::Response>: + # * body<~String> - url for object + # + # ==== See Also + # http://docs.rackspace.com/files/api/v1/cf-devguide/content/Create_TempURL-d1a444.html + def get_object_https_url(container, object, expires, options = {}) + if @rackspace_temp_url_key.nil? + raise ArgumentError, "Storage must my instantiated with the :rackspace_temp_url_key option" + end + + method = 'GET' + expires = expires.to_i + object_path_escaped = "#{@path}/#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object,"/")}" + object_path_unescaped = "#{@path}/#{Fog::OpenStack.escape(container)}/#{object}" + string_to_sign = "#{method}\n#{expires}\n#{object_path_unescaped}" + + hmac = Fog::HMAC.new('sha1', @rackspace_temp_url_key) + sig = sig_to_hex(hmac.sign(string_to_sign)) + + "https://#{@host}#{object_path_escaped}?temp_url_sig=#{sig}&temp_url_expires=#{expires}" + end + + private + + def sig_to_hex(str) + str.unpack("C*").map { |c| + c.to_s(16) + }.map { |h| + h.size == 1 ? "0#{h}" : h + }.join + end + + end + + end + end +end diff --git a/lib/fog/openstack/requests/storage/head_container.rb b/lib/fog/openstack/requests/storage/head_container.rb new file mode 100644 index 000000000..83e4bdcf8 --- /dev/null +++ b/lib/fog/openstack/requests/storage/head_container.rb @@ -0,0 +1,28 @@ +module Fog + module Storage + class OpenStack + class Real + + # List number of objects and total bytes stored + # + # ==== Parameters + # * container<~String> - Name of container to retrieve info for + # + # ==== Returns + # * response<~Excon::Response>: + # * headers<~Hash>: + # * 'X-Container-Object-Count'<~String> - Count of containers + # * 'X-Container-Bytes-Used'<~String> - Bytes used + def head_container(container) + request( + :expects => 204, + :method => 'HEAD', + :path => Fog::OpenStack.escape(container), + :query => {'format' => 'json'} + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/head_containers.rb b/lib/fog/openstack/requests/storage/head_containers.rb new file mode 100644 index 000000000..74cc4eaf8 --- /dev/null +++ b/lib/fog/openstack/requests/storage/head_containers.rb @@ -0,0 +1,25 @@ +module Fog + module Storage + class OpenStack + class Real + + # List number of containers and total bytes stored + # + # ==== Returns + # * response<~Excon::Response>: + # * headers<~Hash>: + # * 'X-Account-Container-Count'<~String> - Count of containers + # * 'X-Account-Bytes-Used'<~String> - Bytes used + def head_containers + request( + :expects => 204, + :method => 'HEAD', + :path => '', + :query => {'format' => 'json'} + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/head_object.rb b/lib/fog/openstack/requests/storage/head_object.rb new file mode 100644 index 000000000..35396b16b --- /dev/null +++ b/lib/fog/openstack/requests/storage/head_object.rb @@ -0,0 +1,23 @@ +module Fog + module Storage + class OpenStack + class Real + + # Get headers for object + # + # ==== Parameters + # * container<~String> - Name of container to look in + # * object<~String> - Name of object to look for + # + def head_object(container, object) + request({ + :expects => 200, + :method => 'HEAD', + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + }, false) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/post_set_meta_temp_url_key.rb b/lib/fog/openstack/requests/storage/post_set_meta_temp_url_key.rb new file mode 100644 index 000000000..a7cf9719e --- /dev/null +++ b/lib/fog/openstack/requests/storage/post_set_meta_temp_url_key.rb @@ -0,0 +1,37 @@ +module Fog + module Storage + class OpenStack + + class Real + + # Set the account wide Temp URL Key. This is a secret key that's + # used to generate signed expiring URLs. + # + # Once the key has been set with this request you should create new + # Storage objects with the :rackspace_temp_url_key option then use + # the get_object_https_url method to generate expiring URLs. + # + # *** CAUTION *** changing this secret key will invalidate any expiring + # URLS generated with old keys. + # + # ==== Parameters + # * key<~String> - The new Temp URL Key + # + # ==== Returns + # * response<~Excon::Response> + # + # ==== See Also + # http://docs.rackspace.com/files/api/v1/cf-devguide/content/Set_Account_Metadata-d1a4460.html + def post_set_meta_temp_url_key(key) + request( + :expects => [201, 202, 204], + :method => 'POST', + :headers => {'X-Account-Meta-Temp-Url-Key' => key} + ) + end + + end + + end + end +end diff --git a/lib/fog/openstack/requests/storage/put_container.rb b/lib/fog/openstack/requests/storage/put_container.rb new file mode 100644 index 000000000..5a003879f --- /dev/null +++ b/lib/fog/openstack/requests/storage/put_container.rb @@ -0,0 +1,22 @@ +module Fog + module Storage + class OpenStack + class Real + + # Create a new container + # + # ==== Parameters + # * name<~String> - Name for container, should be < 256 bytes and must not contain '/' + # + def put_container(name) + request( + :expects => [201, 202], + :method => 'PUT', + :path => Fog::OpenStack.escape(name) + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/put_object.rb b/lib/fog/openstack/requests/storage/put_object.rb new file mode 100644 index 000000000..40681c7bc --- /dev/null +++ b/lib/fog/openstack/requests/storage/put_object.rb @@ -0,0 +1,30 @@ +module Fog + module Storage + class OpenStack + class Real + + # Create a new object + # + # ==== Parameters + # * container<~String> - Name for container, should be < 256 bytes and must not contain '/' + # * object<~String> - Name for object + # * data<~String|File> - data to upload + # * options<~Hash> - config headers for object. Defaults to {}. + # + def put_object(container, object, data, options = {}) + data = Fog::Storage.parse_data(data) + headers = data[:headers].merge!(options) + request( + :body => data[:body], + :expects => 201, + :idempotent => true, + :headers => headers, + :method => 'PUT', + :path => "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/requests/storage/put_object_manifest.rb b/lib/fog/openstack/requests/storage/put_object_manifest.rb new file mode 100644 index 000000000..cb49c10b0 --- /dev/null +++ b/lib/fog/openstack/requests/storage/put_object_manifest.rb @@ -0,0 +1,25 @@ +module Fog + module Storage + class OpenStack + class Real + + # Create a new object + # + # ==== Parameters + # * container<~String> - Name for container, should be < 256 bytes and must not contain '/' + # * object<~String> - Name for object + # + def put_object_manifest(container, object) + path = "#{Fog::OpenStack.escape(container)}/#{Fog::OpenStack.escape(object)}" + request( + :expects => 201, + :headers => {'X-Object-Manifest' => path}, + :method => 'PUT', + :path => path + ) + end + + end + end + end +end diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb new file mode 100644 index 000000000..ef0c376ec --- /dev/null +++ b/lib/fog/openstack/storage.rb @@ -0,0 +1,151 @@ +require 'fog/openstack' +require 'fog/storage' + +module Fog + module Storage + class OpenStack < Fog::Service + + requires :openstack_auth_url, :openstack_username, + :openstack_api_key + recognizes :persistent + + model_path 'fog/openstack/models/storage' + model :directory + collection :directories + model :file + collection :files + + request_path 'fog/openstack/requests/storage' + request :copy_object + request :delete_container + request :delete_object + request :get_container + request :get_containers + request :get_object + request :get_object_https_url + request :head_container + request :head_containers + request :head_object + request :put_container + request :put_object + request :put_object_manifest + + class Mock + + def self.data + @data ||= Hash.new do |hash, key| + hash[key] = {} + end + end + + def self.reset + @data = nil + end + + def initialize(options={}) + require 'mime/types' + @openstack_api_key = options[:openstack_api_key] + @openstack_username = options[:openstack_username] + end + + def data + self.class.data[@openstack_username] + end + + def reset_data + self.class.data.delete(@openstack_username) + end + + end + + class Real + + def initialize(options={}) + require 'mime/types' + @openstack_api_key = options[:openstack_api_key] + @openstack_username = options[:openstack_username] + @openstack_auth_url = options[:openstack_auth_url] + @openstack_auth_token = options[:openstack_auth_token] + @openstack_storage_url = options[:openstack_storage_url] + @openstack_must_reauthenticate = false + @connection_options = options[:connection_options] || {} + authenticate + @persistent = options[:persistent] || false + @connection = Fog::Connection.new("#{@scheme}://#{@host}:#{@port}", @persistent, @connection_options) + end + + def reload + @connection.reset + end + + def request(params, parse_json = true, &block) + begin + response = @connection.request(params.merge({ + :headers => { + 'Content-Type' => 'application/json', + 'X-Auth-Token' => @auth_token + }.merge!(params[:headers] || {}), + :host => @host, + :path => "#{@path}/#{params[:path]}", + }), &block) + rescue Excon::Errors::Unauthorized => error + if error.response.body != 'Bad username or password' # token expiration + @openstack_must_reauthenticate = true + authenticate + retry + else # bad credentials + raise error + end + rescue Excon::Errors::HTTPStatusError => error + raise case error + when Excon::Errors::NotFound + Fog::Storage::OpenStack::NotFound.slurp(error) + else + error + end + end + if !response.body.empty? && parse_json && response.headers['Content-Type'] =~ %r{application/json} + response.body = Fog::JSON.decode(response.body) + end + response + end + + private + + def authenticate + if !@openstack_management_url || @openstack_must_reauthenticate + options = { + :openstack_api_key => @openstack_api_key, + :openstack_username => @openstack_username, + :openstack_auth_uri => URI.parse(@openstack_auth_url), + :openstack_service_type => 'object-store', + :openstack_service_name => 'object-store', + :openstack_endpoint_type => 'publicURL' + } + + credentials = Fog::OpenStack.authenticate_v2(options, @connection_options) + + @current_user = credentials[:user] + @current_tenant = credentials[:tenant] + + @openstack_must_reauthenticate = false + @auth_token = credentials[:token] + @openstack_management_url = credentials[:server_management_url] + uri = URI.parse(@openstack_management_url) + else + @auth_token = @openstack_auth_token + uri = URI.parse(@openstack_management_url) + end + + @host = uri.host + @path = uri.path + @path.sub!(/\/$/, '') + @port = uri.port + @scheme = uri.scheme + true + end + + end + end + end +end From 2deb89e017c8edacb907587381b8e3e92e3b7b4a Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Wed, 23 Jan 2013 20:31:04 +0100 Subject: [PATCH 2/7] [openstack|storage] added OpenStack Storage to lib/fog/storage.rb --- lib/fog/storage.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/fog/storage.rb b/lib/fog/storage.rb index 044014ee5..cc9143470 100644 --- a/lib/fog/storage.rb +++ b/lib/fog/storage.rb @@ -32,6 +32,9 @@ module Fog when :rackspace require 'fog/rackspace/storage' Fog::Storage::Rackspace.new(attributes) + when :openstack + require 'fog/openstack/storage' + Fog::Storage::OpenStack.new(attributes) else raise ArgumentError.new("#{provider} is not a recognized storage provider") end From 433985bfee240834520de92e66ba28df95105489 Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Wed, 23 Jan 2013 20:53:29 +0100 Subject: [PATCH 3/7] [openstack|storage] Added OpenStack.escape utility method Required by OpenStack Storage service. --- lib/fog/openstack.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/fog/openstack.rb b/lib/fog/openstack.rb index 8dfcac803..d37d2b849 100644 --- a/lib/fog/openstack.rb +++ b/lib/fog/openstack.rb @@ -200,6 +200,13 @@ module Fog Fog::JSON.decode(response.body) end + + # CGI.escape, but without special treatment on spaces + def self.escape(str,extra_exclude_chars = '') + str.gsub(/([^a-zA-Z0-9_.-#{extra_exclude_chars}]+)/) do + '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase + end + end end end From e82ad3c07d38dbc6753f0678851ec4a78f6b2c85 Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Wed, 23 Jan 2013 20:54:33 +0100 Subject: [PATCH 4/7] [openstack|storage] Added storage service to lib/fog/bin/openstack.rb --- lib/fog/bin/openstack.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/fog/bin/openstack.rb b/lib/fog/bin/openstack.rb index def1c696c..a43aab3dc 100644 --- a/lib/fog/bin/openstack.rb +++ b/lib/fog/bin/openstack.rb @@ -9,6 +9,8 @@ class OpenStack < Fog::Bin Fog::Identity::OpenStack when :network Fog::Network::OpenStack + when :storage + Fog::Storage::OpenStack else raise ArgumentError, "Unrecognized service: #{key}" end @@ -26,6 +28,9 @@ class OpenStack < Fog::Bin when :network Fog::Logger.warning("OpenStack[:network] is not recommended, use Network[:openstack] for portability") Fog::Network.new(:provider => 'OpenStack') + when :storage + Fog::Logger.warning("OpenStack[:storage] is not recommended, use Storage[:openstack] for portability") + Fog::Network.new(:provider => 'OpenStack') else raise ArgumentError, "Unrecognized service: #{key.inspect}" end From 74ce4467eb61af9a2c37e76350a2e46307d968dc Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Wed, 23 Jan 2013 20:55:15 +0100 Subject: [PATCH 5/7] [openstack|storage] Added OpenStack Storage service tests Mostly copy&paste from the Rackspace Storage service --- tests/openstack/models/storage/file_tests.rb | 178 ++++++++++++++++++ .../requests/storage/container_tests.rb | 64 +++++++ .../requests/storage/large_object_tests.rb | 47 +++++ .../requests/storage/object_tests.rb | 84 +++++++++ 4 files changed, 373 insertions(+) create mode 100644 tests/openstack/models/storage/file_tests.rb create mode 100644 tests/openstack/requests/storage/container_tests.rb create mode 100644 tests/openstack/requests/storage/large_object_tests.rb create mode 100644 tests/openstack/requests/storage/object_tests.rb diff --git a/tests/openstack/models/storage/file_tests.rb b/tests/openstack/models/storage/file_tests.rb new file mode 100644 index 000000000..0b3e61752 --- /dev/null +++ b/tests/openstack/models/storage/file_tests.rb @@ -0,0 +1,178 @@ +Shindo.tests('Fog::OpenStack::Storage | file', ['openstack']) do + + pending if Fog.mocking? + + def object_meta_attributes + @instance.service.head_object(@directory.key, @instance.key).headers.reject {|k, v| !(k =~ /X-Object-Meta-/)} + end + + def clear_metadata + @instance.metadata.tap do |metadata| + metadata.each_pair {|k, v| metadata[k] = nil } + end + end + + file_attributes = { + :key => 'fog_file_tests', + :body => lorem_file + } + + directory_attributes = { + # Add a random suffix to prevent collision + :key => "fogfilestests-#{rand(65536)}" + } + + @directory = Fog::Storage[:openstack]. + directories. + create(directory_attributes) + + model_tests(@directory.files, file_attributes.merge(:etag => 'foo'), Fog.mocking?) do + tests('#save should not blow up with etag') do + @instance.save + end + end + + model_tests(@directory.files, file_attributes, Fog.mocking?) do + + tests("#metadata should load empty metadata").returns({}) do + @instance.metadata + end + + tests('#save') do + + tests('#metadata') do + + before do + @instance.metadata[:foo] = 'bar' + @instance.save + end + + after do + clear_metadata + @instance.save + end + + tests("should update metadata").returns('bar') do + object_meta_attributes['X-Object-Meta-Foo'] + end + + tests('should cache metadata').returns('bar') do + @instance.metadata[:foo] + end + + tests('should remove empty metadata').returns({}) do + @instance.metadata[:foo] = nil + @instance.save + object_meta_attributes + end + + end + + tests('#metadata keys') do + + after do + clear_metadata + @instance.save + end + + @instance.metadata[:foo_bar] = 'baz' + tests("should support compound key names").returns('baz') do + @instance.save + object_meta_attributes['X-Object-Meta-Foo-Bar'] + end + + @instance.metadata['foo'] = 'bar' + tests("should support string keys").returns('bar') do + @instance.save + object_meta_attributes['X-Object-Meta-Foo'] + end + + @instance.metadata['foo_bar'] = 'baz' + tests("should support compound string key names").returns('baz') do + @instance.save + object_meta_attributes['X-Object-Meta-Foo-Bar'] + end + + @instance.metadata['foo-bar'] = 'baz' + tests("should support hyphenated keys").returns('baz') do + @instance.save + object_meta_attributes['X-Object-Meta-Foo-Bar'] + end + + @instance.metadata['foo-bar'] = 'baz' + @instance.metadata[:'foo_bar'] = 'bref' + tests("should only support one value per metadata key").returns('bref') do + @instance.save + object_meta_attributes['X-Object-Meta-Foo-Bar'] + end + + end + + end + + tests("#access_control_allow_origin") do + + tests("#access_control_allow_origin should default to nil").returns(nil) do + @instance.access_control_allow_origin + end + + @instance.access_control_allow_origin = 'http://example.com' + @instance.save + tests("#access_control_allow_origin should return access control attribute").returns('http://example.com') do + @instance.access_control_allow_origin + end + + @instance.access_control_allow_origin = 'foo' + @instance.save + tests("#access_control_allow_origin= should update access_control_allow_origin").returns('bar') do + @instance.access_control_allow_origin = 'bar' + @instance.save + @instance.access_control_allow_origin + end + + tests("#access_control_allow_origin= should not blow up on nil") do + @instance.access_control_allow_origin = nil + @instance.save + end + + end + + end + + + model_tests(@directory.files, file_attributes, Fog.mocking?) do + + tests("#origin") do + + tests("#origin should default to nil").returns(nil) do + @instance.save + @instance.origin + end + + @instance.origin = 'http://example.com' + @instance.save + tests("#origin should return access control attributes").returns('http://example.com') do + @instance.origin + end + @instance.attributes.delete('Origin') + + @instance.origin = 'foo' + @instance.save + tests("#origin= should update origin").returns('bar') do + @instance.origin = 'bar' + @instance.save + @instance.origin + end + + tests("#origin= should not blow up on nil") do + @instance.origin = nil + @instance.save + end + + end + + end + + @directory.destroy + +end diff --git a/tests/openstack/requests/storage/container_tests.rb b/tests/openstack/requests/storage/container_tests.rb new file mode 100644 index 000000000..517bcc19d --- /dev/null +++ b/tests/openstack/requests/storage/container_tests.rb @@ -0,0 +1,64 @@ +Shindo.tests('Fog::Storage[:openstack] | container requests', ["openstack"]) do + + @container_format = [String] + + @containers_format = [{ + 'bytes' => Integer, + 'count' => Integer, + 'name' => String + }] + + tests('success') do + + tests("#put_container('fogcontainertests')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].put_container('fogcontainertests') + end + + tests("#get_container('fogcontainertests')").formats(@container_format) do + pending if Fog.mocking? + Fog::Storage[:openstack].get_container('fogcontainertests').body + end + + tests("#get_containers").formats(@containers_format) do + pending if Fog.mocking? + Fog::Storage[:openstack].get_containers.body + end + + tests("#head_container('fogcontainertests')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].head_container('fogcontainertests') + end + + tests("#head_containers").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].head_containers + end + + tests("#delete_container('fogcontainertests')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].delete_container('fogcontainertests') + end + + end + + tests('failure') do + + tests("#get_container('fognoncontainer')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].get_container('fognoncontainer') + end + + tests("#head_container('fognoncontainer')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].head_container('fognoncontainer') + end + + tests("#delete_container('fognoncontainer')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].delete_container('fognoncontainer') + end + + end + +end diff --git a/tests/openstack/requests/storage/large_object_tests.rb b/tests/openstack/requests/storage/large_object_tests.rb new file mode 100644 index 000000000..78b5738b7 --- /dev/null +++ b/tests/openstack/requests/storage/large_object_tests.rb @@ -0,0 +1,47 @@ +Shindo.tests('Fog::Storage[:openstack] | large object requests', ["openstack"]) do + + unless Fog.mocking? + @directory = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests') + end + + tests('success') do + + tests("#put_object('foglargeobjecttests', 'fog_large_object/1', ('x' * 6 * 1024 * 1024))").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/1', ('x' * 6 * 1024 * 1024)) + end + + tests("#put_object('foglargeobjecttests', 'fog_large_object/2', ('x' * 4 * 1024 * 1024))").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].put_object(@directory.identity, 'fog_large_object/2', ('x' * 4 * 1024 * 1024)) + end + + tests("#put_object_manifest('foglargeobjecttests', 'fog_large_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object') + end + + tests("#get_object('foglargeobjecttests', 'fog_large_object').body").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == ('x' * 10 * 1024 * 1024) + end + + unless Fog.mocking? + ['fog_large_object', 'fog_large_object/1', 'fog_large_object/2'].each do |key| + @directory.files.new(:key => key).destroy + end + end + + end + + tests('failure') do + + tests("put_object_manifest") + + end + + unless Fog.mocking? + @directory.destroy + end + +end diff --git a/tests/openstack/requests/storage/object_tests.rb b/tests/openstack/requests/storage/object_tests.rb new file mode 100644 index 000000000..a6e2a972e --- /dev/null +++ b/tests/openstack/requests/storage/object_tests.rb @@ -0,0 +1,84 @@ +Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do + + unless Fog.mocking? + @directory = Fog::Storage[:openstack].directories.create(:key => 'fogobjecttests') + end + + module OpenStackStorageHelpers + def override_path(path) + @path = path + end + end + + tests('success') do + + tests("#put_object('fogobjecttests', 'fog_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file) + end + + tests("#get_object('fogobjectests', 'fog_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object') + end + + tests("#get_object('fogobjecttests', 'fog_object', &block)").returns(lorem_file.read) do + pending if Fog.mocking? + data = '' + Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object') do |chunk, remaining_bytes, total_bytes| + data << chunk + end + data + end + + tests("#head_object('fogobjectests', 'fog_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].head_object('fogobjecttests', 'fog_object') + end + + tests("#delete_object('fogobjecttests', 'fog_object')").succeeds do + pending if Fog.mocking? + Fog::Storage[:openstack].delete_object('fogobjecttests', 'fog_object') + end + + end + + tests('failure') do + + tests("#get_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_non_object') + end + + tests("#get_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].get_object('fognoncontainer', 'fog_non_object') + end + + tests("#head_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].head_object('fogobjecttests', 'fog_non_object') + end + + tests("#head_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].head_object('fognoncontainer', 'fog_non_object') + end + + tests("#delete_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].delete_object('fogobjecttests', 'fog_non_object') + end + + tests("#delete_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do + pending if Fog.mocking? + Fog::Storage[:openstack].delete_object('fognoncontainer', 'fog_non_object') + end + + end + + unless Fog.mocking? + @directory.destroy + end + +end From bab6f73a686fd97947769945670a05d190b5e7fd Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Mon, 28 Jan 2013 14:35:29 +0100 Subject: [PATCH 6/7] [openstack|storage] configurable service_type and service_name - openstack_service_type defaults to object_storage - openstack_service_name defaults to nil refs #1496 --- lib/fog/openstack/storage.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index ef0c376ec..04364d8ab 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -7,7 +7,8 @@ module Fog requires :openstack_auth_url, :openstack_username, :openstack_api_key - recognizes :persistent + recognizes :persistent, :openstack_service_name, + :openstack_service_type model_path 'fog/openstack/models/storage' model :directory @@ -68,6 +69,8 @@ module Fog @openstack_auth_token = options[:openstack_auth_token] @openstack_storage_url = options[:openstack_storage_url] @openstack_must_reauthenticate = false + @openstack_service_type = options[:openstack_service_type] || 'object_store' + @openstack_service_name = options[:openstack_service_name] @connection_options = options[:connection_options] || {} authenticate @persistent = options[:persistent] || false @@ -118,8 +121,8 @@ module Fog :openstack_api_key => @openstack_api_key, :openstack_username => @openstack_username, :openstack_auth_uri => URI.parse(@openstack_auth_url), - :openstack_service_type => 'object-store', - :openstack_service_name => 'object-store', + :openstack_service_type => @openstack_service_type, + :openstack_service_name => @openstack_service_name, :openstack_endpoint_type => 'publicURL' } From 31a3bfa2fef92d44ff2903712591b282334d6695 Mon Sep 17 00:00:00 2001 From: Sergio Rubio <rubiojr@frameos.org> Date: Mon, 28 Jan 2013 16:38:23 +0100 Subject: [PATCH 7/7] [openstack|storage] added openstack_tenant and openstack_region params Added openstack_tenant and openstack_region as recognized parameters and pass them to OpenStack.authenticate_v2. --- lib/fog/openstack/storage.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/fog/openstack/storage.rb b/lib/fog/openstack/storage.rb index 04364d8ab..39fb8a592 100644 --- a/lib/fog/openstack/storage.rb +++ b/lib/fog/openstack/storage.rb @@ -8,7 +8,8 @@ module Fog requires :openstack_auth_url, :openstack_username, :openstack_api_key recognizes :persistent, :openstack_service_name, - :openstack_service_type + :openstack_service_type, :openstack_tenant, + :openstack_region model_path 'fog/openstack/models/storage' model :directory @@ -71,6 +72,8 @@ module Fog @openstack_must_reauthenticate = false @openstack_service_type = options[:openstack_service_type] || 'object_store' @openstack_service_name = options[:openstack_service_name] + @openstack_region = options[:openstack_region] + @openstack_tenant = options[:openstack_tenant] @connection_options = options[:connection_options] || {} authenticate @persistent = options[:persistent] || false @@ -123,6 +126,8 @@ module Fog :openstack_auth_uri => URI.parse(@openstack_auth_url), :openstack_service_type => @openstack_service_type, :openstack_service_name => @openstack_service_name, + :openstack_region => @openstack_region, + :openstack_tenant => @openstack_tenant, :openstack_endpoint_type => 'publicURL' }