[rackspace|storage] updated put_object to allow blocks to upload large files; added large file upload example and documentation

lib/fog/rackspace/docs/storage.md

@@ -335,6 +335,8 @@ To upload a file into a directory:
 
 	file = directory.files.create :key => 'space.jpg', :body => File.open "space.jpg"
 	
+**Note**: For files larger than 5 GB please refer to the [Upload Large Files](#upload_large_files) section.
+
 ### Additional Parameters
 
 The `create` method also supports the following key values:
@@ -366,6 +368,37 @@ The `create` method also supports the following key values:
 	</tr>
 </table>
 
+## Upload Large Files
+
+Cloud Files requires files larger than 5 GB to be uploaded into segments along with an accompanying manifest file. All of the segments must be uploaded to the same container.
+
+	SEGMENT_LIMIT = 5368709119.0  # 5GB -1
+	BUFFER_SIZE = 1024 * 1024 # 1MB
+
+	File.open("large_file") do |f|
+	  num_segments = (f.size / SEGMENT_LIMIT).round + 1
+	  1.upto(num_segments) do |segment|
+	    offset = 0
+	    read = 0
+	    # upload segment to cloud files
+	    service.put_object("my_container", "large_file/#{segment}", nil, options = {}) do
+	      if (offset < SEGMENT_LIMIT) && (read.zero? || read == BUFFER_SIZE)
+	        buf = f.sysread(BUFFER_SIZE)
+	        read = buf.size
+	        offset += read
+	        buf
+	      else
+	        ""
+	      end
+	    end
+	  end
+	end
+
+	# write manifest file
+	service.put_object_manifest("my_container", "large_file")
+
+Segmented files are downloaded like ordinary files. See [Download Files](#download-files) section for more information.
+
 ## Download Files
 
 The most efficient way to download files from a private or public directory is as follows:

lib/fog/rackspace/examples/storage/upload_large_files.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+
+# This example demonstrates uploading large files in segments
+
+require 'rubygems' #required for Ruby 1.8.x
+require 'fog'
+
+# Size of segment. The Rackspace cloud currently requires files larger than 5GB to be segmented so we will choose 5GB -1 for a size
+# http://docs.rackspace.com/files/api/v1/cf-devguide/content/Large_Object_Creation-d1e2019.html
+SEGMENT_LIMIT = 5368709119.0
+
+# Size of buffer to use for transfers. Use Excon's default chunk size and if that's not avaliable we will default to 1 MB
+BUFFER_SIZE = Excon.defaults[:chunk_size] || 1024 * 1024
+
+def get_user_input(prompt)
+  print "\n#{prompt}: "
+  gets.chomp
+end
+
+def select_directory(directories)
+  abort "\nThere are not any directories in the Chicago region. Try running create_private_directory.rb\n\n" if directories.empty?
+
+  puts "\nSelect Directory:\n\n"
+  directories.each_with_index do |dir, i|
+    puts "\t #{i}. #{dir.key} [#{dir.count} objects]"
+  end
+
+  select_str = get_user_input "Enter Directory Number"
+  directories[select_str.to_i]
+end
+
+# Use username defined in ~/.fog file, if absent prompt for username.
+# For more details on ~/.fog refer to http://fog.io/about/getting_started.html
+def rackspace_username
+  Fog.credentials[:rackspace_username] || get_user_input("Enter Rackspace Username")
+end
+
+# Use api key defined in ~/.fog file, if absent prompt for api key
+# For more details on ~/.fog refer to http://fog.io/about/getting_started.html
+def rackspace_api_key
+  Fog.credentials[:rackspace_api_key] || get_user_input("Enter Rackspace API key")
+end
+
+# create Cloud Files service
+service = Fog::Storage.new({
+  :provider             => 'Rackspace',
+  :rackspace_username   => rackspace_username,
+  :rackspace_api_key    => rackspace_api_key,
+  :rackspace_region     => :ord
+  })
+
+
+# retrieve directories with files
+directories = service.directories
+
+# prompt for directory
+directory = select_directory(directories)
+
+# prompt for file name
+file_name = get_user_input "Enter full path of file to upload"
+segment_name = File.basename(file_name)
+
+File.open(file_name) do |f|
+  num_segments = (f.size / SEGMENT_LIMIT).round + 1
+  puts "\nThis upload of '#{file_name}' will require #{num_segments} segment(s) and 1 manifest file\n"
+
+  1.upto(num_segments) do |segment|
+    print "\n\tUploading segment #{segment} "
+    offset = 0
+    read = 0
+    service.put_object(directory.key, "#{segment_name}/#{segment}", nil, options = {}) do
+      if (offset < SEGMENT_LIMIT) && (read.zero? || read == BUFFER_SIZE)
+        print "."
+        buf = f.sysread(BUFFER_SIZE)
+        read = buf.size
+        offset += read
+        buf
+      else
+        ""
+      end
+    end
+  end
+end
+
+puts "\n\n\tWriting manifest #{segment_name}\n\n"
+service.put_object_manifest(directory.key, segment_name)
+
+puts <<-NOTE
+You should now be able to download #{segment_name} from the cloud control panel or using the following code:
+
+    directory = service.directories.get('#{directory.key}')
+    File.open('downloaded_#{segment_name}', 'w') do | f |
+      directory.files.get(#{segment_name}) do | data, remaining, content_length |
+        print "."
+        f.syswrite data
+      end
+    end
+
+NOTE

lib/fog/rackspace/requests/storage/put_object.rb

@@ -14,19 +14,22 @@ module Fog
         # @raise [Fog::Storage::Rackspace::BadRequest] - HTTP 400
         # @raise [Fog::Storage::Rackspace::InternalServerError] - HTTP 500
         # @raise [Fog::Storage::Rackspace::ServiceError]
-        def put_object(container, object, data, options = {})
+        def put_object(container, object, data, options = {}, &block)
           data = Fog::Storage.parse_data(data)
           headers = data[:headers].merge!(options)
-          request(
-            :body       => data[:body],
+
+          params = block_given? ? { :request_block => block } : { :body => data[:body] }
+
+          params.merge!(
             :expects    => 201,
             :idempotent => true,
             :headers    => headers,
             :method     => 'PUT',
             :path       => "#{Fog::Rackspace.escape(container)}/#{Fog::Rackspace.escape(object)}"
           )
-        end
 
+          request(params)
+        end
       end
     end
   end

tests/rackspace/requests/storage/object_tests.rb

@@ -17,9 +17,9 @@ Shindo.tests('Fog::Storage[:rackspace] | object requests', ["rackspace"]) do
       Fog::Storage[:rackspace].put_object('fogobjecttests', 'fog_object', lorem_file)
     end
 
-    tests("#get_object('fogobjectests', 'fog_object')").succeeds do
+    tests("#get_object('fogobjectests', 'fog_object')").returns(lorem_file.read) do
       pending if Fog.mocking?
-      Fog::Storage[:rackspace].get_object('fogobjecttests', 'fog_object')
+      Fog::Storage[:rackspace].get_object('fogobjecttests', 'fog_object').body
     end
 
     tests("#get_object('fogobjecttests', 'fog_object', &block)").returns(lorem_file.read) do
@@ -74,6 +74,36 @@ Shindo.tests('Fog::Storage[:rackspace] | object requests', ["rackspace"]) do
       object_url =~ /https:\/\/.*clouddrive.com\/[^\/]+\/[^\/]+\/fogobjecttests\/fog%2Dobject\?temp_url_sig=a24dd5fc955a57adce7d1b5bc4ec2c7660ab8396&temp_url_expires=1344149532/
     end
 
+    tests("put_object with block") do
+      tests("#put_object('fogobjecttests', 'fog_object', &block)") do
+        pending if Fog.mocking?
+
+        begin
+          file = lorem_file
+          buffer_size = file.size / 2 # chop it up into two buffers
+          Fog::Storage[:rackspace].put_object('fogobjecttests', 'fog_block_object', nil) do
+            if file.pos < file.size
+              file.sysread(buffer_size)
+            else
+              ""
+            end
+          end
+        ensure
+          file.close
+        end
+      end
+
+      tests("object successfully uploaded?").returns(lorem_file.read) do
+        pending if Fog.mocking?
+        Fog::Storage[:rackspace].get_object('fogobjecttests', 'fog_block_object').body
+      end
+
+      tests("delete file").succeeds do
+        Fog::Storage[:rackspace].delete_object('fogobjecttests', 'fog_block_object')
+      end
+
+    end
+
   end
 
   tests('failure') do