Stream data for S3 get/put object

lib/fog/aws/requests/s3/get_object.rb

@@ -23,7 +23,7 @@ unless Fog.mocking?
         #     * 'Content-Type'<~String> - MIME type of object
         #     * 'ETag'<~String> - Etag of object
         #     * 'Last-Modified'<~String> - Last modified timestamp for object
-        def get_object(bucket_name, object_name, options = {})
+        def get_object(bucket_name, object_name, options = {}, &block)
           unless bucket_name
             raise ArgumentError.new('bucket_name is required')
           end
@@ -39,7 +39,8 @@ unless Fog.mocking?
             :headers  => headers,
             :host     => "#{bucket_name}.#{@host}",
             :method   => 'GET',
-            :path     => object_name
+            :path     => object_name,
+            :block    => block
           })
         end
 

lib/fog/aws/requests/s3/put_object.rb

@@ -9,7 +9,7 @@ unless Fog.mocking?
         # ==== Parameters
         # * bucket_name<~String> - Name of bucket to create object in
         # * object_name<~String> - Name of object to create
-        # * object<~String> - File to create object from
+        # * data<~File> - File or String to create object from
         # * options<~Hash>:
         #   * 'Cache-Control'<~String> - Caching behaviour
         #   * 'Content-Disposition'<~String> - Presentational information for the object

lib/fog/aws/s3.rb

@@ -87,20 +87,18 @@ module Fog
           :headers => {}
         }
 
-        if data.respond_to?(:path)
+        if data.is_a?(String)
+          metadata[:body] = data
+          metadata[:headers]['Content-Length'] = metadata[:body].size.to_s
+        else
           filename = File.basename(data.path)
           unless (mime_types = MIME::Types.of(filename)).empty?
             metadata[:headers]['Content-Type'] = mime_types.first.content_type
           end
-        end
-
-        if data.respond_to?(:read)
           metadata[:body] = data.read
-        else
+          metadata[:headers]['Content-Length'] = File.size(data.path)
-          metadata[:body] = data
         end
-        metadata[:headers]['Content-Length'] = metadata[:body].size.to_s
+        # metadata[:headers]['Content-MD5'] = Base64.encode64(Digest::MD5.digest(metadata[:body])).strip
-        metadata[:headers]['Content-MD5'] = Base64.encode64(Digest::MD5.digest(metadata[:body])).strip
         metadata
       end
 
@@ -143,6 +141,7 @@ DATA
         params[:headers]['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
 
         response = @connection.request({
+          :block    => params[:block],
           :body     => params[:body],
           :expects  => params[:expects],
           :headers  => params[:headers],

lib/fog/connection.rb

@@ -34,15 +34,26 @@ unless Fog.mocking?
         request = "#{params[:method]} #{params[:path]} HTTP/1.1\r\n"
         params[:headers] ||= {}
         params[:headers]['Host'] = params[:host]
-        if params[:body]
+        if params[:body] && !params[:headers]['Content-Length']
           params[:headers]['Content-Length'] = params[:body].length
         end
         for key, value in params[:headers]
           request << "#{key}: #{value}\r\n"
         end
-        request << "\r\n#{params[:body]}"
+        request << "\r\n"
         @connection.write(request)
 
+        if params[:body]
+          if params[:body].is_a?(String)
+            body = StringIO.new(params[:body])
+          else
+            body = params[:body]
+          end
+          while chunk = body.read(1048576) # 1 megabyte
+            @connection.write(chunk)
+          end
+        end
+
         response = Fog::Response.new
         response.request = params
         response.status = @connection.readline[9..11].to_i
@@ -65,19 +76,30 @@ unless Fog.mocking?
             parser = params[:parser]
           end
           body = Nokogiri::XML::SAX::PushParser.new(parser)
+        elsif params[:block]
+          body = nil
         else
           body = ''
         end
 
         unless params[:method] == 'HEAD'
           if response.headers['Content-Length']
-            body << @connection.read(response.headers['Content-Length'].to_i)
+            content = @connection.read(response.headers['Content-Length'].to_i)
+            unless params[:block]
+              body << content
+            else
+              params[:block].call(content)
+            end
           elsif response.headers['Transfer-Encoding'] == 'chunked'
             while true
               # 2 == "/r/n".length
               chunk_size = @connection.readline.chomp!.to_i(16) + 2
-              chunk = @connection.read(chunk_size)
+              chunk = @connection.read(chunk_size)[0...-2]
-              body << chunk[0...-2]
+              unless params[:block]
+                body << chunk
+              else
+                params[:block].call(chunk)
+              end
               if chunk_size == 2
                 break
               end

spec/aws/requests/s3/get_object_spec.rb

@@ -26,6 +26,15 @@ describe 'S3.get_object' do
       actual.headers['Last-Modified'].should be_a(String)
     end
 
+    it 'should return chunks with optional block' do
+      file = File.open(File.dirname(__FILE__) + '/../../../lorem.txt', 'r')
+      data = ''
+      s3.get_object('foggetobject', 'fog_get_object') do |chunk|
+        data << chunk
+      end
+      data.should == file.read
+    end
+
   end
   describe 'failure' do