Merge commit 'technoweenie/multipart_streaming' into prereorg

Conflicts:
	README.rdoc
	lib/rest_client.rb
	lib/rest_client/resource.rb
	rest-client.gemspec
	spec/rest_client_spec.rb

README.rdoc

@@ -14,14 +14,37 @@ of specifying actions: get, put, post, delete.
 
   RestClient.delete 'http://example.com/resource'
   
-See RestClient module docs for details.
+== Multipart
+
+Yeah, that's right!  This does multipart sends for you!
+
+  RestClient.post '/data', :myfile => File.new("/path/to/image.jpg")
+
+This does two things for you:
+
+* Auto-detects that you have a File value sends it as multipart
+* Auto-detects the mime of the file and sets it in the HEAD of the payload for each entry
+
+If you are sending params that do not contain a File object but the payload needs to be multipart then:
+
+  RestClient.post '/data', :foo => 'bar', :multipart => true
+
+== Streaming downloads
+
+RestClient.get('http://some/resource/lotsofdata') do |res|
+  res.read_body do |chunk|
+    .. do something with chunk ..
+  end
+end
+
+See RestClient module docs for more details.
 
 == Usage: ActiveResource-Style
 
   resource = RestClient::Resource.new 'http://example.com/resource'
   resource.get
 
-  private_resource = RestClient::Resource.new 'https://example.com/private/resource', :user => 'adam', :password => 'secret', :timeout => 20
+  private_resource = RestClient::Resource.new 'https://example.com/private/resource', 'user', 'pass'
   private_resource.put File.read('pic.jpg'), :content_type => 'image/jpg'
 
 See RestClient::Resource module docs for details.
@@ -65,53 +88,13 @@ Then invoke:
 
   $ restclient private_site
   
-Use as a one-off, curl-style:
-
-  $ restclient get http://example.com/resource > output_body
-
-  $ restclient put http://example.com/resource < input_body
-
-== Logging
-
-Write calls to a log filename (can also be "stdout" or "stderr"):
-
-  RestClient.log = '/tmp/restclient.log'
-
-Or set an environment variable to avoid modifying the code:
-
-  $ RESTCLIENT_LOG=stdout path/to/my/program
-
-Either produces logs like this:
-
-  RestClient.get "http://some/resource"
-  # => 200 OK | text/html 250 bytes
-  RestClient.put "http://some/resource", "payload"
-  # => 401 Unauthorized | application/xml 340 bytes
-
-Note that these logs are valid Ruby, so you can paste them into the restclient
-shell or a script to replay your sequence of rest calls.
-
-== Proxy
-
-All calls to RestClient, including Resources, will use the proxy specified by
-RestClient.proxy:
-
-  RestClient.proxy = "http://proxy.example.com/"
-  RestClient.get "http://some/resource"
-  # => response from some/resource as proxied through proxy.example.com
-
-Often the proxy url is set in an environment variable, so you can do this to
-use whatever proxy the system is configured to use:
-
-  RestClient.proxy = ENV['http_proxy']
-
 == Meta
 
 Written by Adam Wiggins (adam at heroku dot com)
 
-Patches contributed by: Chris Anderson, Greg Borenstein, Ardekantur, Pedro
+Major modifications by Blake Mizerany
-Belo, Rafael Souza, Rick Olson, Aman Gupta, Blake Mizerany, Brian Donovan, Ivan
+
-Makfinsky, Marc-André Cournoyer, Coda Hale, and Tetsuo Watanabe
+Patches contributed by: Chris Anderson, Greg Borenstein, Ardekantur, Pedro Belo, Rafael Souza, Rick Olson, and Aman Gupta
 
 Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
 

Rakefile

@@ -76,7 +76,7 @@ Rake::RDocTask.new do |t|
 	t.title    = "rest-client, fetch RESTful resources effortlessly"
 	t.options << '--line-numbers' << '--inline-source' << '-A cattr_accessor=object'
 	t.options << '--charset' << 'utf-8'
-	t.rdoc_files.include('README')
+	t.rdoc_files.include('README.rdoc')
 	t.rdoc_files.include('lib/*.rb')
 end
 

lib/rest_client.rb

@@ -3,8 +3,10 @@ require 'net/https'
 require 'zlib'
 require 'stringio'
 
-require File.dirname(__FILE__) + '/resource'
+require File.dirname(__FILE__) + '/rest_client/resource'
-require File.dirname(__FILE__) + '/request_errors'
+require File.dirname(__FILE__) + '/rest_client/request_errors'
+require File.dirname(__FILE__) + '/rest_client/payload'
+require File.dirname(__FILE__) + '/rest_client/net_http_ext'
 
 # This module's static methods are the entry point for using the REST client.
 #
@@ -43,30 +45,30 @@ require File.dirname(__FILE__) + '/request_errors'
 #   => "PUT http://rest-test.heroku.com/resource with a 7 byte payload, content type application/x-www-form-urlencoded {\"foo\"=>\"baz\"}"
 #
 module RestClient
-	def self.get(url, headers={})
+	def self.get(url, headers={}, &b)
 		Request.execute(:method => :get,
 			:url => url,
-			:headers => headers)
+			:headers => headers, &b)
 	end
 
-	def self.post(url, payload, headers={})
+	def self.post(url, payload, headers={}, &b)
 		Request.execute(:method => :post,
 			:url => url,
 			:payload => payload,
-			:headers => headers)
+			:headers => headers, &b)
 	end
 
-	def self.put(url, payload, headers={})
+	def self.put(url, payload, headers={}, &b)
 		Request.execute(:method => :put,
 			:url => url,
 			:payload => payload,
-			:headers => headers)
+			:headers => headers, &b)
 	end
 
-	def self.delete(url, headers={})
+	def self.delete(url, headers={}, &b)
 		Request.execute(:method => :delete,
 			:url => url,
-			:headers => headers)
+			:headers => headers, &b)
 	end
 
 	class <<self
@@ -89,30 +91,30 @@ module RestClient
 	class Request
 		attr_reader :method, :url, :payload, :headers, :user, :password, :timeout
 
-		def self.execute(args)
+		def self.execute(args, &b)
-			new(args).execute
+			new(args).execute(&b)
 		end
 
 		def initialize(args)
 			@method = args[:method] or raise ArgumentError, "must pass :method"
 			@url = args[:url] or raise ArgumentError, "must pass :url"
+			@payload = Payload.generate(args[:payload] || '')
 			@headers = args[:headers] || {}
-			@payload = process_payload(args[:payload])
 			@user = args[:user]
 			@password = args[:password]
 			@timeout = args[:timeout]
 		end
 
-		def execute
+		def execute(&b)
-			execute_inner
+			execute_inner(&b)
 		rescue Redirect => e
 			@url = e.url
-			execute
+			execute(&b)
 		end
 
-		def execute_inner
+		def execute_inner(&b)
 			uri = parse_url_with_auth(url)
-			transmit uri, net_http_request_class(method).new(uri.request_uri, make_headers(headers)), payload
+			transmit(uri, net_http_request_class(method).new(uri.request_uri, make_headers(headers)), payload, &b)
 		end
 
 		def make_headers(user_headers)
@@ -164,7 +166,7 @@ module RestClient
 			end
 		end
 
-		def transmit(uri, req, payload)
+		def transmit(uri, req, payload, &b)
 			setup_credentials(req)
 
 			net = net_http_class.new(uri.host, uri.port)
@@ -177,7 +179,10 @@ module RestClient
 				http.read_timeout = @timeout if @timeout
 				res = http.request(req, payload)
 				display_log response_log(res)
-				string = process_result(res)
+				## Ok. I know this is weird but it's a hack for now
+				## this lets process_result determine if it should read the body
+				## into memory or not
+				string = process_result(http.request(req, payload || "", &b), &b)
 				if string
 					Response.new(string, res)
 				else
@@ -188,15 +193,17 @@ module RestClient
 			raise RestClient::ServerBrokeConnection
 		rescue Timeout::Error
 			raise RestClient::RequestTimeout
+		ensure
+			payload.close
 		end
 
 		def setup_credentials(req)
 			req.basic_auth(user, password) if user
 		end
 
-		def process_result(res)
+		def process_result(res, &b)
 			if %w(200 201 202).include? res.code
-				decode res['content-encoding'], res.body
+				decode(res['content-encoding'], res.body) unless b
 			elsif %w(301 302 303).include? res.code
 				url = res.header['Location']
 
@@ -218,6 +225,9 @@ module RestClient
 			end
 		end
 		
+		def payload
+			@payload
+
 		def decode(content_encoding, body)
 			if content_encoding == 'gzip'
 				Zlib::GzipReader.new(StringIO.new(body)).read
@@ -253,7 +263,7 @@ module RestClient
 		end
 
 		def default_headers
-			{ :accept => 'application/xml', :accept_encoding => 'gzip, deflate' }
+			@payload.headers.merge({ :accept => 'application/xml', :accept_encoding => 'gzip, deflate' })
 		end
 	end
 

lib/rest_client/net_http_ext.rb

@@ -0,0 +1,21 @@
+#
+# Replace the request method in Net::HTTP to sniff the body type
+# and set the stream if appropriate
+#
+# Taken from:	
+# http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
+
+module Net
+	class HTTP
+		alias __request__ request
+
+		def request(req, body=nil, &block)
+			if body != nil && body.respond_to?(:read)
+				req.body_stream = body
+				return __request__(req, nil, &block)
+			else
+				return __request__(req, body, &block)
+			end
+		end
+	end
+end

lib/rest_client/payload.rb

@@ -0,0 +1,184 @@
+require "tempfile"
+require "stringio"
+
+module RestClient
+	module Payload
+		extend self
+
+		def generate(params)
+			if params.is_a?(String)
+				Base.new(params)
+			elsif params.delete(:multipart) == true || 
+				params.any? { |_,v| v.respond_to?(:path) && v.respond_to?(:read) }
+				Multipart.new(params)
+			else
+				UrlEncoded.new(params)
+			end
+		end
+
+		class Base
+			def initialize(params)
+				build_stream(params)
+			end
+
+			def build_stream(params)
+				@stream = StringIO.new(params)
+				@stream.seek(0)
+			end
+
+			def read(bytes=nil)
+				@stream.read(bytes)
+			end
+			alias :to_s :read
+
+			def escape(v)
+				URI.escape(v.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+			end
+
+			def headers
+				{ 'Content-Length' => size.to_s }
+			end
+
+			def size
+				@stream.size
+			end
+			alias :length :size
+
+			def close
+				@stream.close
+			end
+		end
+
+		class UrlEncoded < Base
+			def build_stream(params)
+				@stream = StringIO.new(params.map do |k,v| 
+					"#{escape(k)}=#{escape(v)}"
+				end.join("&"))
+				@stream.seek(0)
+			end
+
+			def headers
+				super.merge({ 'Content-Type' => 'application/x-www-form-urlencoded' })
+			end
+		end
+
+		class Multipart < Base
+			EOL = "\r\n"
+
+			def build_stream(params)
+				b = "--#{boundary}"
+
+				@stream = Tempfile.new("RESTClient.Stream.#{rand(1000)}")
+				@stream.write(b + EOL)
+				params.each do |k,v|
+					if v.respond_to?(:read) && v.respond_to?(:path)
+						create_file_field(@stream, k,v)
+					else
+						create_regular_field(@stream, k,v)
+					end
+					@stream.write(EOL + b)
+				end
+				@stream.write('--')
+				@stream.write(EOL)
+				@stream.seek(0)
+			end
+
+			def create_regular_field(s, k, v)
+				s.write("Content-Disposition: multipart/form-data; name=\"#{k}\"")
+				s.write(EOL)
+				s.write(EOL)
+				s.write(v)
+			end
+
+			def create_file_field(s, k, v)
+				begin
+					s.write("Content-Disposition: multipart/form-data; name=\"#{k}\"; filename=\"#{v.path}\"#{EOL}")
+					s.write("Content-Type: #{mime_for(v.path)}#{EOL}")
+					s.write(EOL)
+					while data = v.read(8124)
+						s.write(data)
+					end
+				ensure
+					v.close
+				end
+			end
+
+			def mime_for(path)
+				ext = File.extname(path)[1..-1]
+				MIME_TYPES[ext] || 'text/plain'
+			end
+
+			def boundary
+				@boundary ||= rand(1_000_000).to_s
+			end
+
+			def headers
+				super.merge({'Content-Type' => %Q{multipart/form-data; boundary="#{boundary}"}})
+			end
+
+			def close
+				@stream.close
+			end
+		end
+
+		# :stopdoc:
+		# From WEBrick.
+		MIME_TYPES = {
+			"ai"    => "application/postscript",
+			"asc"   => "text/plain",
+			"avi"   => "video/x-msvideo",
+			"bin"   => "application/octet-stream",
+			"bmp"   => "image/bmp",
+			"class" => "application/octet-stream",
+			"cer"   => "application/pkix-cert",
+			"crl"   => "application/pkix-crl",
+			"crt"   => "application/x-x509-ca-cert",
+			"css"   => "text/css",
+			"dms"   => "application/octet-stream",
+			"doc"   => "application/msword",
+			"dvi"   => "application/x-dvi",
+			"eps"   => "application/postscript",
+			"etx"   => "text/x-setext",
+			"exe"   => "application/octet-stream",
+			"gif"   => "image/gif",
+			"gz"   => "application/x-gzip",
+			"htm"   => "text/html",
+			"html"  => "text/html",
+			"jpe"   => "image/jpeg",
+			"jpeg"  => "image/jpeg",
+			"jpg"   => "image/jpeg",
+			"js"    => "text/javascript",
+			"lha"   => "application/octet-stream",
+			"lzh"   => "application/octet-stream",
+			"mov"   => "video/quicktime",
+			"mpe"   => "video/mpeg",
+			"mpeg"  => "video/mpeg",
+			"mpg"   => "video/mpeg",
+			"pbm"   => "image/x-portable-bitmap",
+			"pdf"   => "application/pdf",
+			"pgm"   => "image/x-portable-graymap",
+			"png"   => "image/png",
+			"pnm"   => "image/x-portable-anymap",
+			"ppm"   => "image/x-portable-pixmap",
+			"ppt"   => "application/vnd.ms-powerpoint",
+			"ps"    => "application/postscript",
+			"qt"    => "video/quicktime",
+			"ras"   => "image/x-cmu-raster",
+			"rb"    => "text/plain",
+			"rd"    => "text/plain",
+			"rtf"   => "application/rtf",
+			"sgm"   => "text/sgml",
+			"sgml"  => "text/sgml",
+			"tif"   => "image/tiff",
+			"tiff"  => "image/tiff",
+			"txt"   => "text/plain",
+			"xbm"   => "image/x-xbitmap",
+			"xls"   => "application/vnd.ms-excel",
+			"xml"   => "text/xml",
+			"xpm"   => "image/x-xpixmap",
+			"xwd"   => "image/x-xwindowdump",
+			"zip"   => "application/zip",
+		}
+		# :startdoc:
+	end
+end

lib/rest_client/resource.rb

@@ -41,38 +41,34 @@ module RestClient
 			end
 		end
 
-		def get(additional_headers={})
+		def get(additional_headers={}, &b)
 			Request.execute(options.merge(
 				:method => :get,
 				:url => url,
-				:headers => headers.merge(additional_headers)
+				:headers => additional_headers), &b)
-			))
 		end
 
-		def post(payload, additional_headers={})
+		def post(payload, additional_headers={}, &b)
 			Request.execute(options.merge(
 				:method => :post,
 				:url => url,
 				:payload => payload,
-				:headers => headers.merge(additional_headers)
+				:headers => additional_headers), &b)
-			))
 		end
 
-		def put(payload, additional_headers={})
+		def put(payload, additional_headers={}, &b)
 			Request.execute(options.merge(
 				:method => :put,
 				:url => url,
 				:payload => payload,
-				:headers => headers.merge(additional_headers)
+				:headers => additional_headers), &b)
-			))
 		end
 
-		def delete(additional_headers={})
+		def delete(additional_headers={}, &b)
 			Request.execute(options.merge(
 				:method => :delete,
 				:url => url,
-				:headers => headers.merge(additional_headers)
+				:headers => additional_headers), &b)
-			))
 		end
 
 		def to_s
@@ -93,6 +89,9 @@ module RestClient
 
 		def timeout
 			options[:timeout]
+				:user => user,
+				:password => password,
+				:headers => headers, &b)
 		end
 
 		# Construct a subresource, preserving authentication.

spec/base.rb

@@ -2,3 +2,4 @@ require 'rubygems'
 require 'spec'
 
 require File.dirname(__FILE__) + '/../lib/rest_client'
+

spec/payload_spec.rb

@@ -0,0 +1,67 @@
+require File.dirname(__FILE__) + "/base"
+
+describe RestClient::Payload do
+	context "A regular Payload" do
+		it "should should default content-type to standard enctype" do
+			RestClient::Payload::UrlEncoded.new({}).headers['Content-Type'].
+				should == 'application/x-www-form-urlencoded'
+		end
+
+		it "should form properly encoded params" do
+			RestClient::Payload::UrlEncoded.new({:foo => 'bar'}).to_s.
+				should == "foo=bar"
+		end
+	end
+
+	context "A multipart Payload" do
+		it "should should default content-type to standard enctype" do
+			m = RestClient::Payload::Multipart.new({})
+			m.stub!(:boundary).and_return(123)
+			m.headers['Content-Type'].should == 'multipart/form-data; boundary="123"'
+		end
+
+		it "should form properly seperated multipart data" do
+			m = RestClient::Payload::Multipart.new({:foo => "bar"})
+			m.to_s.should == <<-EOS
+--#{m.boundary}\r
+Content-Disposition: multipart/form-data; name="foo"\r
+\r
+bar\r
+--#{m.boundary}--\r
+EOS
+		end
+
+		it "should form properly seperated multipart data" do
+			f = File.new(File.dirname(__FILE__) + "/master_shake.jpg")
+			m = RestClient::Payload::Multipart.new({:foo => f})
+			m.to_s.should == <<-EOS
+--#{m.boundary}\r
+Content-Disposition: multipart/form-data; name="foo"; filename="./spec/master_shake.jpg"\r
+Content-Type: image/jpeg\r
+\r
+#{IO.read(f.path)}\r
+--#{m.boundary}--\r
+EOS
+		end
+	end
+
+	context "Payload generation" do
+		it "should recognize standard urlencoded params" do
+			RestClient::Payload.generate({"foo" => 'bar'}).should be_kind_of(RestClient::Payload::UrlEncoded)
+		end
+		
+		it "should recognize multipart params" do
+			f = File.new(File.dirname(__FILE__) + "/master_shake.jpg")
+
+			RestClient::Payload.generate({"foo" => f}).should be_kind_of(RestClient::Payload::Multipart)
+		end
+		
+		it "should be multipart if forced" do
+			RestClient::Payload.generate({"foo" => "bar", :multipart => true}).should be_kind_of(RestClient::Payload::Multipart)
+		end
+
+		it "should return data if no of the above" do
+		  RestClient::Payload.generate("data").should be_kind_of(RestClient::Payload::Base)
+		end
+	end
+end

spec/rest_client_spec.rb

@@ -1,5 +1,9 @@
 require File.dirname(__FILE__) + '/base'
 
+def generate_payload(v)
+	RestClient::Payload::Base.new(v)
+end
+
 describe RestClient do
 	context "public API" do
 		it "GET" do
@@ -140,15 +144,15 @@ describe RestClient do
 			klass = mock("net:http class")
 			@request.should_receive(:net_http_request_class).with(:put).and_return(klass)
 			klass.should_receive(:new).and_return('result')
-			@request.should_receive(:transmit).with(@uri, 'result', 'payload')
+			@request.should_receive(:transmit).with(@uri, 'result', be_kind_of(RestClient::Payload::Base))
 			@request.execute_inner
 		end
 
 		it "transmits the request with Net::HTTP" do
-			@http.should_receive(:request).with('req', 'payload')
+			@http.should_receive(:request).with('req', be_kind_of(RestClient::Payload::Base))
 			@request.should_receive(:process_result)
 			@request.should_receive(:response_log)
-			@request.transmit(@uri, 'req', 'payload')
+			@request.transmit(@uri, 'req', generate_payload('payload'))
 		end
 
 		it "uses SSL when the URI refers to a https address" do
@@ -157,7 +161,7 @@ describe RestClient do
 			@http.stub!(:request)
 			@request.stub!(:process_result)
 			@request.stub!(:response_log)
-			@request.transmit(@uri, 'req', 'payload')
+			@request.transmit(@uri, 'req', generate_payload('payload'))
 		end
 
 		it "sends nil payloads" do
@@ -196,7 +200,7 @@ describe RestClient do
 			@request.stub!(:password).and_return('mypass')
 			@request.should_receive(:setup_credentials).with('req')
 
-			@request.transmit(@uri, 'req', nil)
+			@request.transmit(@uri, 'req', generate_payload(''))
 		end
 
 		it "does not attempt to send any credentials if user is nil" do
@@ -216,7 +220,8 @@ describe RestClient do
 
 		it "catches EOFError and shows the more informative ServerBrokeConnection" do
 			@http.stub!(:request).and_raise(EOFError)
-			lambda { @request.transmit(@uri, 'req', nil) }.should raise_error(RestClient::ServerBrokeConnection)
+			lambda { @request.transmit(@uri, 'req', generate_payload('')) }.
+				should raise_error(RestClient::ServerBrokeConnection)
 		end
 
 		it "execute calls execute_inner" do