Merge commit 'technoweenie/multipart_streaming' into prereorg

Conflicts:
	README.rdoc
	lib/rest_client.rb
	lib/rest_client/resource.rb
	rest-client.gemspec
	spec/rest_client_spec.rb
This commit is contained in:
François Beausoleil 2009-08-12 12:37:21 -04:00
commit 280e344de4
11 changed files with 357 additions and 87 deletions

View File

@ -13,15 +13,38 @@ of specifying actions: get, put, post, delete.
RestClient.post 'http://example.com/resource', :param1 => 'one', :nested => { :param2 => 'two' }
RestClient.delete 'http://example.com/resource'
== Multipart
See RestClient module docs for details.
Yeah, that's right! This does multipart sends for you!
RestClient.post '/data', :myfile => File.new("/path/to/image.jpg")
This does two things for you:
* Auto-detects that you have a File value sends it as multipart
* Auto-detects the mime of the file and sets it in the HEAD of the payload for each entry
If you are sending params that do not contain a File object but the payload needs to be multipart then:
RestClient.post '/data', :foo => 'bar', :multipart => true
== Streaming downloads
RestClient.get('http://some/resource/lotsofdata') do |res|
res.read_body do |chunk|
.. do something with chunk ..
end
end
See RestClient module docs for more details.
== Usage: ActiveResource-Style
resource = RestClient::Resource.new 'http://example.com/resource'
resource.get
private_resource = RestClient::Resource.new 'https://example.com/private/resource', :user => 'adam', :password => 'secret', :timeout => 20
private_resource = RestClient::Resource.new 'https://example.com/private/resource', 'user', 'pass'
private_resource.put File.read('pic.jpg'), :content_type => 'image/jpg'
See RestClient::Resource module docs for details.
@ -65,53 +88,13 @@ Then invoke:
$ restclient private_site
Use as a one-off, curl-style:
$ restclient get http://example.com/resource > output_body
$ restclient put http://example.com/resource < input_body
== Logging
Write calls to a log filename (can also be "stdout" or "stderr"):
RestClient.log = '/tmp/restclient.log'
Or set an environment variable to avoid modifying the code:
$ RESTCLIENT_LOG=stdout path/to/my/program
Either produces logs like this:
RestClient.get "http://some/resource"
# => 200 OK | text/html 250 bytes
RestClient.put "http://some/resource", "payload"
# => 401 Unauthorized | application/xml 340 bytes
Note that these logs are valid Ruby, so you can paste them into the restclient
shell or a script to replay your sequence of rest calls.
== Proxy
All calls to RestClient, including Resources, will use the proxy specified by
RestClient.proxy:
RestClient.proxy = "http://proxy.example.com/"
RestClient.get "http://some/resource"
# => response from some/resource as proxied through proxy.example.com
Often the proxy url is set in an environment variable, so you can do this to
use whatever proxy the system is configured to use:
RestClient.proxy = ENV['http_proxy']
== Meta
Written by Adam Wiggins (adam at heroku dot com)
Patches contributed by: Chris Anderson, Greg Borenstein, Ardekantur, Pedro
Belo, Rafael Souza, Rick Olson, Aman Gupta, Blake Mizerany, Brian Donovan, Ivan
Makfinsky, Marc-André Cournoyer, Coda Hale, and Tetsuo Watanabe
Major modifications by Blake Mizerany
Patches contributed by: Chris Anderson, Greg Borenstein, Ardekantur, Pedro Belo, Rafael Souza, Rick Olson, and Aman Gupta
Released under the MIT License: http://www.opensource.org/licenses/mit-license.php

View File

@ -76,7 +76,7 @@ Rake::RDocTask.new do |t|
t.title = "rest-client, fetch RESTful resources effortlessly"
t.options << '--line-numbers' << '--inline-source' << '-A cattr_accessor=object'
t.options << '--charset' << 'utf-8'
t.rdoc_files.include('README')
t.rdoc_files.include('README.rdoc')
t.rdoc_files.include('lib/*.rb')
end

View File

@ -3,8 +3,10 @@ require 'net/https'
require 'zlib'
require 'stringio'
require File.dirname(__FILE__) + '/resource'
require File.dirname(__FILE__) + '/request_errors'
require File.dirname(__FILE__) + '/rest_client/resource'
require File.dirname(__FILE__) + '/rest_client/request_errors'
require File.dirname(__FILE__) + '/rest_client/payload'
require File.dirname(__FILE__) + '/rest_client/net_http_ext'
# This module's static methods are the entry point for using the REST client.
#
@ -43,30 +45,30 @@ require File.dirname(__FILE__) + '/request_errors'
# => "PUT http://rest-test.heroku.com/resource with a 7 byte payload, content type application/x-www-form-urlencoded {\"foo\"=>\"baz\"}"
#
module RestClient
def self.get(url, headers={})
def self.get(url, headers={}, &b)
Request.execute(:method => :get,
:url => url,
:headers => headers)
:headers => headers, &b)
end
def self.post(url, payload, headers={})
def self.post(url, payload, headers={}, &b)
Request.execute(:method => :post,
:url => url,
:payload => payload,
:headers => headers)
:headers => headers, &b)
end
def self.put(url, payload, headers={})
def self.put(url, payload, headers={}, &b)
Request.execute(:method => :put,
:url => url,
:payload => payload,
:headers => headers)
:headers => headers, &b)
end
def self.delete(url, headers={})
def self.delete(url, headers={}, &b)
Request.execute(:method => :delete,
:url => url,
:headers => headers)
:headers => headers, &b)
end
class <<self
@ -89,30 +91,30 @@ module RestClient
class Request
attr_reader :method, :url, :payload, :headers, :user, :password, :timeout
def self.execute(args)
new(args).execute
def self.execute(args, &b)
new(args).execute(&b)
end
def initialize(args)
@method = args[:method] or raise ArgumentError, "must pass :method"
@url = args[:url] or raise ArgumentError, "must pass :url"
@payload = Payload.generate(args[:payload] || '')
@headers = args[:headers] || {}
@payload = process_payload(args[:payload])
@user = args[:user]
@password = args[:password]
@timeout = args[:timeout]
end
def execute
execute_inner
def execute(&b)
execute_inner(&b)
rescue Redirect => e
@url = e.url
execute
execute(&b)
end
def execute_inner
def execute_inner(&b)
uri = parse_url_with_auth(url)
transmit uri, net_http_request_class(method).new(uri.request_uri, make_headers(headers)), payload
transmit(uri, net_http_request_class(method).new(uri.request_uri, make_headers(headers)), payload, &b)
end
def make_headers(user_headers)
@ -164,7 +166,7 @@ module RestClient
end
end
def transmit(uri, req, payload)
def transmit(uri, req, payload, &b)
setup_credentials(req)
net = net_http_class.new(uri.host, uri.port)
@ -177,7 +179,10 @@ module RestClient
http.read_timeout = @timeout if @timeout
res = http.request(req, payload)
display_log response_log(res)
string = process_result(res)
## Ok. I know this is weird but it's a hack for now
## this lets process_result determine if it should read the body
## into memory or not
string = process_result(http.request(req, payload || "", &b), &b)
if string
Response.new(string, res)
else
@ -188,15 +193,17 @@ module RestClient
raise RestClient::ServerBrokeConnection
rescue Timeout::Error
raise RestClient::RequestTimeout
ensure
payload.close
end
def setup_credentials(req)
req.basic_auth(user, password) if user
end
def process_result(res)
def process_result(res, &b)
if %w(200 201 202).include? res.code
decode res['content-encoding'], res.body
decode(res['content-encoding'], res.body) unless b
elsif %w(301 302 303).include? res.code
url = res.header['Location']
@ -217,6 +224,9 @@ module RestClient
raise RequestFailed, res
end
end
def payload
@payload
def decode(content_encoding, body)
if content_encoding == 'gzip'
@ -253,7 +263,7 @@ module RestClient
end
def default_headers
{ :accept => 'application/xml', :accept_encoding => 'gzip, deflate' }
@payload.headers.merge({ :accept => 'application/xml', :accept_encoding => 'gzip, deflate' })
end
end

View File

@ -0,0 +1,21 @@
#
# Replace the request method in Net::HTTP to sniff the body type
# and set the stream if appropriate
#
# Taken from:
# http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
module Net
class HTTP
alias __request__ request
def request(req, body=nil, &block)
if body != nil && body.respond_to?(:read)
req.body_stream = body
return __request__(req, nil, &block)
else
return __request__(req, body, &block)
end
end
end
end

184
lib/rest_client/payload.rb Normal file
View File

@ -0,0 +1,184 @@
require "tempfile"
require "stringio"
module RestClient
module Payload
extend self
def generate(params)
if params.is_a?(String)
Base.new(params)
elsif params.delete(:multipart) == true ||
params.any? { |_,v| v.respond_to?(:path) && v.respond_to?(:read) }
Multipart.new(params)
else
UrlEncoded.new(params)
end
end
class Base
def initialize(params)
build_stream(params)
end
def build_stream(params)
@stream = StringIO.new(params)
@stream.seek(0)
end
def read(bytes=nil)
@stream.read(bytes)
end
alias :to_s :read
def escape(v)
URI.escape(v.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
end
def headers
{ 'Content-Length' => size.to_s }
end
def size
@stream.size
end
alias :length :size
def close
@stream.close
end
end
class UrlEncoded < Base
def build_stream(params)
@stream = StringIO.new(params.map do |k,v|
"#{escape(k)}=#{escape(v)}"
end.join("&"))
@stream.seek(0)
end
def headers
super.merge({ 'Content-Type' => 'application/x-www-form-urlencoded' })
end
end
class Multipart < Base
EOL = "\r\n"
def build_stream(params)
b = "--#{boundary}"
@stream = Tempfile.new("RESTClient.Stream.#{rand(1000)}")
@stream.write(b + EOL)
params.each do |k,v|
if v.respond_to?(:read) && v.respond_to?(:path)
create_file_field(@stream, k,v)
else
create_regular_field(@stream, k,v)
end
@stream.write(EOL + b)
end
@stream.write('--')
@stream.write(EOL)
@stream.seek(0)
end
def create_regular_field(s, k, v)
s.write("Content-Disposition: multipart/form-data; name=\"#{k}\"")
s.write(EOL)
s.write(EOL)
s.write(v)
end
def create_file_field(s, k, v)
begin
s.write("Content-Disposition: multipart/form-data; name=\"#{k}\"; filename=\"#{v.path}\"#{EOL}")
s.write("Content-Type: #{mime_for(v.path)}#{EOL}")
s.write(EOL)
while data = v.read(8124)
s.write(data)
end
ensure
v.close
end
end
def mime_for(path)
ext = File.extname(path)[1..-1]
MIME_TYPES[ext] || 'text/plain'
end
def boundary
@boundary ||= rand(1_000_000).to_s
end
def headers
super.merge({'Content-Type' => %Q{multipart/form-data; boundary="#{boundary}"}})
end
def close
@stream.close
end
end
# :stopdoc:
# From WEBrick.
MIME_TYPES = {
"ai" => "application/postscript",
"asc" => "text/plain",
"avi" => "video/x-msvideo",
"bin" => "application/octet-stream",
"bmp" => "image/bmp",
"class" => "application/octet-stream",
"cer" => "application/pkix-cert",
"crl" => "application/pkix-crl",
"crt" => "application/x-x509-ca-cert",
"css" => "text/css",
"dms" => "application/octet-stream",
"doc" => "application/msword",
"dvi" => "application/x-dvi",
"eps" => "application/postscript",
"etx" => "text/x-setext",
"exe" => "application/octet-stream",
"gif" => "image/gif",
"gz" => "application/x-gzip",
"htm" => "text/html",
"html" => "text/html",
"jpe" => "image/jpeg",
"jpeg" => "image/jpeg",
"jpg" => "image/jpeg",
"js" => "text/javascript",
"lha" => "application/octet-stream",
"lzh" => "application/octet-stream",
"mov" => "video/quicktime",
"mpe" => "video/mpeg",
"mpeg" => "video/mpeg",
"mpg" => "video/mpeg",
"pbm" => "image/x-portable-bitmap",
"pdf" => "application/pdf",
"pgm" => "image/x-portable-graymap",
"png" => "image/png",
"pnm" => "image/x-portable-anymap",
"ppm" => "image/x-portable-pixmap",
"ppt" => "application/vnd.ms-powerpoint",
"ps" => "application/postscript",
"qt" => "video/quicktime",
"ras" => "image/x-cmu-raster",
"rb" => "text/plain",
"rd" => "text/plain",
"rtf" => "application/rtf",
"sgm" => "text/sgml",
"sgml" => "text/sgml",
"tif" => "image/tiff",
"tiff" => "image/tiff",
"txt" => "text/plain",
"xbm" => "image/x-xbitmap",
"xls" => "application/vnd.ms-excel",
"xml" => "text/xml",
"xpm" => "image/x-xpixmap",
"xwd" => "image/x-xwindowdump",
"zip" => "application/zip",
}
# :startdoc:
end
end

View File

@ -41,38 +41,34 @@ module RestClient
end
end
def get(additional_headers={})
def get(additional_headers={}, &b)
Request.execute(options.merge(
:method => :get,
:url => url,
:headers => headers.merge(additional_headers)
))
:headers => additional_headers), &b)
end
def post(payload, additional_headers={})
def post(payload, additional_headers={}, &b)
Request.execute(options.merge(
:method => :post,
:url => url,
:payload => payload,
:headers => headers.merge(additional_headers)
))
:headers => additional_headers), &b)
end
def put(payload, additional_headers={})
def put(payload, additional_headers={}, &b)
Request.execute(options.merge(
:method => :put,
:url => url,
:payload => payload,
:headers => headers.merge(additional_headers)
))
:headers => additional_headers), &b)
end
def delete(additional_headers={})
def delete(additional_headers={}, &b)
Request.execute(options.merge(
:method => :delete,
:url => url,
:headers => headers.merge(additional_headers)
))
:headers => additional_headers), &b)
end
def to_s
@ -93,6 +89,9 @@ module RestClient
def timeout
options[:timeout]
:user => user,
:password => password,
:headers => headers, &b)
end
# Construct a subresource, preserving authentication.

View File

@ -2,3 +2,4 @@ require 'rubygems'
require 'spec'
require File.dirname(__FILE__) + '/../lib/rest_client'

BIN
spec/master_shake.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

67
spec/payload_spec.rb Normal file
View File

@ -0,0 +1,67 @@
require File.dirname(__FILE__) + "/base"
describe RestClient::Payload do
context "A regular Payload" do
it "should should default content-type to standard enctype" do
RestClient::Payload::UrlEncoded.new({}).headers['Content-Type'].
should == 'application/x-www-form-urlencoded'
end
it "should form properly encoded params" do
RestClient::Payload::UrlEncoded.new({:foo => 'bar'}).to_s.
should == "foo=bar"
end
end
context "A multipart Payload" do
it "should should default content-type to standard enctype" do
m = RestClient::Payload::Multipart.new({})
m.stub!(:boundary).and_return(123)
m.headers['Content-Type'].should == 'multipart/form-data; boundary="123"'
end
it "should form properly seperated multipart data" do
m = RestClient::Payload::Multipart.new({:foo => "bar"})
m.to_s.should == <<-EOS
--#{m.boundary}\r
Content-Disposition: multipart/form-data; name="foo"\r
\r
bar\r
--#{m.boundary}--\r
EOS
end
it "should form properly seperated multipart data" do
f = File.new(File.dirname(__FILE__) + "/master_shake.jpg")
m = RestClient::Payload::Multipart.new({:foo => f})
m.to_s.should == <<-EOS
--#{m.boundary}\r
Content-Disposition: multipart/form-data; name="foo"; filename="./spec/master_shake.jpg"\r
Content-Type: image/jpeg\r
\r
#{IO.read(f.path)}\r
--#{m.boundary}--\r
EOS
end
end
context "Payload generation" do
it "should recognize standard urlencoded params" do
RestClient::Payload.generate({"foo" => 'bar'}).should be_kind_of(RestClient::Payload::UrlEncoded)
end
it "should recognize multipart params" do
f = File.new(File.dirname(__FILE__) + "/master_shake.jpg")
RestClient::Payload.generate({"foo" => f}).should be_kind_of(RestClient::Payload::Multipart)
end
it "should be multipart if forced" do
RestClient::Payload.generate({"foo" => "bar", :multipart => true}).should be_kind_of(RestClient::Payload::Multipart)
end
it "should return data if no of the above" do
RestClient::Payload.generate("data").should be_kind_of(RestClient::Payload::Base)
end
end
end

View File

@ -1,5 +1,9 @@
require File.dirname(__FILE__) + '/base'
def generate_payload(v)
RestClient::Payload::Base.new(v)
end
describe RestClient do
context "public API" do
it "GET" do
@ -140,15 +144,15 @@ describe RestClient do
klass = mock("net:http class")
@request.should_receive(:net_http_request_class).with(:put).and_return(klass)
klass.should_receive(:new).and_return('result')
@request.should_receive(:transmit).with(@uri, 'result', 'payload')
@request.should_receive(:transmit).with(@uri, 'result', be_kind_of(RestClient::Payload::Base))
@request.execute_inner
end
it "transmits the request with Net::HTTP" do
@http.should_receive(:request).with('req', 'payload')
@http.should_receive(:request).with('req', be_kind_of(RestClient::Payload::Base))
@request.should_receive(:process_result)
@request.should_receive(:response_log)
@request.transmit(@uri, 'req', 'payload')
@request.transmit(@uri, 'req', generate_payload('payload'))
end
it "uses SSL when the URI refers to a https address" do
@ -157,7 +161,7 @@ describe RestClient do
@http.stub!(:request)
@request.stub!(:process_result)
@request.stub!(:response_log)
@request.transmit(@uri, 'req', 'payload')
@request.transmit(@uri, 'req', generate_payload('payload'))
end
it "sends nil payloads" do
@ -196,7 +200,7 @@ describe RestClient do
@request.stub!(:password).and_return('mypass')
@request.should_receive(:setup_credentials).with('req')
@request.transmit(@uri, 'req', nil)
@request.transmit(@uri, 'req', generate_payload(''))
end
it "does not attempt to send any credentials if user is nil" do
@ -216,7 +220,8 @@ describe RestClient do
it "catches EOFError and shows the more informative ServerBrokeConnection" do
@http.stub!(:request).and_raise(EOFError)
lambda { @request.transmit(@uri, 'req', nil) }.should raise_error(RestClient::ServerBrokeConnection)
lambda { @request.transmit(@uri, 'req', generate_payload('')) }.
should raise_error(RestClient::ServerBrokeConnection)
end
it "execute calls execute_inner" do