kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

webrick/httpproxy: stream request and response bodies

Browse source 
Reading entire request or response bodies into memory can lead
to trivial denial-of-service attacks.  Introduce Fibers in both
cases to allow streaming.

WEBrick::HTTPRequest gains a new body_reader method to prepare
itself as a source for IO.copy_stream.  This allows the
WEBrick::HTTPRequest object to be used as the
Net::HTTPGenericRequest#body_stream= arg for Net::HTTP.

For HTTP proxy response bodies, we also use a Fiber to
to make the HTTP request and read the response body.

* lib/webrick/httprequest.rb (body_reader): new method
  (readpartial): ditto
* lib/webrick/httpproxy.rb (perform_proxy_request): use Fiber
  to stream response body
  (do_GET, do_HEAD): adjust call
  (do_POST): adjust call and supply body_reader
* test/webrick/test_httprequest.rb (test_chunked): test
  for IO.copy_stream compatibility
* test/webrick/test_httpproxy.rb (test_big_bodies): new test

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62966 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
normal 2018-03-28 08:06:55 +00:00
parent 32e277acbf
commit 706c028909
4 changed files with 172 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
lib/webrick/httpproxy.rb
View file

@ -211,21 +211,15 @@ module WEBrick
end
def do_GET(req, res)
perform_proxy_request(req, res) do |http, path, header|
http.get(path, header)
end
perform_proxy_request(req, res, Net::HTTP::Get)
end
def do_HEAD(req, res)
perform_proxy_request(req, res) do |http, path, header|
http.head(path, header)
end
perform_proxy_request(req, res, Net::HTTP::Head)
end
def do_POST(req, res)
perform_proxy_request(req, res) do |http, path, header|
http.post(path, req.body || "", header)
end
perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader)
end
def do_OPTIONS(req, res)
@ -301,38 +295,56 @@ module WEBrick
return FakeProxyURI
end
def perform_proxy_request(req, res)
def perform_proxy_request(req, res, req_class, body_stream = nil)
uri = req.request_uri
path = uri.path.dup
path << "?" << uri.query if uri.query
header = setup_proxy_header(req, res)
upstream = setup_upstream_proxy_authentication(req, res, header)
response = nil
body_tmp = []
http = Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port)
http.start do
if @config[:ProxyTimeout]
################################## these issues are
http.open_timeout = 30 # secs # necessary (maybe because
http.read_timeout = 60 # secs # Ruby's bug, but why?)
##################################
req_fib = Fiber.new do
http.start do
if @config[:ProxyTimeout]
################################## these issues are
http.open_timeout = 30 # secs # necessary (maybe because
http.read_timeout = 60 # secs # Ruby's bug, but why?)
##################################
end
if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i
header['Transfer-Encoding'] = 'chunked'
end
http_req = req_class.new(path, header)
http_req.body_stream = body_stream if body_stream
http.request(http_req) do |response|
# Persistent connection requirements are mysterious for me.
# So I will close the connection in every response.
res['proxy-connection'] = "close"
res['connection'] = "close"
# stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse
res.status = response.code.to_i
res.chunked = response.chunked?
choose_header(response, res)
set_cookie(response, res)
set_via(res)
response.read_body do |buf|
body_tmp << buf
Fiber.yield # wait for res.body Proc#call
end
end # http.request
end
end
req_fib.resume # read HTTP response headers and first chunk of the body
res.body = ->(socket) do
while buf = body_tmp.shift
socket.write(buf)
buf.clear
req_fib.resume # continue response.read_body
end
response = yield(http, path, header)
end
# Persistent connection requirements are mysterious for me.
# So I will close the connection in every response.
res['proxy-connection'] = "close"
res['connection'] = "close"
# Convert Net::HTTP::HTTPResponse to WEBrick::HTTPResponse
res.status = response.code.to_i
choose_header(response, res)
set_cookie(response, res)
set_via(res)
res.body = response.body
end
# :stopdoc:
end
end

26
lib/webrick/httprequest.rb
View file

@ -257,6 +257,32 @@ module WEBrick
@body.empty? ? nil : @body
end
##
# Prepares the HTTPRequest object for use as the
# source for IO.copy_stream
def body_reader
@body_tmp = []
@body_rd = Fiber.new do
body do |buf|
@body_tmp << buf
Fiber.yield
end
end
@body_rd.resume # grab the first chunk and yield
self
end
# for IO.copy_stream. Note: we may return a larger string than +size+
# here; but IO.copy_stream does not care.
def readpartial(size, buf = ''.b) # :nodoc
res = @body_tmp.shift or raise EOFError, 'end of file reached'
buf.replace(res)
res.clear
@body_rd.resume # get more chunks
buf
end
##
# Request query as a Hash

94
test/webrick/test_httpproxy.rb
View file

@ -118,6 +118,100 @@ class TestWEBrickHTTPProxy < Test::Unit::TestCase
}
end
def test_big_bodies
require 'digest/md5'
rand_str = File.read(__FILE__)
rand_str.freeze
nr = 1024 ** 2 / rand_str.size # bigger works, too
exp = Digest::MD5.new
nr.times { exp.update(rand_str) }
exp = exp.hexdigest
TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log|
o_server.mount_proc('/') do |req, res|
case req.request_method
when 'GET'
res['content-type'] = 'application/octet-stream'
if req.path == '/length'
res['content-length'] = (nr * rand_str.size).to_s
else
res.chunked = true
end
res.body = ->(socket) { nr.times { socket.write(rand_str) } }
when 'POST'
dig = Digest::MD5.new
req.body { |buf| dig.update(buf); buf.clear }
res['content-type'] = 'text/plain'
res['content-length'] = '32'
res.body = dig.hexdigest
end
end
http = Net::HTTP.new(o_addr, o_port)
IO.pipe do |rd, wr|
headers = {
'Content-Type' => 'application/octet-stream',
'Transfer-Encoding' => 'chunked',
}
post = Net::HTTP::Post.new('/', headers)
th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
post.body_stream = rd
http.request(post) do |res|
assert_equal 'text/plain', res['content-type']
assert_equal 32, res.content_length
assert_equal exp, res.body
end
assert_nil th.value
end
TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log|
http = Net::HTTP.new(o_addr, o_port, p_addr, p_port)
http.request_get('/length') do |res|
assert_equal(nr * rand_str.size, res.content_length)
dig = Digest::MD5.new
res.read_body { |buf| dig.update(buf); buf.clear }
assert_equal exp, dig.hexdigest
end
http.request_get('/') do |res|
assert_predicate res, :chunked?
dig = Digest::MD5.new
res.read_body { |buf| dig.update(buf); buf.clear }
assert_equal exp, dig.hexdigest
end
IO.pipe do |rd, wr|
headers = {
'Content-Type' => 'application/octet-stream',
'Content-Length' => (nr * rand_str.size).to_s,
}
post = Net::HTTP::Post.new('/', headers)
th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
post.body_stream = rd
http.request(post) do |res|
assert_equal 'text/plain', res['content-type']
assert_equal 32, res.content_length
assert_equal exp, res.body
end
assert_nil th.value
end
IO.pipe do |rd, wr|
headers = {
'Content-Type' => 'application/octet-stream',
'Transfer-Encoding' => 'chunked',
}
post = Net::HTTP::Post.new('/', headers)
th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
post.body_stream = rd
http.request(post) do |res|
assert_equal 'text/plain', res['content-type']
assert_equal 32, res.content_length
assert_equal exp, res.body
end
assert_nil th.value
end
end
end
end
def make_certificate(key, cn)
subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}")
exts = [

10
test/webrick/test_httprequest.rb
View file

@ -237,6 +237,7 @@ GET /
def test_chunked
crlf = "\x0d\x0a"
expect = File.read(__FILE__).freeze
msg = <<-_end_of_message_
POST /path HTTP/1.1
Host: test.ruby-lang.org:8080
@ -253,7 +254,14 @@ GET /
msg << "0" << crlf
req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
req.parse(StringIO.new(msg))
assert_equal(File.read(__FILE__), req.body)
assert_equal(expect, req.body)
# chunked req.body_reader
req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
req.parse(StringIO.new(msg))
dst = StringIO.new
IO.copy_stream(req.body_reader, dst)
assert_equal(expect, dst.string)
end
def test_forwarded