diff --git a/ext/http11/http11.c b/ext/http11/http11.c index fea54687..93362511 100644 --- a/ext/http11/http11.c +++ b/ext/http11/http11.c @@ -24,7 +24,6 @@ static VALUE eHttpParserError; -#define id_http_body rb_intern("@http_body") #define HTTP_PREFIX "HTTP_" #define HTTP_PREFIX_LEN (sizeof(HTTP_PREFIX) - 1) @@ -57,6 +56,7 @@ DEF_MAX_LENGTH(HEADER, (1024 * (80 + 32))); struct common_field { const signed long len; const char *name; + int raw; VALUE value; }; @@ -66,7 +66,8 @@ struct common_field { * objects to be used with rb_hash_aset(). */ static struct common_field common_http_fields[] = { -# define f(N) { (sizeof(N) - 1), N, Qnil } +# define f(N) { (sizeof(N) - 1), N, 0, Qnil } +# define fr(N) { (sizeof(N) - 1), N, 1, Qnil } f("ACCEPT"), f("ACCEPT_CHARSET"), f("ACCEPT_ENCODING"), @@ -76,8 +77,8 @@ static struct common_field common_http_fields[] = { f("CACHE_CONTROL"), f("CONNECTION"), f("CONTENT_ENCODING"), - f("CONTENT_LENGTH"), - f("CONTENT_TYPE"), + fr("CONTENT_LENGTH"), + fr("CONTENT_TYPE"), f("COOKIE"), f("DATE"), f("EXPECT"), @@ -129,8 +130,12 @@ static void init_common_fields(void) memcpy(tmp, HTTP_PREFIX, HTTP_PREFIX_LEN); for(i = 0; i < ARRAY_SIZE(common_http_fields); cf++, i++) { - memcpy(tmp + HTTP_PREFIX_LEN, cf->name, cf->len + 1); - cf->value = rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len); + if(cf->raw) { + cf->value = rb_str_new(cf->name, cf->len); + } else { + memcpy(tmp + HTTP_PREFIX_LEN, cf->name, cf->len + 1); + cf->value = rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len); + } rb_global_variable(&cf->value); } @@ -255,10 +260,7 @@ void http_version(http_parser* hp, const char *at, size_t length) void header_done(http_parser* hp, const char *at, size_t length) { - VALUE req = hp->request; - - /* grab the initial body and stuff it into an ivar */ - rb_ivar_set(req, id_http_body, rb_str_new(at, length)); + hp->body = rb_str_new(at, length); } @@ -272,6 +274,7 @@ void HttpParser_free(void *data) { void HttpParser_mark(http_parser* hp) { if(hp->request) rb_gc_mark(hp->request); + if(hp->body) rb_gc_mark(hp->body); } VALUE HttpParser_alloc(VALUE klass) @@ -437,6 +440,19 @@ VALUE HttpParser_nread(VALUE self) return INT2FIX(http->nread); } +/** + * call-seq: + * parser.body -> nil or String + * + * If the request included a body, returns it. + */ +VALUE HttpParser_body(VALUE self) { + http_parser *http = NULL; + DATA_GET(self, http_parser, http); + + return http->body; +} + void Init_http11() { @@ -454,12 +470,13 @@ void Init_http11() VALUE cHttpParser = rb_define_class_under(mMongrel, "HttpParser", rb_cObject); rb_define_alloc_func(cHttpParser, HttpParser_alloc); - rb_define_method(cHttpParser, "initialize", HttpParser_init,0); - rb_define_method(cHttpParser, "reset", HttpParser_reset,0); - rb_define_method(cHttpParser, "finish", HttpParser_finish,0); - rb_define_method(cHttpParser, "execute", HttpParser_execute,3); - rb_define_method(cHttpParser, "error?", HttpParser_has_error,0); - rb_define_method(cHttpParser, "finished?", HttpParser_is_finished,0); - rb_define_method(cHttpParser, "nread", HttpParser_nread,0); + rb_define_method(cHttpParser, "initialize", HttpParser_init, 0); + rb_define_method(cHttpParser, "reset", HttpParser_reset, 0); + rb_define_method(cHttpParser, "finish", HttpParser_finish, 0); + rb_define_method(cHttpParser, "execute", HttpParser_execute, 3); + rb_define_method(cHttpParser, "error?", HttpParser_has_error, 0); + rb_define_method(cHttpParser, "finished?", HttpParser_is_finished, 0); + rb_define_method(cHttpParser, "nread", HttpParser_nread, 0); + rb_define_method(cHttpParser, "body", HttpParser_body, 0); init_common_fields(); } diff --git a/ext/http11/http11_parser.c b/ext/http11/http11_parser.c index 5fec04e4..a11a4ed8 100644 --- a/ext/http11/http11_parser.c +++ b/ext/http11/http11_parser.c @@ -62,6 +62,7 @@ int http_parser_init(http_parser *parser) { parser->field_len = 0; parser->field_start = 0; parser->request = Qnil; + parser->body = Qnil; return 1; } @@ -81,7 +82,7 @@ size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, assert(pe - p == len - off && "pointers aren't same distance"); -#line 85 "ext/http11/http11_parser.c" +#line 86 "ext/http11/http11_parser.c" { if ( p == pe ) goto _test_eof; @@ -112,7 +113,7 @@ st2: if ( ++p == pe ) goto _test_eof2; case 2: -#line 116 "ext/http11/http11_parser.c" +#line 117 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr2; case 36: goto st38; @@ -137,7 +138,7 @@ st3: if ( ++p == pe ) goto _test_eof3; case 3: -#line 141 "ext/http11/http11_parser.c" +#line 142 "ext/http11/http11_parser.c" switch( (*p) ) { case 42: goto tr4; case 43: goto tr5; @@ -161,7 +162,7 @@ st4: if ( ++p == pe ) goto _test_eof4; case 4: -#line 165 "ext/http11/http11_parser.c" +#line 166 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr8; case 35: goto tr9; @@ -223,7 +224,7 @@ st5: if ( ++p == pe ) goto _test_eof5; case 5: -#line 227 "ext/http11/http11_parser.c" +#line 228 "ext/http11/http11_parser.c" if ( (*p) == 72 ) goto tr10; goto st0; @@ -235,7 +236,7 @@ st6: if ( ++p == pe ) goto _test_eof6; case 6: -#line 239 "ext/http11/http11_parser.c" +#line 240 "ext/http11/http11_parser.c" if ( (*p) == 84 ) goto st7; goto st0; @@ -316,7 +317,7 @@ st14: if ( ++p == pe ) goto _test_eof14; case 14: -#line 320 "ext/http11/http11_parser.c" +#line 321 "ext/http11/http11_parser.c" if ( (*p) == 10 ) goto st15; goto st0; @@ -367,7 +368,7 @@ st57: if ( ++p == pe ) goto _test_eof57; case 57: -#line 371 "ext/http11/http11_parser.c" +#line 372 "ext/http11/http11_parser.c" goto st0; tr21: #line 37 "ext/http11/http11_parser.rl" @@ -383,7 +384,7 @@ st17: if ( ++p == pe ) goto _test_eof17; case 17: -#line 387 "ext/http11/http11_parser.c" +#line 388 "ext/http11/http11_parser.c" switch( (*p) ) { case 33: goto tr23; case 58: goto tr24; @@ -422,7 +423,7 @@ st18: if ( ++p == pe ) goto _test_eof18; case 18: -#line 426 "ext/http11/http11_parser.c" +#line 427 "ext/http11/http11_parser.c" switch( (*p) ) { case 13: goto tr26; case 32: goto tr27; @@ -436,7 +437,7 @@ st19: if ( ++p == pe ) goto _test_eof19; case 19: -#line 440 "ext/http11/http11_parser.c" +#line 441 "ext/http11/http11_parser.c" if ( (*p) == 13 ) goto tr29; goto st19; @@ -482,7 +483,7 @@ st20: if ( ++p == pe ) goto _test_eof20; case 20: -#line 486 "ext/http11/http11_parser.c" +#line 487 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr31; case 37: goto tr32; @@ -504,7 +505,7 @@ st21: if ( ++p == pe ) goto _test_eof21; case 21: -#line 508 "ext/http11/http11_parser.c" +#line 509 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr34; case 37: goto st22; @@ -526,7 +527,7 @@ st22: if ( ++p == pe ) goto _test_eof22; case 22: -#line 530 "ext/http11/http11_parser.c" +#line 531 "ext/http11/http11_parser.c" if ( (*p) < 65 ) { if ( 48 <= (*p) && (*p) <= 57 ) goto st23; @@ -557,7 +558,7 @@ st24: if ( ++p == pe ) goto _test_eof24; case 24: -#line 561 "ext/http11/http11_parser.c" +#line 562 "ext/http11/http11_parser.c" switch( (*p) ) { case 43: goto st24; case 58: goto st25; @@ -582,7 +583,7 @@ st25: if ( ++p == pe ) goto _test_eof25; case 25: -#line 586 "ext/http11/http11_parser.c" +#line 587 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr8; case 34: goto st0; @@ -629,7 +630,7 @@ st28: if ( ++p == pe ) goto _test_eof28; case 28: -#line 633 "ext/http11/http11_parser.c" +#line 634 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr42; case 34: goto st0; @@ -680,7 +681,7 @@ st31: if ( ++p == pe ) goto _test_eof31; case 31: -#line 684 "ext/http11/http11_parser.c" +#line 685 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr8; case 34: goto st0; @@ -730,7 +731,7 @@ st34: if ( ++p == pe ) goto _test_eof34; case 34: -#line 734 "ext/http11/http11_parser.c" +#line 735 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr53; case 34: goto st0; @@ -751,7 +752,7 @@ st35: if ( ++p == pe ) goto _test_eof35; case 35: -#line 755 "ext/http11/http11_parser.c" +#line 756 "ext/http11/http11_parser.c" switch( (*p) ) { case 32: goto tr57; case 34: goto st0; @@ -772,7 +773,7 @@ st36: if ( ++p == pe ) goto _test_eof36; case 36: -#line 776 "ext/http11/http11_parser.c" +#line 777 "ext/http11/http11_parser.c" if ( (*p) < 65 ) { if ( 48 <= (*p) && (*p) <= 57 ) goto st37; @@ -1188,7 +1189,7 @@ case 56: _out: {} } -#line 113 "ext/http11/http11_parser.rl" +#line 114 "ext/http11/http11_parser.rl" if (!http_parser_has_error(parser)) parser->cs = cs; diff --git a/ext/http11/http11_parser.h b/ext/http11/http11_parser.h index c9a055e0..13e4a241 100644 --- a/ext/http11/http11_parser.h +++ b/ext/http11/http11_parser.h @@ -36,6 +36,7 @@ typedef struct http_parser { size_t query_start; VALUE request; + VALUE body; field_cb http_field; element_cb request_method; diff --git a/ext/http11/http11_parser.rl b/ext/http11/http11_parser.rl index 99492c4e..459cc709 100644 --- a/ext/http11/http11_parser.rl +++ b/ext/http11/http11_parser.rl @@ -91,6 +91,7 @@ int http_parser_init(http_parser *parser) { parser->field_len = 0; parser->field_start = 0; parser->request = Qnil; + parser->body = Qnil; return 1; } diff --git a/lib/mongrel.rb b/lib/mongrel.rb index dc4fbff3..fa1fcccd 100644 --- a/lib/mongrel.rb +++ b/lib/mongrel.rb @@ -22,7 +22,6 @@ require 'mongrel/gems' require 'thread' # Ruby Mongrel -require 'mongrel/cgi' require 'mongrel/handlers' require 'mongrel/command' require 'mongrel/tcphack' @@ -32,358 +31,19 @@ require 'mongrel/const' require 'mongrel/http_request' require 'mongrel/header_out' require 'mongrel/http_response' +require 'mongrel/server' -# Mongrel module containing all of the classes (include C extensions) for running -# a Mongrel web server. It contains a minimalist HTTP server with just enough -# functionality to service web application requests fast as possible. +# Mongrel module containing all of the classes (include C extensions) +# for running a Mongrel web server. It contains a minimalist HTTP server +# with just enough functionality to service web application requests +# fast as possible. module Mongrel - # Used to stop the HttpServer via Thread.raise. - class StopServer < Exception; end - # Thrown at a thread when it is timed out. - class TimeoutError < Exception; end + class TimeoutError < RuntimeError; end - # A Hash with one extra parameter for the HTTP body, used internally. - class HttpParams < Hash - attr_accessor :http_body - end - - - # This is the main driver of Mongrel, while the Mongrel::HttpParser and Mongrel::URIClassifier - # make up the majority of how the server functions. It's a very simple class that just - # has a thread accepting connections and a simple HttpServer.process_client function - # to do the heavy lifting with the IO and Ruby. - # - # You use it by doing the following: - # - # server = HttpServer.new("0.0.0.0", 3000) - # server.register("/stuff", MyNiftyHandler.new) - # server.run.join - # - # The last line can be just server.run if you don't want to join the thread used. - # If you don't though Ruby will mysteriously just exit on you. - # - # Ruby's thread implementation is "interesting" to say the least. Experiments with - # *many* different types of IO processing simply cannot make a dent in it. Future - # releases of Mongrel will find other creative ways to make threads faster, but don't - # hold your breath until Ruby 1.9 is actually finally useful. - class HttpServer - - include Mongrel::Const - - attr_reader :acceptor - attr_reader :workers - attr_reader :classifier - attr_reader :host - attr_reader :port - attr_reader :throttle - attr_reader :timeout - attr_reader :num_processors - - # Creates a working server on host:port (strange things happen if port isn't a Number). - # Use HttpServer::run to start the server and HttpServer.acceptor.join to - # join the thread that's processing incoming requests on the socket. - # - # The num_processors optional argument is the maximum number of concurrent - # processors to accept, anything over this is closed immediately to maintain - # server processing performance. This may seem mean but it is the most efficient - # way to deal with overload. Other schemes involve still parsing the client's request - # which defeats the point of an overload handling system. - # - # The throttle parameter is a sleep timeout (in hundredths of a second) that is placed between - # socket.accept calls in order to give the server a cheap throttle time. It defaults to 0 and - # actually if it is 0 then the sleep is not done at all. - def initialize(host, port, num_processors=950, throttle=0, timeout=60) - @socket = TCPServer.new(host, port) - - @classifier = URIClassifier.new - @host = host - @port = port - @workers = ThreadGroup.new - @throttle = throttle / 100.0 - @num_processors = num_processors - @timeout = timeout - end - - # Does the majority of the IO processing. It has been written in Ruby using - # about 7 different IO processing strategies and no matter how it's done - # the performance just does not improve. It is currently carefully constructed - # to make sure that it gets the best possible performance, but anyone who - # thinks they can make it faster is more than welcome to take a crack at it. - def process_client(client) - begin - parser = HttpParser.new - params = HttpParams.new - request = nil - data = client.readpartial(CHUNK_SIZE) - nparsed = 0 - - # Assumption: nparsed will always be less since data will get filled with more - # after each parsing. If it doesn't get more then there was a problem - # with the read operation on the client socket. Effect is to stop processing when the - # socket can't fill the buffer for further parsing. - while nparsed < data.length - nparsed = parser.execute(params, data, nparsed) - - if parser.finished? - - if host = params[HTTP_HOST] - if colon = host.index(":") - params[SERVER_NAME] = host[0, colon] - params[SERVER_PORT] = host[colon+1, host.size] - else - params[SERVER_NAME] = host - params[SERVER_PORT] = PORT_80 - end - end - - if len = params[HTTP_CONTENT_LENGTH] - params[CONTENT_LENGTH] = len - end - - if type = params[HTTP_CONTENT_TYPE] - params[RAW_CONTENT_TYPE] = type - end - - params[SERVER_PROTOCOL] = HTTP_11 - params[SERVER_SOFTWARE] = MONGREL_VERSION - params[GATEWAY_INTERFACE] = CGI_VER - - unless params[REQUEST_PATH] - # it might be a dumbass full host request header - uri = URI.parse(params[REQUEST_URI]) - params[REQUEST_PATH] = uri.path - - raise "No REQUEST PATH" unless params[REQUEST_PATH] - end - - script_name, path_info, handlers = - @classifier.resolve(params[REQUEST_PATH]) - - if handlers - params[PATH_INFO] = path_info - params[SCRIPT_NAME] = script_name - - # From http://www.ietf.org/rfc/rfc3875 : - # "Script authors should be aware that the REMOTE_ADDR and REMOTE_HOST - # meta-variables (see sections 4.1.8 and 4.1.9) may not identify the - # ultimate source of the request. They identify the client for the - # immediate request to the server; that client may be a proxy, gateway, - # or other intermediary acting on behalf of the actual source client." - params[REMOTE_ADDR] = client.peeraddr.last - - # select handlers that want more detailed request notification - notifiers = handlers.select { |h| h.request_notify } - request = HttpRequest.new(params, client, notifiers) - - # in the case of large file uploads the user could close the socket, so skip those requests - break if request.body == nil # nil signals from HttpRequest::initialize that the request was aborted - - # request is good so far, continue processing the response - response = HttpResponse.new(client) - - # Process each handler in registered order until we run out or one finalizes the response. - handlers.each do |handler| - handler.process(request, response) - break if response.done or client.closed? - end - - # And finally, if nobody closed the response off, we finalize it. - unless response.done or client.closed? - response.finished - end - else - # Didn't find it, return a stock 404 response. - client.write(ERROR_404_RESPONSE) - end - - break #done - else - # Parser is not done, queue up more data to read and continue parsing - chunk = client.readpartial(CHUNK_SIZE) - break if !chunk or chunk.length == 0 # read failed, stop processing - - data << chunk - if data.length >= MAX_HEADER - raise HttpParserError.new("HEADER is longer than allowed, aborting client early.") - end - end - end - rescue EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF - client.close rescue nil - rescue HttpParserError => e - STDERR.puts "#{Time.now}: HTTP parse error, malformed request (#{params[HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #{e.inspect}" - STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n" - rescue Errno::EMFILE - reap_dead_workers('too many files') - rescue Object => e - STDERR.puts "#{Time.now}: Read error: #{e.inspect}" - STDERR.puts e.backtrace.join("\n") - ensure - begin - client.close - rescue IOError - # Already closed - rescue Object => e - STDERR.puts "#{Time.now}: Client error: #{e.inspect}" - STDERR.puts e.backtrace.join("\n") - end - request.body.close! if request and request.body.class == Tempfile - end - end - - # Used internally to kill off any worker threads that have taken too long - # to complete processing. Only called if there are too many processors - # currently servicing. It returns the count of workers still active - # after the reap is done. It only runs if there are workers to reap. - def reap_dead_workers(reason='unknown') - if @workers.list.length > 0 - STDERR.puts "#{Time.now}: Reaping #{@workers.list.length} threads for slow workers because of '#{reason}'" - error_msg = "Mongrel timed out this thread: #{reason}" - mark = Time.now - @workers.list.each do |worker| - worker[:started_on] = Time.now if not worker[:started_on] - - if mark - worker[:started_on] > @timeout + @throttle - STDERR.puts "Thread #{worker.inspect} is too old, killing." - worker.raise(TimeoutError.new(error_msg)) - end - end - end - - return @workers.list.length - end - - # Performs a wait on all the currently running threads and kills any that take - # too long. It waits by @timeout seconds, which can be set in .initialize or - # via mongrel_rails. The @throttle setting does extend this waiting period by - # that much longer. - def graceful_shutdown - while reap_dead_workers("shutdown") > 0 - STDERR.puts "Waiting for #{@workers.list.length} requests to finish, could take #{@timeout + @throttle} seconds." - sleep @timeout / 10 - end - end - - def configure_socket_options - @tcp_defer_accept_opts = nil - @tcp_cork_opts = nil - - case RUBY_PLATFORM - when /linux/ - # 9 is currently TCP_DEFER_ACCEPT - @tcp_defer_accept_opts = [Socket::SOL_TCP, 9, 1] - @tcp_cork_opts = [Socket::SOL_TCP, 3, 1] - - when /freebsd(([1-4]\..{1,2})|5\.[0-4])/ - # Do nothing, just closing a bug when freebsd <= 5.4 - when /freebsd/ - # Use the HTTP accept filter if available. - # The struct made by pack() is defined in /usr/include/sys/socket.h as accept_filter_arg - unless `/sbin/sysctl -nq net.inet.accf.http`.empty? - @tcp_defer_accept_opts = [Socket::SOL_SOCKET, Socket::SO_ACCEPTFILTER, ['httpready', nil].pack('a16a240')] - end - end - end - - # Runs the thing. It returns the thread used so you can "join" it. You can also - # access the HttpServer::acceptor attribute to get the thread later. - def run - BasicSocket.do_not_reverse_lookup=true - - configure_socket_options - - if @tcp_defer_accept_opts - @socket.setsockopt(*@tcp_defer_accept_opts) - end - - tcp_cork_opts = @tcp_cork_opts - - @acceptor = Thread.new do - begin - while true - begin - client = @socket.accept - - client.setsockopt(*tcp_cork_opts) if tcp_cork_opts - - worker_list = @workers.list - - if worker_list.length >= @num_processors - STDERR.puts "Server overloaded with #{worker_list.length} processors (#@num_processors max). Dropping connection." - client.close rescue nil - reap_dead_workers("max processors") - else - thread = Thread.new(client) { |c| process_client(c) } - thread[:started_on] = Time.now - @workers.add(thread) - - sleep @throttle if @throttle > 0 - end - rescue StopServer - break - rescue Errno::EMFILE - reap_dead_workers("too many open files") - sleep 0.5 - rescue Errno::ECONNABORTED - # client closed the socket even before accept - client.close rescue nil - rescue Object => e - STDERR.puts "#{Time.now}: Unhandled listen loop exception #{e.inspect}." - STDERR.puts e.backtrace.join("\n") - end - end - graceful_shutdown - ensure - @socket.close - # STDERR.puts "#{Time.now}: Closed socket." - end - end - - return @acceptor - end - - # Simply registers a handler with the internal URIClassifier. When the URI is - # found in the prefix of a request then your handler's HttpHandler::process method - # is called. See Mongrel::URIClassifier#register for more information. - # - # If you set in_front=true then the passed in handler will be put in the front of the list - # for that particular URI. Otherwise it's placed at the end of the list. - def register(uri, handler, in_front=false) - begin - @classifier.register(uri, [handler]) - rescue URIClassifier::RegistrationError => e - handlers = @classifier.resolve(uri)[2] - if handlers - # Already registered - method_name = in_front ? 'unshift' : 'push' - handlers.send(method_name, handler) - else - raise - end - end - handler.listener = self - end - - # Removes any handlers registered at the given URI. See Mongrel::URIClassifier#unregister - # for more information. Remember this removes them *all* so the entire - # processing chain goes away. - def unregister(uri) - @classifier.unregister(uri) - end - - # Stops the acceptor thread and then causes the worker threads to finish - # off the request queue before finally exiting. - def stop(synchronous=false) - @acceptor.raise(StopServer.new) - - if synchronous - sleep(0.5) while @acceptor.alive? - end - end - - end + class BodyReadError < RuntimeError; end end -Mongrel::Gems.require 'mongrel_experimental', ">=#{Mongrel::Const::MONGREL_VERSION}" +Mongrel::Gems.require "mongrel_experimental", + ">=#{Mongrel::Const::MONGREL_VERSION}" diff --git a/lib/mongrel/camping.rb b/lib/mongrel/camping.rb deleted file mode 100644 index 31bd196f..00000000 --- a/lib/mongrel/camping.rb +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2005 Zed A. Shaw -# You can redistribute it and/or modify it under the same terms as Ruby. -# -# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html -# for more information. - -require 'mongrel' - - -module Mongrel - # Support for the Camping micro framework at http://camping.rubyforge.org - # This implements the unusually long Postamble that Camping usually - # needs and shrinks it down to just a single line or two. - # - # Your Postamble would now be: - # - # Mongrel::Camping::start("0.0.0.0",3001,"/tepee",Tepee).join - # - # If you wish to get fancier than this then you can use the - # Camping::CampingHandler directly instead and do your own - # wiring: - # - # h = Mongrel::HttpServer.new(server, port) - # h.register(uri, CampingHandler.new(Tepee)) - # h.register("/favicon.ico", Mongrel::Error404Handler.new("")) - # - # I add the /favicon.ico since camping apps typically don't - # have them and it's just annoying anyway. - module Camping - - # This is a specialized handler for Camping applications - # that has them process the request and then translates - # the results into something the Mongrel::HttpResponse - # needs. - class CampingHandler < Mongrel::HttpHandler - attr_reader :files - attr_reader :guard - @@file_only_methods = ["GET","HEAD"] - - def initialize(klass) - @files = Mongrel::DirHandler.new(nil, false) - @guard = Mutex.new - @klass = klass - end - - def process(request, response) - if response.socket.closed? - return - end - - controller = nil - @guard.synchronize { - controller = @klass.run(request.body, request.params) - } - - sendfile, clength = nil - response.status = controller.status - controller.headers.each do |k, v| - if k =~ /^X-SENDFILE$/i - sendfile = v - elsif k =~ /^CONTENT-LENGTH$/i - clength = v.to_i - else - [*v].each do |vi| - response.header[k] = vi - end - end - end - - if sendfile - request.params[Mongrel::Const::PATH_INFO] = sendfile - @files.process(request, response) - elsif controller.body.respond_to? :read - response.send_status(clength) - response.send_header - while chunk = controller.body.read(16384) - response.write(chunk) - end - if controller.body.respond_to? :close - controller.body.close - end - else - body = controller.body.to_s - response.send_status(body.length) - response.send_header - response.write(body) - end - end - end - - # This is a convenience method that wires up a CampingHandler - # for your application on a given port and uri. It's pretty - # much all you need for a camping application to work right. - # - # It returns the Mongrel::HttpServer which you should either - # join or somehow manage. The thread is running when - # returned. - - def Camping.start(server, port, uri, klass) - h = Mongrel::HttpServer.new(server, port) - h.register(uri, CampingHandler.new(klass)) - h.register("/favicon.ico", Mongrel::Error404Handler.new("")) - h.run - return h - end - end -end diff --git a/lib/mongrel/cgi.rb b/lib/mongrel/cgi.rb deleted file mode 100644 index 39576111..00000000 --- a/lib/mongrel/cgi.rb +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (c) 2005 Zed A. Shaw -# You can redistribute it and/or modify it under the same terms as Ruby. -# -# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html -# for more information. - -require 'cgi' - -module Mongrel - # The beginning of a complete wrapper around Mongrel's internal HTTP processing - # system but maintaining the original Ruby CGI module. Use this only as a crutch - # to get existing CGI based systems working. It should handle everything, but please - # notify me if you see special warnings. This work is still very alpha so I need - # testers to help work out the various corner cases. - # - # The CGIWrapper.handler attribute is normally not set and is available for - # frameworks that need to get back to the handler. Rails uses this to give - # people access to the RailsHandler#files (DirHandler really) so they can - # look-up paths and do other things with the files managed there. - # - # In Rails you can get the real file for a request with: - # - # path = @request.cgi.handler.files.can_serve(@request['PATH_INFO']) - # - # Which is ugly but does the job. Feel free to write a Rails helper for that. - # Refer to DirHandler#can_serve for more information on this. - class CGIWrapper < ::CGI - public :env_table - attr_reader :head - attr_accessor :handler - # Set this to false if you want calls to CGIWrapper.out to not actually send - # the response until you force it. - attr_accessor :default_really_final - - # these are stripped out of any keys passed to CGIWrapper.header function - REMOVED_KEYS = [ "nph","status","server","connection","type", - "charset","length","language","expires"] - - # Takes an HttpRequest and HttpResponse object, plus any additional arguments - # normally passed to CGI. These are used internally to create a wrapper around - # the real CGI while maintaining Mongrel's view of the world. - def initialize(request, response, *args) - @request = request - @response = response - @args = *args - @input = request.body - @head = {} - @out_called = false - @default_really_final=true - super(*args) - end - - # The header is typically called to send back the header. In our case we - # collect it into a hash for later usage. - # - # nph -- Mostly ignored. It'll output the date. - # connection -- Completely ignored. Why is CGI doing this? - # length -- Ignored since Mongrel figures this out from what you write to output. - # - def header(options = "text/html") - # if they pass in a string then just write the Content-Type - if options.class == String - @head['Content-Type'] = options unless @head['Content-Type'] - else - # convert the given options into what Mongrel wants - @head['Content-Type'] = options['type'] || "text/html" - @head['Content-Type'] += "; charset=" + options['charset'] if options.has_key? "charset" if options['charset'] - - # setup date only if they use nph - @head['Date'] = CGI::rfc1123_date(Time.now) if options['nph'] - - # setup the server to use the default or what they set - @head['Server'] = options['server'] || env_table['SERVER_SOFTWARE'] - - # remaining possible options they can give - @head['Status'] = options['status'] if options['status'] - @head['Content-Language'] = options['language'] if options['language'] - @head['Expires'] = options['expires'] if options['expires'] - - # drop the keys we don't want anymore - REMOVED_KEYS.each {|k| options.delete(k) } - - # finally just convert the rest raw (which puts 'cookie' directly) - # 'cookie' is translated later as we write the header out - options.each{|k,v| @head[k] = v} - end - - # doing this fakes out the cgi library to think the headers are empty - # we then do the real headers in the out function call later - "" - end - - # Takes any 'cookie' setting and sends it over the Mongrel header, - # then removes the setting from the options. If cookie is an - # Array or Hash then it sends those on with .to_s, otherwise - # it just calls .to_s on it and hopefully your "cookie" can - # write itself correctly. - def send_cookies(to) - # convert the cookies based on the myriad of possible ways to set a cookie - if @head['cookie'] - cookie = @head['cookie'] - case cookie - when Array - cookie.each {|c| to['Set-Cookie'] = c.to_s } - when Hash - cookie.each_value {|c| to['Set-Cookie'] = c.to_s} - else - to['Set-Cookie'] = head['cookie'].to_s - end - - @head.delete('cookie') - end - - # @output_cookies seems to never be used, but we'll process it just in case - @output_cookies.each {|c| to['Set-Cookie'] = c.to_s } if @output_cookies - end - - # The dumb thing is people can call header or this or both and in any order. - # So, we just reuse header and then finalize the HttpResponse the right way. - # Status is taken from the various options and converted to what Mongrel needs - # via the CGIWrapper.status function. - # - # We also prevent Rails from actually doing the final send by adding a - # second parameter "really_final". Only Mongrel calls this after Rails - # is done. Since this will break other frameworks, it defaults to - # a different setting for rails (false) and (true) for others. - def out(options = "text/html", really_final=@default_really_final) - if @out_called || !really_final - # don't do it more than once or if it's not the really final call - return - end - - header(options) - - @response.start status do |head, body| - send_cookies(head) - - @head.each {|k,v| head[k] = v} - body.write(yield || "") - end - - @out_called = true - end - - # Computes the status once, but lazily so that people who call header twice - # don't get penalized. Because CGI insists on including the options status - # message in the status we have to do a bit of parsing. - def status - if not @status - stat = @head["Status"] - stat = stat.split(' ')[0] if stat - - @status = stat || "200" - end - - @status - end - - # Used to wrap the normal args variable used inside CGI. - def args - @args - end - - # Used to wrap the normal env_table variable used inside CGI. - def env_table - @request.params - end - - # Used to wrap the normal stdinput variable used inside CGI. - def stdinput - @input - end - - # The stdoutput should be completely bypassed but we'll drop a warning just in case - def stdoutput - STDERR.puts "WARNING: Your program is doing something not expected. Please tell Zed that stdoutput was used and what software you are running. Thanks." - @response.body - end - - end -end diff --git a/lib/mongrel/const.rb b/lib/mongrel/const.rb index 6775238a..b6ba2ceb 100644 --- a/lib/mongrel/const.rb +++ b/lib/mongrel/const.rb @@ -89,7 +89,9 @@ module Mongrel # A frozen format for this is about 15% faster STATUS_FORMAT = "HTTP/1.1 %d %s\r\nConnection: close\r\n" + CONTENT_TYPE = "Content-Type" + LAST_MODIFIED = "Last-Modified" ETAG = "ETag" SLASH = "/" @@ -98,7 +100,6 @@ module Mongrel HEAD="HEAD" # ETag is based on the apache standard of hex mtime-size-inode (inode is 0 on win32) ETAG_FORMAT="\"%x-%x-%x\"" - HEADER_FORMAT="%s: %s\r\n" LINE_END="\r\n" REMOTE_ADDR="REMOTE_ADDR" HTTP_X_FORWARDED_FOR="HTTP_X_FORWARDED_FOR" @@ -119,9 +120,7 @@ module Mongrel GATEWAY_INTERFACE = "GATEWAY_INTERFACE" CGI_VER = "CGI/1.2" - HTTP_CONTENT_LENGTH = "HTTP_CONTENT_LENGTH" + STOP_COMMAND = "!" - HTTP_CONTENT_TYPE = "HTTP_CONTENT_TYPE" - RAW_CONTENT_TYPE = "CONTENT_TYPE" end end diff --git a/lib/mongrel/header_out.rb b/lib/mongrel/header_out.rb index b34e95e9..065addb0 100644 --- a/lib/mongrel/header_out.rb +++ b/lib/mongrel/header_out.rb @@ -1,28 +1,40 @@ module Mongrel - # This class implements a simple way of constructing the HTTP headers dynamically - # via a Hash syntax. Think of it as a write-only Hash. Refer to HttpResponse for - # information on how this is used. + # This class implements a simple way of constructing the HTTP headers + # dynamically via a Hash syntax. Think of it as a write-only Hash. + # Refer to HttpResponse for information on how this is used. # - # One consequence of this write-only nature is that you can write multiple headers - # by just doing them twice (which is sometimes needed in HTTP), but that the normal - # semantics for Hash (where doing an insert replaces) is not there. + # One consequence of this write-only nature is that you can write multiple + # headers by just doing them twice (which is sometimes needed in HTTP), + # but that the normal semantics for Hash (where doing an insert replaces) + # is not there. class HeaderOut attr_reader :out attr_accessor :allowed_duplicates def initialize(out) @sent = {} - @allowed_duplicates = {"Set-Cookie" => true, "Set-Cookie2" => true, - "Warning" => true, "WWW-Authenticate" => true} + + @allowed_duplicates = { + "Set-Cookie" => true, + "Set-Cookie2" => true, + "Warning" => true, + "WWW-Authenticate" => true + } + @out = out end # Simply writes "#{key}: #{value}" to an output buffer. - def[]=(key,value) - if not @sent.has_key?(key) or @allowed_duplicates.has_key?(key) + def []=(key,value) + if !@sent.has_key?(key) or @allowed_duplicates.has_key?(key) @sent[key] = true - @out.write(Const::HEADER_FORMAT % [key, value]) + o = @out + + o.write key + o.write ": " + o.write value + o.write "\r\n" end end end -end \ No newline at end of file +end diff --git a/lib/mongrel/http_request.rb b/lib/mongrel/http_request.rb index c8d4ce40..5d66268c 100644 --- a/lib/mongrel/http_request.rb +++ b/lib/mongrel/http_request.rb @@ -4,17 +4,14 @@ module Mongrel # When a handler is found for a registered URI then this class is constructed # and passed to your HttpHandler::process method. You should assume that # *one* handler processes all requests. Included in the HttpRequest is a - # HttpRequest.params Hash that matches common CGI params, and a HttpRequest.body - # which is a string containing the request body (raw for now). + # HttpRequest#params Hash that matches common CGI params, and a + # HttpRequest#body which is a string containing the request body + # (raw for now). # - # The HttpRequest.initialize method will convert any request that is larger than - # Const::MAX_BODY into a Tempfile and use that as the body. Otherwise it uses - # a StringIO object. To be safe, you should assume it works like a file. - # - # The HttpHandler.request_notify system is implemented by having HttpRequest call - # HttpHandler.request_begins, HttpHandler.request_progress, HttpHandler.process during - # the IO processing. This adds a small amount of overhead but lets you implement - # finer controlled handlers and filters. + # The HttpRequest#initialize method will convert any request that is larger + # than Const::MAX_BODY into a Tempfile and use that as the body. + # Otherwise it uses a StringIO object. To be safe, you should assume it + # works like a file. # class HttpRequest attr_reader :body, :params @@ -22,24 +19,20 @@ module Mongrel # You don't really call this. It's made for you. # Main thing it does is hook up the params, and store any remaining # body data into the HttpRequest.body attribute. - def initialize(params, socket, dispatchers) + def initialize(params, socket, body) @params = params @socket = socket - @dispatchers = dispatchers content_length = @params[Const::CONTENT_LENGTH].to_i - remain = content_length - @params.http_body.length - - # tell all dispatchers the request has begun - @dispatchers.each do |dispatcher| - dispatcher.request_begins(@params) - end unless @dispatchers.nil? || @dispatchers.empty? - # Some clients (like FF1.0) report 0 for body and then send a body. This will probably truncate them but at least the request goes through usually. + remain = content_length - body.size + + # Some clients (like FF1.0) report 0 for body and then send a body. + # This will probably truncate them but at least the request goes + # through usually. + # if remain <= 0 # we've got everything, pack it up - @body = StringIO.new - @body.write @params.http_body - update_request_progress(0, content_length) + @body = StringIO.new body elsif remain > 0 # must read more data to complete body if remain > Const::MAX_BODY @@ -51,55 +44,50 @@ module Mongrel @body = StringIO.new end - @body.write @params.http_body - read_body(remain, content_length) + @body.write body + + read_body remain, content_length end @body.rewind if @body end - # updates all dispatchers about our progress - def update_request_progress(clen, total) - return if @dispatchers.nil? || @dispatchers.empty? - @dispatchers.each do |dispatcher| - dispatcher.request_progress(@params, clen, total) - end - end - private :update_request_progress - # Does the heavy lifting of properly reading the larger body requests in # small chunks. It expects @body to be an IO object, @socket to be valid, - # and will set @body = nil if the request fails. It also expects any initial - # part of the body that has been read to be in the @body already. + # and will set @body = nil if the request fails. It also expects any + # initial part of the body that has been read to be in the @body already. def read_body(remain, total) begin # write the odd sized chunk first - @params.http_body = read_socket(remain % Const::CHUNK_SIZE) + chunk = read_socket(remain % Const::CHUNK_SIZE) - remain -= @body.write(@params.http_body) - - update_request_progress(remain, total) + remain -= @body.write(chunk) # then stream out nothing but perfectly sized chunks until remain <= 0 or @socket.closed? - # ASSUME: we are writing to a disk and these writes always write the requested amount - @params.http_body = read_socket(Const::CHUNK_SIZE) - remain -= @body.write(@params.http_body) - - update_request_progress(remain, total) + # ASSUME: we are writing to a disk and these writes always + # write the requested amount + chunk = read_socket(Const::CHUNK_SIZE) + remain -= @body.write(chunk) end - rescue Object => e - STDERR.puts "#{Time.now}: Error reading HTTP body: #{e.inspect}" - STDERR.puts e.backtrace.join("\n") - # any errors means we should delete the file, including if the file is dumped + rescue RuntimeError + # any errors means we should delete the file, including if the + # file is dumped @socket.close rescue nil - @body.close! if @body.class == Tempfile - @body = nil # signals that there was a problem + close_body + + raise BodyReadError end end + + def close_body + @body.close! if @body.kind_of? IO + end def read_socket(len) - if !@socket.closed? + if @socket.closed? + raise "Socket already closed when reading." + else data = @socket.read(len) if !data raise "Socket read return nil" @@ -108,8 +96,6 @@ module Mongrel else data end - else - raise "Socket already closed when reading." end end @@ -152,4 +138,4 @@ module Mongrel return params end end -end \ No newline at end of file +end diff --git a/lib/mongrel/http_response.rb b/lib/mongrel/http_response.rb index 30767127..d8c7be3f 100644 --- a/lib/mongrel/http_response.rb +++ b/lib/mongrel/http_response.rb @@ -85,7 +85,7 @@ module Mongrel end def send_status(content_length=@body.length) - if not @status_sent + unless @status_sent @header['Content-Length'] = content_length if content_length and @status != 304 write(Const::STATUS_FORMAT % [@status, @reason || HTTP_STATUS_CODES[@status]]) @status_sent = true @@ -93,7 +93,7 @@ module Mongrel end def send_header - if not @header_sent + unless @header_sent @header.out.rewind write(@header.out.read + Const::LINE_END) @header_sent = true @@ -101,7 +101,7 @@ module Mongrel end def send_body - if not @body_sent + unless @body_sent @body.rewind write(@body.read) @body_sent = true @@ -163,4 +163,4 @@ module Mongrel end end -end \ No newline at end of file +end diff --git a/lib/mongrel/rails.rb b/lib/mongrel/rails.rb deleted file mode 100644 index 54bb830b..00000000 --- a/lib/mongrel/rails.rb +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (c) 2005 Zed A. Shaw -# You can redistribute it and/or modify it under the same terms as Ruby. -# -# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html -# for more information. - -require 'mongrel' -require 'cgi' - - -module Mongrel - module Rails - # Implements a handler that can run Rails and serve files out of the - # Rails application's public directory. This lets you run your Rails - # application with Mongrel during development and testing, then use it - # also in production behind a server that's better at serving the - # static files. - # - # The RailsHandler takes a mime_map parameter which is a simple suffix=mimetype - # mapping that it should add to the list of valid mime types. - # - # It also supports page caching directly and will try to resolve a request - # in the following order: - # - # * If the requested exact PATH_INFO exists as a file then serve it. - # * If it exists at PATH_INFO+".html" exists then serve that. - # * Finally, construct a Mongrel::CGIWrapper and run Dispatcher.dispatch to have Rails go. - # - # This means that if you are using page caching it will actually work with Mongrel - # and you should see a decent speed boost (but not as fast as if you use a static - # server like Apache or Litespeed). - class RailsHandler < Mongrel::HttpHandler - attr_reader :files - attr_reader :guard - @@file_only_methods = ["GET","HEAD"] - - def initialize(dir, mime_map = {}) - @files = Mongrel::DirHandler.new(dir,false) - @guard = Mutex.new - - # Register the requested MIME types - mime_map.each {|k,v| Mongrel::DirHandler::add_mime_type(k,v) } - end - - # Attempts to resolve the request as follows: - # - # * If the requested exact PATH_INFO exists as a file then serve it. - # * If it exists at PATH_INFO+".html" exists then serve that. - # * Finally, construct a Mongrel::CGIWrapper and run Dispatcher.dispatch to have Rails go. - def process(request, response) - return if response.socket.closed? - - path_info = request.params[Mongrel::Const::PATH_INFO] - rest_operator = request.params[Mongrel::Const::REQUEST_URI][/^#{Regexp.escape path_info}(;[^\?]+)/, 1].to_s - path_info.chomp!("/") - - page_cached = path_info + rest_operator + ActionController::Base.page_cache_extension - get_or_head = @@file_only_methods.include? request.params[Mongrel::Const::REQUEST_METHOD] - - if get_or_head and @files.can_serve(path_info) - # File exists as-is so serve it up - @files.process(request,response) - elsif get_or_head and @files.can_serve(page_cached) - # Possible cached page, serve it up - request.params[Mongrel::Const::PATH_INFO] = page_cached - @files.process(request,response) - else - begin - cgi = Mongrel::CGIWrapper.new(request, response) - cgi.handler = self - # We don't want the output to be really final until we're out of the lock - cgi.default_really_final = false - - @guard.synchronize { - @active_request_path = request.params[Mongrel::Const::PATH_INFO] - Dispatcher.dispatch(cgi, ActionController::CgiRequest::DEFAULT_SESSION_OPTIONS, response.body) - @active_request_path = nil - } - - # This finalizes the output using the proper HttpResponse way - cgi.out("text/html",true) {""} - rescue Errno::EPIPE - response.socket.close - rescue Object => rails_error - STDERR.puts "#{Time.now}: Error calling Dispatcher.dispatch #{rails_error.inspect}" - STDERR.puts rails_error.backtrace.join("\n") - end - end - end - - # Does the internal reload for Rails. It might work for most cases, but - # sometimes you get exceptions. In that case just do a real restart. - def reload! - begin - @guard.synchronize { - $".replace $orig_dollar_quote - GC.start - Dispatcher.reset_application! - ActionController::Routing::Routes.reload - } - end - end - end - - # Creates Rails specific configuration options for people to use - # instead of the base Configurator. - class RailsConfigurator < Mongrel::Configurator - - # Creates a single rails handler and returns it so you - # can add it to a URI. You can actually attach it to - # as many URIs as you want, but this returns the - # same RailsHandler for each call. - # - # Requires the following options: - # - # * :docroot => The public dir to serve from. - # * :environment => Rails environment to use. - # * :cwd => The change to working directory - # - # And understands the following optional settings: - # - # * :mime => A map of mime types. - # - # Because of how Rails is designed you can only have - # one installed per Ruby interpreter (talk to them - # about thread safety). Because of this the first - # time you call this function it does all the config - # needed to get your Rails working. After that - # it returns the one handler you've configured. - # This lets you attach Rails to any URI(s) you want, - # but it still protects you from threads destroying - # your handler. - def rails(options={}) - - return @rails_handler if @rails_handler - - ops = resolve_defaults(options) - - # fix up some defaults - ops[:environment] ||= "development" - ops[:docroot] ||= "public" - ops[:mime] ||= {} - - $orig_dollar_quote = $".clone - ENV['RAILS_ENV'] = ops[:environment] - env_location = "#{ops[:cwd]}/config/environment" - require env_location - require 'dispatcher' - require 'mongrel/rails' - - ActionController::AbstractRequest.relative_url_root = ops[:prefix] if ops[:prefix] - - @rails_handler = RailsHandler.new(ops[:docroot], ops[:mime]) - end - - # Reloads Rails. This isn't too reliable really, but it - # should work for most minimal reload purposes. The only reliable - # way to reload properly is to stop and then start the process. - def reload! - if not @rails_handler - raise "Rails was not configured. Read the docs for RailsConfigurator." - end - - log "Reloading Rails..." - @rails_handler.reload! - log "Done reloading Rails." - - end - - # Takes the exact same configuration as Mongrel::Configurator (and actually calls that) - # but sets up the additional HUP handler to call reload!. - def setup_rails_signals(options={}) - ops = resolve_defaults(options) - setup_signals(options) - - unless RbConfig::CONFIG['host_os'] =~ /mingw|mswin/ - # rails reload - trap("HUP") { log "HUP signal received."; reload! } - - log "Rails signals registered. HUP => reload (without restart). It might not work well." - end - end - end - end -end diff --git a/lib/mongrel/server.rb b/lib/mongrel/server.rb new file mode 100644 index 00000000..4adb6e5b --- /dev/null +++ b/lib/mongrel/server.rb @@ -0,0 +1,463 @@ +require 'rack' + +module Mongrel + # Thrown at a thread when it is timed out. + class TimeoutError < RuntimeError; end + + # This is the main driver of Mongrel, while the Mongrel::HttpParser and + # Mongrel::URIClassifier make up the majority of how the server functions. + # It's a very simple class that just has a thread accepting connections and + # a simple HttpServer.process_client function to do the heavy lifting with + # the IO and Ruby. + # + # You use it by doing the following: + # + # server = HttpServer.new("0.0.0.0", 3000) + # server.register("/stuff", MyNiftyHandler.new) + # server.run.join + # + # The last line can be just server.run if you don't want to join the + # thread used. If you don't though Ruby will mysteriously just exit on you. + # + # Ruby's thread implementation is "interesting" to say the least. + # Experiments with *many* different types of IO processing simply cannot + # make a dent in it. Future releases of Mongrel will find other creative + # ways to make threads faster, but don't hold your breath until Ruby 1.9 + # is actually finally useful. + + class Server + + include Mongrel::Const + + attr_reader :acceptor + attr_reader :workers + attr_reader :host + attr_reader :port + attr_reader :throttle + attr_reader :timeout + attr_reader :num_processors + + # Creates a working server on host:port (strange things happen if port + # isn't a Number). + # + # Use HttpServer.run to start the server and HttpServer.acceptor.join to + # join the thread that's processing incoming requests on the socket. + # + # The num_processors optional argument is the maximum number of concurrent + # processors to accept, anything over this is closed immediately to maintain + # server processing performance. This may seem mean but it is the most + # efficient way to deal with overload. Other schemes involve still + # parsing the client's request which defeats the point of an overload + # handling system. + # + # The throttle parameter is a sleep timeout (in hundredths of a second) + # that is placed between socket.accept calls in order to give the server + # a cheap throttle time. It defaults to 0 and actually if it is 0 then + # the sleep is not done at all. + def initialize(host, port, num_processors=950, throttle=0, timeout=60) + @socket = TCPServer.new(host, port) + + @host = host + @port = port + @workers = ThreadGroup.new + @throttle = throttle / 100.0 + @num_processors = num_processors + @timeout = timeout + + @check, @notify = IO.pipe + @running = true + end + + def handle_request(params, client, body) + if host = params[HTTP_HOST] + if colon = host.index(":") + params[SERVER_NAME] = host[0, colon] + params[SERVER_PORT] = host[colon+1, host.size] + else + params[SERVER_NAME] = host + params[SERVER_PORT] = PORT_80 + end + end + + params[SERVER_PROTOCOL] = HTTP_11 + params[SERVER_SOFTWARE] = MONGREL_VERSION + params[GATEWAY_INTERFACE] = CGI_VER + + unless params[REQUEST_PATH] + # it might be a dumbass full host request header + uri = URI.parse(params[REQUEST_URI]) + params[REQUEST_PATH] = uri.path + + raise "No REQUEST PATH" unless params[REQUEST_PATH] + end + + # From http://www.ietf.org/rfc/rfc3875 : + # "Script authors should be aware that the REMOTE_ADDR and + # REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9) + # may not identify the ultimate source of the request. + # They identify the client for the immediate request to the + # server; that client may be a proxy, gateway, or other + # intermediary acting on behalf of the actual source client." + # + params[REMOTE_ADDR] = client.peeraddr.last + + process params, client, body + end + + # Does the majority of the IO processing. It has been written in Ruby using + # about 7 different IO processing strategies and no matter how it's done + # the performance just does not improve. It is currently carefully constructed + # to make sure that it gets the best possible performance, but anyone who + # thinks they can make it faster is more than welcome to take a crack at it. + def process_client(client) + begin + parser = HttpParser.new + params = {} + data = client.readpartial(CHUNK_SIZE) + nparsed = 0 + + # Assumption: nparsed will always be less since data will get filled + # with more after each parsing. If it doesn't get more then there was + # a problem with the read operation on the client socket. + # Effect is to stop processing when the socket can't fill the buffer + # for further parsing. + while nparsed < data.length + nparsed = parser.execute(params, data, nparsed) + + if parser.finished? + handle_request params, client, parser.body + break + else + # Parser is not done, queue up more data to read and continue parsing + chunk = client.readpartial(CHUNK_SIZE) + break if !chunk or chunk.length == 0 # read failed, stop processing + + data << chunk + if data.length >= MAX_HEADER + raise HttpParserError, + "HEADER is longer than allowed, aborting client early." + end + end + end + rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EINVAL, + Errno::EBADF + client.close rescue nil + + rescue HttpParserError => e + STDERR.puts "#{Time.now}: HTTP parse error, malformed request (#{params[HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #{e.inspect}" + STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n" + + rescue Errno::EMFILE + reap_dead_workers('too many files') + + rescue Object => e + STDERR.puts "#{Time.now}: Read error: #{e.inspect}" + STDERR.puts e.backtrace.join("\n") + + ensure + begin + client.close + rescue IOError + # Already closed + rescue Object => e + STDERR.puts "#{Time.now}: Client error: #{e.inspect}" + STDERR.puts e.backtrace.join("\n") + end + + request.close_body if request + end + end + + # Used internally to kill off any worker threads that have taken too long + # to complete processing. Only called if there are too many processors + # currently servicing. It returns the count of workers still active + # after the reap is done. It only runs if there are workers to reap. + def reap_dead_workers(reason='unknown') + if @workers.list.length > 0 + STDERR.puts "#{Time.now}: Reaping #{@workers.list.length} threads for slow workers because of '#{reason}'" + error_msg = "Mongrel timed out this thread: #{reason}" + mark = Time.now + @workers.list.each do |worker| + worker[:started_on] = Time.now if not worker[:started_on] + + if mark - worker[:started_on] > @timeout + @throttle + STDERR.puts "Thread #{worker.inspect} is too old, killing." + worker.raise(TimeoutError.new(error_msg)) + end + end + end + + return @workers.list.length + end + + # Performs a wait on all the currently running threads and kills any that + # take too long. It waits by @timeout seconds, which can be set in + # .initialize or via mongrel_rails. The @throttle setting does extend + # this waiting period by that much longer. + def graceful_shutdown + while reap_dead_workers("shutdown") > 0 + STDERR.puts "Waiting for #{@workers.list.length} requests to finish, could take #{@timeout + @throttle} seconds." + sleep @timeout / 10 + end + end + + def configure_socket_options + @tcp_defer_accept_opts = nil + @tcp_cork_opts = nil + + case RUBY_PLATFORM + when /linux/ + # 9 is currently TCP_DEFER_ACCEPT + @tcp_defer_accept_opts = [Socket::SOL_TCP, 9, 1] + @tcp_cork_opts = [Socket::SOL_TCP, 3, 1] + + when /freebsd(([1-4]\..{1,2})|5\.[0-4])/ + # Do nothing, just closing a bug when freebsd <= 5.4 + when /freebsd/ + # Use the HTTP accept filter if available. + # The struct made by pack() is defined in /usr/include/sys/socket.h as accept_filter_arg + unless `/sbin/sysctl -nq net.inet.accf.http`.empty? + @tcp_defer_accept_opts = [Socket::SOL_SOCKET, Socket::SO_ACCEPTFILTER, ['httpready', nil].pack('a16a240')] + end + end + end + + def handle_check + cmd = @check.read(1) + + case cmd + when STOP_COMMAND + @running = false + return true + end + + return false + end + + def handle_overload(client) + STDERR.puts "Server overloaded with #{@workers.list.size} processors (#@num_processors max). Dropping connection." + client.close rescue nil + reap_dead_workers "max processors" + end + + # Runs the thing. It returns the thread used so you can "join" it. + # You can also access the HttpServer::acceptor attribute to get the + # thread later. + def run + BasicSocket.do_not_reverse_lookup = true + + configure_socket_options + + if @tcp_defer_accept_opts + @socket.setsockopt(*@tcp_defer_accept_opts) + end + + tcp_cork_opts = @tcp_cork_opts + + @acceptor = Thread.new do + begin + check = @check + sockets = [check, @socket] + + while @running + begin + ios = IO.select sockets + ios.first.each do |sock| + if sock == check + break if handle_check + else + client = sock.accept + + client.setsockopt(*tcp_cork_opts) if tcp_cork_opts + + worker_list = @workers.list + + if worker_list.length >= @num_processors + handle_overload(client) + else + thread = Thread.new(client) { |c| process_client(c) } + thread[:started_on] = Time.now + @workers.add(thread) + + sleep @throttle if @throttle > 0 + end + end + end + rescue Errno::EMFILE + reap_dead_workers("too many open files") + sleep 0.5 + rescue Errno::ECONNABORTED + # client closed the socket even before accept + client.close rescue nil + rescue Object => e + STDERR.puts "#{Time.now}: Unhandled listen loop exception #{e.inspect}." + STDERR.puts e.backtrace.join("\n") + end + end + graceful_shutdown + ensure + @socket.close + # STDERR.puts "#{Time.now}: Closed socket." + end + end + + return @acceptor + end + + # Stops the acceptor thread and then causes the worker threads to finish + # off the request queue before finally exiting. + def stop(sync=false) + @notify << STOP_COMMAND + + @acceptor.join if sync + end + + end + + class HttpServer < Server + attr_reader :classifier + + def initialize(*) + super + @classifier = URIClassifier.new + end + + def process(params, client, body) + script_name, path_info, handlers = + @classifier.resolve(params[REQUEST_PATH]) + + if handlers + params[PATH_INFO] = path_info + params[SCRIPT_NAME] = script_name + + begin + request = HttpRequest.new(params, client, body) + + # in the case of large file uploads the user could close + # the socket, so skip those requests + rescue BodyReadError => e + return + end + + # request is good so far, continue processing the response + response = HttpResponse.new(client) + + # Process each handler in registered order until we run out + # or one finalizes the response. + handlers.each do |handler| + handler.process(request, response) + break if response.done or client.closed? + end + + # And finally, if nobody closed the response off, we finalize it. + unless response.done or client.closed? + response.finished + end + else + # Didn't find it, return a stock 404 response. + client.write(ERROR_404_RESPONSE) + end + + end + + # Simply registers a handler with the internal URIClassifier. + # When the URI is found in the prefix of a request then your handler's + # HttpHandler::process method is called. + # See Mongrel::URIClassifier#register for more information. + # + # If you set in_front=true then the passed in handler will be put in + # the front of the list for that particular URI. Otherwise it's placed + # at the end of the list. + def register(uri, handler, in_front=false) + begin + @classifier.register(uri, [handler]) + rescue URIClassifier::RegistrationError => e + handlers = @classifier.resolve(uri)[2] + if handlers + # Already registered + method_name = in_front ? 'unshift' : 'push' + handlers.send(method_name, handler) + else + raise + end + end + handler.listener = self + end + + # Removes any handlers registered at the given URI. See Mongrel::URIClassifier#unregister + # for more information. Remember this removes them *all* so the entire + # processing chain goes away. + def unregister(uri) + @classifier.unregister(uri) + end + + end + + class RackServer < Server + attr_accessor :app + + def process(params, client, body) + begin + request = HttpRequest.new(params, client, body) + + # in the case of large file uploads the user could close + # the socket, so skip those requests + rescue BodyReadError => e + return + end + + env = params + + env["SCRIPT_NAME"] = "" + + env["rack.version"] = Rack::VERSION + env["rack.input"] = request.body + env["rack.errors"] = $stderr + env["rack.multithread"] = true + env["rack.multiprocess"] = false + env["rack.run_once"] = true + env["rack.url_scheme"] = env["HTTPS"] ? "https" : "http" + + env["CONTENT_TYPE"] ||= "" + env["QUERY_STRING"] ||= "" + + status, headers, body = @app.call(env) + + begin + client.write "HTTP/1.1 " + client.write status.to_s + client.write " " + client.write HTTP_STATUS_CODES[status] + client.write "\r\nConnection: close\r\n" + + colon = ": " + line_ending = "\r\n" + + headers.each { |k, vs| + vs.split("\n").each { |v| + client.write k + client.write colon + client.write v + client.write line_ending + } + } + + client.write line_ending + + if body.kind_of? String + client.write body + client.flush + else + body.each do |part| + client.write part + client.flush + end + end + ensure + body.close if body.respond_to? :close + end + end + + end + +end diff --git a/test/test_cgi_wrapper.rb b/test/test_cgi_wrapper.rb deleted file mode 100644 index 449f6d07..00000000 --- a/test/test_cgi_wrapper.rb +++ /dev/null @@ -1,26 +0,0 @@ - -require 'test/testhelp' - -class MockHttpRequest - attr_reader :body - - def params - return { 'REQUEST_METHOD' => 'GET'} - end -end - -class CGIWrapperTest < Test::Unit::TestCase - - def test_set_cookies_output_cookies - request = MockHttpRequest.new - response = nil # not needed for this test - output_headers = {} - - cgi = Mongrel::CGIWrapper.new(request, response) - session = CGI::Session.new(cgi, 'database_manager' => CGI::Session::MemoryStore) - cgi.send_cookies(output_headers) - - assert(output_headers.has_key?("Set-Cookie")) - assert_equal("_session_id="+session.session_id+"; path=", output_headers["Set-Cookie"]) - end -end \ No newline at end of file diff --git a/test/test_rack_server.rb b/test/test_rack_server.rb new file mode 100644 index 00000000..22320e9c --- /dev/null +++ b/test/test_rack_server.rb @@ -0,0 +1,66 @@ +require 'test/unit' +require 'mongrel' +require 'rack/lint' +require 'test/testhelp' + +class TestRackServer < Test::Unit::TestCase + + class ErrorChecker + def initialize(app) + @app = app + @exception = nil + @env = nil + end + + attr_reader :exception, :env + + def call(env) + begin + @env = env + return @app.call(env) + rescue Exception => e + @exception = e + + [ + 500, + { "X-Exception" => e.message, "X-Exception-Class" => e.class.to_s }, + ["Error detected"] + ] + end + end + end + + class ServerLint < Rack::Lint + def call(env) + assert("No env given") { env } + check_env env + + @app.call(env) + end + end + + def setup + @valid_request = "GET / HTTP/1.1\r\nHost: test.com\r\nContent-Type: text/plain\r\n\r\n" + + @server = Mongrel::RackServer.new("127.0.0.1", 9998) + @simple = lambda { |env| [200, { "X-Header" => "Works" }, "Hello"] } + @server.app = @simple + end + + def teardown + @server.stop(true) + end + + def test_lint + @checker = ErrorChecker.new ServerLint.new(@simple) + @server.app = @checker + + @server.run + + hit(['http://localhost:9998/test']) + + if exc = @checker.exception + raise exc + end + end +end diff --git a/test/test_request_progress.rb b/test/test_request_progress.rb deleted file mode 100644 index ba21c271..00000000 --- a/test/test_request_progress.rb +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2005 Zed A. Shaw -# You can redistribute it and/or modify it under the same terms as Ruby. -# -# Additional work donated by contributors. See http://mongrel.rubyforge.org/attributions.html -# for more information. - -require 'test/testhelp' - -class UploadBeginHandler < Mongrel::HttpHandler - attr_reader :request_began, :request_progressed, :request_processed - - def initialize - @request_notify = true - end - - def reset - @request_began = false - @request_progressed = false - @request_processed = false - end - - def request_begins(params) - @request_began = true - end - - def request_progress(params,len,total) - @request_progressed = true - end - - def process(request, response) - @request_processed = true - response.start do |head,body| - body.write("test") - end - end - -end - -class RequestProgressTest < Test::Unit::TestCase - def setup - redirect_test_io do - @server = Mongrel::HttpServer.new("127.0.0.1", 9998) - end - @handler = UploadBeginHandler.new - @server.register("/upload", @handler) - @server.run - end - - def teardown - @server.stop(true) - end - - def test_begin_end_progress - Net::HTTP.get("localhost", "/upload", 9998) - assert @handler.request_began - assert @handler.request_progressed - assert @handler.request_processed - end - - def call_and_assert_handlers_in_turn(handlers) - # reset all handlers - handlers.each { |h| h.reset } - - # make the call - Net::HTTP.get("localhost", "/upload", 9998) - - # assert that each one was fired - handlers.each { |h| - assert h.request_began && h.request_progressed && h.request_processed, - "Callbacks NOT fired for #{h}" - } - end - - def test_more_than_one_begin_end_progress - handlers = [@handler] - - second = UploadBeginHandler.new - @server.register("/upload", second) - handlers << second - call_and_assert_handlers_in_turn(handlers) - - # check three handlers - third = UploadBeginHandler.new - @server.register("/upload", third) - handlers << third - call_and_assert_handlers_in_turn(handlers) - - # remove handlers to make sure they've all gone away - @server.unregister("/upload") - handlers.each { |h| h.reset } - Net::HTTP.get("localhost", "/upload", 9998) - handlers.each { |h| - assert !h.request_began && !h.request_progressed && !h.request_processed - } - - # re-register upload to the state before this test - @server.register("/upload", @handler) - end -end