add max_request_size attribute to Mojolicious

mojolicious · Jan 29, 2017 · 334e72b · 334e72b
1 parent 644e4df
commit 334e72b
Show file tree

Hide file tree

Showing 15 changed files with 83 additions and 44 deletions.
diff --git a/Changes b/Changes
@@ -1,5 +1,6 @@
 
-7.23  2017-01-28
+7.23  2017-01-29
+  - Added max_request_size attribute to Mojo::UserAgent.
   - Added max_response_size attribute to Mojo::UserAgent.
   - Added to_unsafe_string method to Mojo::URL.
   - Added -S option to get command.

diff --git a/lib/Mojo/Message.pm b/lib/Mojo/Message.pm
@@ -389,10 +389,7 @@ C<MOJO_MAX_LINE_SIZE> environment variable or C<8192> (8KB).
 
 Maximum message size in bytes, defaults to the value of the
 C<MOJO_MAX_MESSAGE_SIZE> environment variable or C<16777216> (16MB). Setting
-the value to C<0> will allow messages of indefinite size. Note that increasing
-this value can also drastically increase memory usage, should you for example,
-attempt to parse an excessively large message body with the L</"body_params">,
-L</"dom"> or L</"json"> methods.
+the value to C<0> will allow messages of indefinite size.
 
 =head2 version
 
@@ -423,7 +420,8 @@ C<multipart/form-data> message body, usually a L<Mojo::Parameters> object. Note
 that this method caches all data, so it should not be called before the entire
 message body has been received. Parts of the message body need to be loaded
 into memory to parse C<POST> parameters, so you have to make sure it is not
-excessively large.
+excessively large. There's a 16MB limit for requests and a 2GB limit for
+responses by default.
 
   # Get POST parameter names and values
   my $hash = $msg->body_params->to_hash;
@@ -481,7 +479,8 @@ an optional selector can be used to call the method L<Mojo::DOM/"find"> on it
 right away, which then returns a L<Mojo::Collection> object. Note that this
 method caches all data, so it should not be called before the entire message
 body has been received. The whole message body needs to be loaded into memory
-to parse it, so you have to make sure it is not excessively large.
+to parse it, so you have to make sure it is not excessively large. There's a
+16MB limit for requests and a 2GB limit for responses by default.
 
   # Perform "find" right away
   say $msg->dom('h1, h2, h3')->map('text')->join("\n");
@@ -603,7 +602,8 @@ Pointer can be used to extract a specific value with L<Mojo::JSON::Pointer>.
 Note that this method caches all data, so it should not be called before the
 entire message body has been received. The whole message body needs to be
 loaded into memory to parse it, so you have to make sure it is not excessively
-large.
+large. There's a 16MB limit for requests and a 2GB limit for responses by
+default.
 
   # Extract JSON values
   say $msg->json->{foo}{bar}[23];

diff --git a/lib/Mojo/Message/Request.pm b/lib/Mojo/Message/Request.pm
@@ -411,6 +411,7 @@ than just the last one, you can use L</"every_param">. Note that this method
 caches all data, so it should not be called before the entire request body has
 been received. Parts of the request body need to be loaded into memory to parse
 C<POST> parameters, so you have to make sure it is not excessively large.
+There's a 16MB limit for requests by default.
 
 =head2 params
 
@@ -421,7 +422,8 @@ C<application/x-www-form-urlencoded> or C<multipart/form-data> message body,
 usually a L<Mojo::Parameters> object. Note that this method caches all data, so
 it should not be called before the entire request body has been received. Parts
 of the request body need to be loaded into memory to parse C<POST> parameters,
-so you have to make sure it is not excessively large.
+so you have to make sure it is not excessively large. There's a 16MB limit for
+requests by default.
 
   # Get parameter names and values
   my $hash = $req->params->to_hash;

diff --git a/lib/Mojo/Message/Response.pm b/lib/Mojo/Message/Response.pm
@@ -6,6 +6,7 @@ use Mojo::Date;
 use Mojo::Util 'deprecated';
 
 has [qw(code message)];
+has max_message_size => sub { $ENV{MOJO_MAX_MESSAGE_SIZE} // 2147483648 };
 
 # Umarked codes are from RFC 7231
 my %MESSAGES = (
@@ -212,6 +213,15 @@ implements the following new ones.
 
 HTTP response status code.
 
+=head2 max_message_size
+
+  my $size = $res->max_message_size;
+  $res     = $res->max_message_size(1024);
+
+Maximum message size in bytes, defaults to the value of the
+C<MOJO_MAX_MESSAGE_SIZE> environment variable or C<2147483648> (2GB). Setting
+the value to C<0> will allow messages of indefinite size.
+
 =head2 message
 
   my $msg = $res->message;

diff --git a/lib/Mojo/UserAgent.pm b/lib/Mojo/UserAgent.pm
@@ -17,15 +17,14 @@ has ca              => sub { $ENV{MOJO_CA_FILE} };
 has cert            => sub { $ENV{MOJO_CERT_FILE} };
 has connect_timeout => sub { $ENV{MOJO_CONNECT_TIMEOUT} || 10 };
 has cookie_jar      => sub { Mojo::UserAgent::CookieJar->new };
-has 'local_address';
+has [qw(local_address max_response_size)];
 has inactivity_timeout => sub { $ENV{MOJO_INACTIVITY_TIMEOUT} // 20 };
 has ioloop             => sub { Mojo::IOLoop->new };
 has key                => sub { $ENV{MOJO_KEY_FILE} };
 has max_connections    => 5;
 has max_redirects => sub { $ENV{MOJO_MAX_REDIRECTS} || 0 };
-has max_response_size => sub { $ENV{MOJO_MAX_RESPONSE_SIZE} // 2147483648 };
-has proxy             => sub { Mojo::UserAgent::Proxy->new };
-has request_timeout   => sub { $ENV{MOJO_REQUEST_TIMEOUT}   // 0 };
+has proxy => sub { Mojo::UserAgent::Proxy->new };
+has request_timeout => sub { $ENV{MOJO_REQUEST_TIMEOUT} // 0 };
 has server => sub { Mojo::UserAgent::Server->new(ioloop => shift->ioloop) };
 has transactor => sub { Mojo::UserAgent::Transactor->new };
 
@@ -299,7 +298,8 @@ sub _start {
   }
 
   $_->prepare($tx) for $self->proxy, $self->cookie_jar;
-  $tx->res->max_message_size($self->max_response_size);
+  my $max = $self->max_response_size;
+  $tx->res->max_message_size($max) if defined $max;
 
   # Connect and add request timeout if necessary
   my $id = $self->emit(start => $tx)->_connection($loop, $tx, $cb);
@@ -580,12 +580,12 @@ C<0>.
   my $max = $ua->max_response_size;
   $ua     = $ua->max_response_size(16777216);
 
-Maximum response size in bytes, defaults to the value of the
-C<MOJO_MAX_RESPONSE_SIZE> environment variable or C<2147483648> (2GB). Setting
-the value to C<0> will allow responses of indefinite size. Note that increasing
-this value can also drastically increase memory usage, should you for example,
-attempt to parse an excessively large response body with the methods
-L<Mojo::Message/"dom"> or L<Mojo::Message/"json">.
+Maximum response size in bytes, defaults to the value of
+L<Mojo::Message::Response/"max_message_size">. Setting the value to C<0> will
+allow responses of indefinite size. Note that increasing this value can also
+drastically increase memory usage, should you for example attempt to parse an
+excessively large response body with the methods L<Mojo::Message/"dom"> or
+L<Mojo::Message/"json">.
 
 =head2 proxy
 

diff --git a/lib/Mojolicious.pm b/lib/Mojolicious.pm
@@ -37,6 +37,7 @@ has log              => sub {
   # Reduced log output outside of development mode
   return $mode eq 'development' ? $log : $log->level('info');
 };
+has 'max_request_size';
 has mode => sub { $ENV{MOJO_MODE} || $ENV{PLACK_ENV} || 'development' };
 has moniker  => sub { Mojo::Util::decamelize ref shift };
 has plugins  => sub { Mojolicious::Plugins->new };
@@ -92,8 +93,12 @@ sub build_controller {
 
 sub build_tx {
   my $self = shift;
-  my $tx   = Mojo::Transaction::HTTP->new;
+
+  my $tx  = Mojo::Transaction::HTTP->new;
+  my $max = $self->max_request_size;
+  $tx->req->max_message_size($max) if defined $max;
   $self->plugins->emit_hook(after_build_tx => $tx, $self);
+
   return $tx;
 }
 
@@ -412,6 +417,18 @@ if a C<log> directory exists.
   # Log debug message
   $app->log->debug('It works');
 
+=head2 max_request_size
+
+  my $max = $app->max_request_size;
+  $app    = $app->max_request_size(16777216);
+
+Maximum request size in bytes, defaults to the value of
+L<Mojo::Message/"max_message_size">. Setting the value to C<0> will allow
+requests of indefinite size. Note that increasing this value can also
+drastically increase memory usage, should you for example attempt to parse an
+excessively large response body with the methods L<Mojo::Message/"dom"> or
+L<Mojo::Message/"json">.
+
 =head2 mode
 
   my $mode = $app->mode;

diff --git a/lib/Mojolicious/Command/get.pm b/lib/Mojolicious/Command/get.pm
@@ -162,9 +162,7 @@ Mojolicious::Command::get - Get command
                                          of MOJO_CONNECT_TIMEOUT or 10
     -r, --redirect                       Follow up to 10 redirects
     -S, --response-size <size>           Maximum response size in bytes,
-                                         defaults to the value of
-                                         MOJO_MAX_RESPONSE_SIZE or
-                                         2147483648 (2GB)
+                                         defaults to 2147483648 (2GB)
     -v, --verbose                        Print request and response headers to
                                          STDERR
 

diff --git a/lib/Mojolicious/Controller.pm b/lib/Mojolicious/Controller.pm
@@ -584,6 +584,7 @@ message body, in that order. If there are multiple values sharing the same
 name, and you want to access more than just the last one, you can use
 L</"every_param">. Parts of the request body need to be loaded into memory to
 parse C<POST> parameters, so you have to make sure it is not excessively large.
+There's a 16MB limit for requests by default.
 
   # Get first value
   my $first = $c->every_param('foo')->[0];
@@ -930,7 +931,7 @@ validate file uploads as well as C<GET> and C<POST> parameters extracted from
 the query string and C<application/x-www-form-urlencoded> or
 C<multipart/form-data> message body. Parts of the request body need to be loaded
 into memory to parse C<POST> parameters, so you have to make sure it is not
-excessively large.
+excessively large. There's a 16MB limit for requests by default.
 
   # Validate GET/POST parameter
   my $validation = $c->validation;

diff --git a/lib/Mojolicious/Guides/Cookbook.pod b/lib/Mojolicious/Guides/Cookbook.pod
@@ -1123,6 +1123,9 @@ To protect you from excessively large files there is also a limit of 2GB by
 default, which you can tweak with the attribute
 L<Mojo::UserAgent/"max_response_size">.
 
+  # Increase limit to 10GB
+  $ua->max_response_size(10737418240);
+
 =head2 Large file upload
 
 Uploading a large file is even easier.

diff --git a/lib/Mojolicious/Guides/FAQ.pod b/lib/Mojolicious/Guides/FAQ.pod
@@ -189,11 +189,10 @@ Hypnotoad.
 
 To protect your applications from excessively large requests and responses, our
 HTTP parser has a cap after which it will automatically stop accepting new
-data, and in most cases force the connection to be closed. The limit is 16MB on
-the server-side, and 2GB on the client-side by default. You can use the
-attributes L<Mojo::Message/"max_message_size"> and
-L<Mojo::UserAgent/"max_response_size"> or C<MOJO_MAX_MESSAGE_SIZE> and
-C<MOJO_MAX_RESPONSE_SIZE> environment variables to change these values.
+data, and in most cases force the connection to be closed. The limit is 16MB for
+requests, and 2GB for responses by default. You can use the attributes
+L<Mojolicious/"max_request_size"> and L<Mojo::UserAgent/"max_response_size"> to
+change these values.
 
 =head2 What does the error "Maximum start-line size exceeded" mean?
 

diff --git a/lib/Mojolicious/Guides/Tutorial.pod b/lib/Mojolicious/Guides/Tutorial.pod
@@ -834,11 +834,10 @@ L<Mojolicious::Plugin::TagHelpers/"form_for">.
 
 To protect you from excessively large files there is also a limit of 16MB by
 default, which you can tweak with the attribute
-L<Mojo::Message/"max_message_size"> or C<MOJO_MAX_MESSAGE_SIZE> environment
-variable.
+L<Mojolicious/"max_request_size">.
 
   # Increase limit to 1GB
-  $ENV{MOJO_MAX_MESSAGE_SIZE} = 1073741824;
+  app->max_request_size(1073741824);
 
 =head2 User agent
 

diff --git a/t/mojo/request.t b/t/mojo/request.t
@@ -10,8 +10,12 @@ use Mojo::Message::Request;
 use Mojo::URL;
 use Mojo::Util 'encode';
 
-# Parse HTTP 1.1 message with huge "Cookie" header exceeding all limits
+# Defaults
 my $req = Mojo::Message::Request->new;
+is $req->max_message_size, 16777216, 'right default';
+
+# Parse HTTP 1.1 message with huge "Cookie" header exceeding all limits
+$req = Mojo::Message::Request->new;
 my $finished;
 $req->max_message_size($req->headers->max_line_size);
 my $huge = 'a=b; ' x $req->max_message_size;

diff --git a/t/mojo/response.t b/t/mojo/response.t
@@ -9,8 +9,12 @@ use Mojo::JSON 'encode_json';
 use Mojo::Message::Response;
 use Mojo::Util 'encode';
 
-# Common status codes
+# Defaults
 my $res = Mojo::Message::Response->new;
+is $res->max_message_size, 2147483648, 'right default';
+
+# Common status codes
+$res = Mojo::Message::Response->new;
 is $res->code(100)->default_message, 'Continue',            'right message';
 is $res->code(101)->default_message, 'Switching Protocols', 'right message';
 is $res->code(102)->default_message, 'Processing',          'right message';

diff --git a/t/mojo/user_agent.t b/t/mojo/user_agent.t
@@ -68,15 +68,6 @@ get '/one' => sub {
   is(Mojo::UserAgent->new->max_redirects, 0, 'right value');
 }
 
-# Max response size
-{
-  is(Mojo::UserAgent->new->max_response_size, 2147483648, 'right value');
-  local $ENV{MOJO_MAX_RESPONSE_SIZE} = 25;
-  is(Mojo::UserAgent->new->max_response_size, 25, 'right value');
-  local $ENV{MOJO_MAX_RESPONSE_SIZE} = 0;
-  is(Mojo::UserAgent->new->max_response_size, 0, 'right value');
-}
-
 # Timeouts
 {
   is(Mojo::UserAgent->new->connect_timeout, 10, 'right value');

diff --git a/t/mojolicious/upload_lite_app.t b/t/mojolicious/upload_lite_app.t
@@ -9,6 +9,11 @@ use Mojo::Content::Single;
 use Mojolicious::Lite;
 use Test::Mojo;
 
+get '/request_size' => sub {
+  my $c = shift;
+  $c->render(text => $c->req->max_message_size);
+};
+
 post '/upload' => sub {
   my $c       = shift;
   my $file    = $c->param('file');
@@ -28,6 +33,11 @@ post '/multi' => sub {
 
 my $t = Test::Mojo->new;
 
+# Request size limit
+$t->get_ok('/request_size')->status_is(200)->content_is(16777216);
+$t->app->max_request_size(33554432);
+$t->get_ok('/request_size')->status_is(200)->content_is(33554432);
+
 # Asset and filename
 my $file = Mojo::Asset::File->new->add_chunk('lalala');
 $t->post_ok('/upload' => form =>