make Hypnotoad a little more resilient to exceptions

mojolicious · Jan 25, 2015 · ce7a188 · ce7a188
1 parent da28997
commit ce7a188
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 20 deletions.
diff --git a/Changes b/Changes
@@ -1,5 +1,10 @@
 
 5.75  2015-01-26
+  - Added healthy method to Mojo::Server::Prefork.
+  - Improved all built-in web servers to die if group or user assignment
+    failed.
+  - Improved Hypnotoad to wait for new workers to be ready before stopping the
+    old ones during hot deployment.
   - Fixed race condition and memory leak in Mojo::Server::Prefork.
 
 5.74  2015-01-25

diff --git a/lib/Mojo/Server.pm b/lib/Mojo/Server.pm
@@ -76,23 +76,23 @@ sub setuidgid {
 
   # Group (make sure secondary groups are reassigned too)
   if (my $group = $self->group) {
-    return $self->_log(qq{Group "$group" does not exist.})
+    $self->_error(qq{Group "$group" does not exist})
       unless defined(my $gid = getgrnam $group);
-    return $self->_log(qq{Can't switch to group "$group": $!})
+    $self->_error(qq{Can't switch to group "$group": $!})
       unless ($( = $) = "$gid $gid") && $) eq "$gid $gid" && $( eq "$gid $gid";
   }
 
   # User
   return $self unless my $user = $self->user;
-  return $self->_log(qq{User "$user" does not exist.})
+  $self->_error(qq{User "$user" does not exist})
     unless defined(my $uid = getpwnam $user);
-  return $self->_log(qq{Can't switch to user "$user": $!})
+  $self->_error(qq{Can't switch to user "$user": $!})
     unless POSIX::setuid($uid);
 
   return $self;
 }
 
-sub _log { $_[0]->app->log->error($_[1]) and return $_[0] }
+sub _error { $_[0]->app->log->error("$_[1].") and croak $_[1] }
 
 1;
 

diff --git a/lib/Mojo/Server/Hypnotoad.pm b/lib/Mojo/Server/Hypnotoad.pm
@@ -94,8 +94,10 @@ sub _manage {
   my $self = shift;
 
   # Upgraded
-  my $log = $self->prefork->app->log;
+  my $prefork = $self->prefork;
+  my $log     = $prefork->app->log;
   if ($ENV{HYPNOTOAD_PID} && $ENV{HYPNOTOAD_PID} ne $$) {
+    return unless $prefork->healthy == $prefork->workers;
     $log->info("Upgrade successful, stopping $ENV{HYPNOTOAD_PID}.");
     kill 'QUIT', $ENV{HYPNOTOAD_PID};
   }

diff --git a/lib/Mojo/Server/Prefork.pm b/lib/Mojo/Server/Prefork.pm
@@ -58,6 +58,10 @@ sub ensure_pid_file {
   print $handle $$;
 }
 
+sub healthy {
+  scalar grep { $_->{healthy} } values %{$_[0]{pool}};
+}
+
 sub run {
   my $self = shift;
 
@@ -78,8 +82,9 @@ sub run {
   local $SIG{INT} = local $SIG{TERM} = sub { $self->_term };
   local $SIG{CHLD} = sub {
     while ((my $pid = waitpid -1, WNOHANG) > 0) {
-      $self->app->log->debug("Worker $pid stopped.")
-        if delete $self->emit(reap => $pid)->{pool}{$pid};
+      next unless my $w = delete $self->emit(reap => $pid)->{pool}{$pid};
+      $self->app->log->debug("Worker $pid stopped.");
+      $self->{finished}++ unless $w->{healthy};
     }
   };
   local $SIG{QUIT} = sub { $self->_term(1) };
@@ -96,7 +101,9 @@ sub run {
   $self->_manage while $self->{running};
 }
 
-sub _heartbeat {
+sub _heartbeat { shift->{writer}->syswrite("$$:$_[0]\n") or exit 0 }
+
+sub _heartbeats {
   my $self = shift;
 
   # Poll for heartbeats
@@ -109,7 +116,7 @@ sub _heartbeat {
   my $time = steady_time;
   while ($chunk =~ /(\d+):(\d)\n/g) {
     next unless my $w = $self->{pool}{$1};
-    $self->emit(heartbeat => $1) and $w->{time} = $time;
+    @$w{qw(healthy time)} = (1, $time) and $self->emit(heartbeat => $1);
     $w->{graceful} ||= $time if $2;
   }
 }
@@ -127,7 +134,7 @@ sub _manage {
   elsif (!keys %{$self->{pool}}) { return delete $self->{running} }
 
   # Wait for heartbeats
-  $self->emit('wait')->_heartbeat;
+  $self->emit('wait')->_heartbeats;
 
   my $interval = $self->heartbeat_interval;
   my $ht       = $self->heartbeat_timeout;
@@ -196,12 +203,9 @@ sub _spawn {
   $loop->unlock(sub { flock $handle, LOCK_UN });
 
   # Heartbeat messages
-  $loop->recurring(
-    $self->heartbeat_interval => sub {
-      my $graceful = shift->max_connections ? 0 : 1;
-      $self->{writer}->syswrite("$$:$graceful\n") or exit 0;
-    }
-  );
+  $loop->next_tick(sub { $self->_heartbeat(0) });
+  my $cb = sub { $self->_heartbeat(shift->max_connections ? 0 : 1) };
+  $loop->recurring($self->heartbeat_interval => $cb);
 
   # Clean worker environment
   $SIG{$_} = 'DEFAULT' for qw(INT TERM CHLD TTIN TTOU);
@@ -497,6 +501,12 @@ is not running.
 
 Ensure L</"pid_file"> exists.
 
+=head2 healthy
+
+  my $healthy = $prefork->healthy;
+
+Number of active worker processes with a heartbeat.
+
 =head2 run
 
   $prefork->run;

diff --git a/t/mojo/prefork.t b/t/mojo/prefork.t
@@ -41,21 +41,24 @@ $prefork->on(
   }
 );
 is $prefork->workers, 4, 'start with four workers';
-my (@spawn, @reap, $worker, $tx, $graceful);
+my (@spawn, @reap, $worker, $tx, $graceful, $healthy);
 $prefork->on(spawn => sub { push @spawn, pop });
 $prefork->once(
   heartbeat => sub {
     my ($prefork, $pid) = @_;
-    $worker = $pid;
-    $tx     = Mojo::UserAgent->new->get("http://127.0.0.1:$port");
+    $worker  = $pid;
+    $healthy = $prefork->healthy;
+    $tx      = Mojo::UserAgent->new->get("http://127.0.0.1:$port");
     kill 'QUIT', $$;
   }
 );
 $prefork->on(reap => sub { push @reap, pop });
 $prefork->on(finish => sub { $graceful = pop });
 my $log = '';
 my $cb = $prefork->app->log->on(message => sub { $log .= pop });
+is $prefork->healthy, 0, 'no healthy workers';
 $prefork->run;
+ok $healthy >= 1, 'healthy workers';
 is scalar @spawn, 4, 'four workers spawned';
 is scalar @reap,  4, 'four workers reaped';
 ok !!grep { $worker eq $_ } @spawn, 'worker has a heartbeat';