NixOS · Sep 24, 2019 · Sep 24, 2019 · Sep 25, 2019 · Nov 5, 2019 · Nov 5, 2019
diff --git a/flake.nix b/flake.nix
@@ -77,6 +77,7 @@
                 LWP
                 LWPProtocolHttps
                 NetAmazonS3
+                NetPrometheus
                 NetStatsd
                 PadWalker
                 Readonly
@@ -324,8 +325,9 @@
       nixosConfigurations.container = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
         modules =
-          [ { imports = [ self.nixosModules.hydraTest self.nixosModules.hydraProxy ];
-              system.configurationRevision = self.rev;
+          [ self.nixosModules.hydraTest
+            self.nixosModules.hydraProxy
+            { system.configurationRevision = self.rev;
 
               boot.isContainer = true;
               networking.useDHCP = false;

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
@@ -504,7 +504,7 @@ sub restart : Chained('buildChain') PathPart Args(0) {
 sub cancel : Chained('buildChain') PathPart Args(0) {
     my ($self, $c) = @_;
     my $build = $c->stash->{build};
-    requireProjectOwner($c, $build->project);
+    requireCancelBuildPrivileges($c, $build->project);
     my $n = cancelBuilds($c->model('DB')->schema, $c->model('DB::Builds')->search({ id => $build->id }));
     error($c, "This build cannot be cancelled.") if $n != 1;
     $c->flash->{successMsg} = "Build has been cancelled.";
@@ -540,7 +540,7 @@ sub bump : Chained('buildChain') PathPart('bump') {
 
     my $build = $c->stash->{build};
 
-    requireProjectOwner($c, $build->project); # FIXME: require admin?
+    requireBumpPrivileges($c, $build->project);
 
     $c->model('DB')->schema->txn_do(sub {
         $build->update({globalpriority => time()});

diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -179,7 +179,7 @@ sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
 
 sub cancel : Chained('evalChain') PathPart('cancel') Args(0) {
     my ($self, $c) = @_;
-    requireProjectOwner($c, $c->stash->{eval}->project);
+    requireCancelBuildPrivileges($c, $c->stash->{eval}->project);
     my $n = cancelBuilds($c->model('DB')->schema, $c->stash->{eval}->builds);
     $c->flash->{successMsg} = "$n builds have been cancelled.";
     $c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for('view'), $c->req->captures));
@@ -210,7 +210,7 @@ sub restart_failed : Chained('evalChain') PathPart('restart-failed') Args(0) {
 
 sub bump : Chained('evalChain') PathPart('bump') Args(0) {
     my ($self, $c) = @_;
-    requireProjectOwner($c, $c->stash->{eval}->project); # FIXME: require admin?
+    requireBumpPrivileges($c, $c->stash->{eval}->project); # FIXME: require admin?
     my $builds = $c->stash->{eval}->builds->search({ finished => 0 });
     my $n = $builds->count();
     $c->model('DB')->schema->txn_do(sub {

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
@@ -6,13 +6,15 @@ use warnings;
 use base 'Hydra::Base::Controller::ListBuilds';
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
+use Hydra::View::TT;
 use Digest::SHA1 qw(sha1_hex);
 use Nix::Store;
 use Nix::Config;
 use Encode;
 use File::Basename;
 use JSON;
 use List::MoreUtils qw{any};
+use Net::Prometheus;
 
 # Put this controller at top-level.
 __PACKAGE__->config->{namespace} = '';
@@ -200,6 +202,49 @@ sub machines :Local Args(0) {
     $self->status_ok($c, entity => $c->stash->{machines});
 }
 
+sub prometheus :Local Args(0) {
+    my ($self, $c) = @_;
+    my $machines = getMachines;
+
+    my $client = Net::Prometheus->new;
+    my $duration = $client->new_histogram(
+        name => "hydra_machine_build_duration",
+        help => "How long builds are taking per server. Note: counts are gauges, NOT counters.",
+        labels => [ "machine" ],
+        buckets => [
+            60,
+            600,
+            1800,
+            3600,
+            7200,
+            21600,
+            43200,
+            86400,
+            172800,
+            259200,
+            345600,
+            518400,
+            604800,
+            691200
+        ]
+    );
+
+    my $steps = dbh($c)->selectall_arrayref(
+        "select machine, s.starttime as starttime " .
+        "from BuildSteps s join Builds b on s.build = b.id " .
+        "where busy != 0 order by machine, stepnr",
+        { Slice => {} });
+
+    foreach my $step (@$steps) {
+        my $name = $step->{machine} ? Hydra::View::TT->stripSSHUser(undef, $step->{machine}) : "";
+        $name = "localhost" unless $name;
+        $duration->labels($name)->observe(time - $step->{starttime});
+    }
+
+    $c->stash->{'plain'} = { data => $client->render };
+    $c->forward('Hydra::View::Plain');
+}
+
 
 # Hydra::Base::Controller::ListBuilds needs this.
 sub get_builds : Chained('/') PathPart('') CaptureArgs(0) {

diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -13,6 +13,8 @@ our @EXPORT = qw(
     searchBuildsAndEvalsForJobset
     error notFound gone accessDenied
     forceLogin requireUser requireProjectOwner requireRestartPrivileges requireAdmin requirePost isAdmin isProjectOwner
+    requireBumpPrivileges
+    requireCancelBuildPrivileges
     trim
     getLatestFinishedEval getFirstEval
     paramToList
@@ -181,6 +183,48 @@ sub isProjectOwner {
          defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
 }
 
+sub hasCancelBuildRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles('cancel-build');
+}
+
+sub mayCancelBuild {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasCancelBuildRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireCancelBuildPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with cancel-build privileges can perform this operation.")
+        unless mayCancelBuild($c, $project);
+}
+
+sub hasBumpJobsRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles('bump-to-front');
+}
+
+sub mayBumpJobs {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasBumpJobsRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireBumpPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with bump-to-front privileges can perform this operation.")
+        unless mayBumpJobs($c, $project);
+}
+
 sub hasRestartJobsRole {
     my ($c) = @_;
     return $c->user_exists && $c->check_user_roles('restart-jobs');

diff --git a/src/root/user.tt b/src/root/user.tt
@@ -81,6 +81,8 @@
             [% INCLUDE roleoption role="admin" %]
             [% INCLUDE roleoption role="create-projects" %]
             [% INCLUDE roleoption role="restart-jobs" %]
+            [% INCLUDE roleoption role="bump-to-front" %]
+            [% INCLUDE roleoption role="cancel-build" %]
           </select>
         </div>
       </div>

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
@@ -440,8 +440,6 @@ sub checkBuild {
         # new build to be scheduled if the meta.maintainers field is
         # changed?
         if (defined $prevEval) {
-            # Only check one output: if it's the same, the other will be as well.
-            my $firstOutput = $outputNames[0];
             my ($prevBuild) = $prevEval->builds->search(
                 # The "project" and "jobset" constraints are
                 # semantically unnecessary (because they're implied by