Skip to content

Commit

Permalink
channel updates: make sequential
Browse files Browse the repository at this point in the history 
Create a sequential ordering with systemd After-relationships between
each channel's update job. Then, remove individual timers and create
one timer for all the jobs.

Fixes #79
  • Loading branch information
@grahamc
grahamc committed Mar 16, 2019
1 parent 28597b4 commit a6862fb
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 37 deletions.
100 changes: 63 additions & 37 deletions modules/hydra-mirror.nix
View file
@@ -1,54 +1,80 @@
{ config, lib, pkgs, ... }:
{ lib, pkgs, ... }:

with lib;

let
channels = {
# Channel name https://hydra.nixos.org/job/<value>/latest-finished
"nixos-unstable" = "nixos/trunk-combined/tested";
"nixos-unstable-small" = "nixos/unstable-small/tested";
"nixpkgs-unstable" = "nixpkgs/trunk/unstable";

"nixos-19.03" = "nixos/release-19.03/tested";
"nixos-19.03-small" = "nixos/release-19.03-small/tested";
"nixpkgs-19.03-darwin" = "nixpkgs/nixpkgs-19.03-darwin/darwin-tested";

"nixos-18.09" = "nixos/release-18.09/tested";
"nixos-18.09-small" = "nixos/release-18.09-small/tested";
"nixpkgs-18.09-darwin" = "nixpkgs/nixpkgs-18.09-darwin/darwin-tested";

"nixos-18.03" = "nixos/release-18.03/tested";
"nixos-18.03-small" = "nixos/release-18.03-small/tested";
"nixpkgs-18.03-darwin" = "nixpkgs/nixpkgs-18.03-darwin/darwin-tested";
};

channelScripts = import <nixos-channel-scripts> { inherit pkgs; };
orderLib = import ./service-order.nix { inherit lib; };

makeUpdateChannel = channelName: mainJob:
{ timers."update-${channelName}" =
{ wantedBy = [ "timers.target" ];
timerConfig.OnUnitInactiveSec = 600;
timerConfig.OnBootSec = 900;
timerConfig.AccuracySec = 300;
};

services."update-${channelName}" =
{ description = "Update Channel ${channelName}";
after = [ "networking.target" ];
path = [ channelScripts ];
script =
''
# FIXME: use IAM role.
export AWS_ACCESS_KEY_ID=$(sed 's/aws_access_key_id=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
export AWS_SECRET_ACCESS_KEY=$(sed 's/aws_secret_access_key=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
exec mirror-nixos-branch ${channelName} https://hydra.nixos.org/job/${mainJob}/latest-finished
''; # */
serviceConfig.User = "hydra-mirror";
environment.TMPDIR = "/scratch/hydra-mirror";
environment.GC_INITIAL_HEAP_SIZE = "4g";
{
name = "update-${channelName}";
value = {
description = "Update Channel ${channelName}";
path = [ channelScripts ];
script =
''
# FIXME: use IAM role.
export AWS_ACCESS_KEY_ID=$(sed 's/aws_access_key_id=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
export AWS_SECRET_ACCESS_KEY=$(sed 's/aws_secret_access_key=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
exec mirror-nixos-branch ${channelName} https://hydra.nixos.org/job/${mainJob}/latest-finished
'';
serviceConfig.User = "hydra-mirror";
unitConfig.After = [ "networking.target" ];
environment.TMPDIR = "/scratch/hydra-mirror";
environment.GC_INITIAL_HEAP_SIZE = "4g";
};
};

updateJobs = orderLib.mkOrderedChain
(mapAttrsToList makeUpdateChannel channels);

in

{
imports = [ ./hydra-mirror-user.nix ];

systemd =
fold recursiveUpdate {} [
(makeUpdateChannel "nixos-19.03" "nixos/release-19.03/tested")
(makeUpdateChannel "nixos-19.03-small" "nixos/release-19.03-small/tested")
(makeUpdateChannel "nixos-18.09" "nixos/release-18.09/tested")
(makeUpdateChannel "nixos-18.09-small" "nixos/release-18.09-small/tested")
(makeUpdateChannel "nixos-18.03" "nixos/release-18.03/tested")
(makeUpdateChannel "nixos-18.03-small" "nixos/release-18.03-small/tested")
(makeUpdateChannel "nixos-unstable" "nixos/trunk-combined/tested")
(makeUpdateChannel "nixos-unstable-small" "nixos/unstable-small/tested")
(makeUpdateChannel "nixpkgs-19.03-darwin" "nixpkgs/nixpkgs-19.03-darwin/darwin-tested")
(makeUpdateChannel "nixpkgs-18.09-darwin" "nixpkgs/nixpkgs-18.09-darwin/darwin-tested")
(makeUpdateChannel "nixpkgs-18.03-darwin" "nixpkgs/nixpkgs-18.03-darwin/darwin-tested")
(makeUpdateChannel "nixpkgs-unstable" "nixpkgs/trunk/unstable")
];
systemd.services = (listToAttrs updateJobs) // {
"update-all-channels" = {
description = "Start all channel updates.";
unitConfig = {
After = builtins.map
(service: "${service.name}.service")
updateJobs;
Wants = builtins.map
(service: "${service.name}.service")
updateJobs;
};
script = "true";
};
};

systemd.timers."update-all-channels" = {
description = "Start all channel updates.";
wantedBy = [ "timers.target" ];
timerConfig = {
OnUnitInactiveSec = 600;
OnBootSec = 900;
AccuracySec = 300;
};
};
}
61 changes: 61 additions & 0 deletions modules/service-order.nix
View file
@@ -0,0 +1,61 @@
# Ordering Services
#
# Given a set of services, make them run one at a time in a specific
# order, on a timer.
{ lib, ... }:
{
# Given a list of systemd service, give each one an After
# attribute, so they start in a specific order. The returned
# list can be converted in to a systemd.services attrset with
# `lib.listToAttrs`.
#
# Example:
#
# mkOrderedChain [
# { name = "foo"; value = { script = "true"; }; }
# { name = "bar"; value = { script = "true"; }; }
# ]
#
# => [
# {
# name = "foo";
# value = {
# script = "true";
# unitConfig = { After = []; };
# };
# }
# {
# name = "bar";
# value = {
# script = "true";
# unitConfig = { After = [ "bar" ]; };
# };
# }
#
mkOrderedChain = jobs: let
unitConfigFrom = job: job.unitConfig or {};
afterFrom = job: (unitConfigFrom job).After or [];
previousFrom = collector:
if collector ? previous
then [collector.previous]
else [];

ordered = builtins.foldl'
(collector: item: {
services = collector.services
++ [{
inherit (item) name;
value = item.value // {
unitConfig = (unitConfigFrom item.value) //
{
After = (afterFrom item.value) ++
(previousFrom collector);
};
};
}];
previous = item.name;
})
{ services = []; }
jobs;
in ordered.services;
}

0 comments on commit a6862fb

Please sign in to comment.