NixOS · Jul 18, 2019 · Jul 18, 2019 · Jul 18, 2019 · Jul 19, 2019 · Aug 18, 2019
diff --git a/delft/ceres.nix b/delft/ceres.nix
@@ -0,0 +1,64 @@
+{ nodes, config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ ./common.nix
+      ./hydra.nix
+      ./hydra-proxy.nix
+      ./fstrim.nix
+      ../modules/wireguard.nix
+      ./packet-importer.nix
+    ];
+
+  deployment.targetEnv = "hetzner";
+  deployment.hetzner.mainIPv4 = "46.4.66.184";
+
+  # FIXME: manually changed nvme0n1p1 to a /boot disk. We can't use
+  # GRUB on a RAID-0 disk because it doesn't support the "large_dir"
+  # ext4 option.
+  /*
+  deployment.hetzner.partitions = ''
+    clearpart --all --initlabel --drives=nvme0n1,nvme1n1
+
+    part raid.1 --ondisk=nvme0n1 --size=16384
+    part raid.2 --ondisk=nvme1n1 --size=16384
+
+    part raid.3 --grow --ondisk=nvme0n1
+    part raid.4 --grow --ondisk=nvme1n1
+
+    raid swap --level=1 --device=md0 --fstype=swap --label=root raid.1 raid.2
+    raid /    --level=1 --device=md1 --fstype=ext4 --label=root raid.3 raid.4
+  '';
+  */
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-label/boot";
+      fsType = "ext4";
+    };
+
+  swapDevices = lib.mkForce [];
+
+  networking = {
+    firewall.allowedTCPPorts = [ 80 443 ];
+    firewall.allowPing = true;
+    firewall.logRefusedConnections = true;
+  };
+
+  services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.254.1.2;user=hydra;";
+  #systemd.services.hydra-init.wants = [ "sys-devices-virtual-net-wg0.device" ];
+
+  services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
+
+  nix.gc.automatic = true;
+  nix.gc.options = ''--max-freed "$((100 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
+  nix.gc.dates = "03,09,15,21:15";
+
+  nix.extraOptions = "gc-keep-outputs = false";
+
+  networking.defaultMailServer.directDelivery = lib.mkForce false;
+  #services.postfix.enable = true;
+  #services.postfix.hostname = "hydra.nixos.org";
+
+  # Don't rate-limit the journal.
+  services.journald.rateLimitBurst = 0;
+}
diff --git a/delft/chef.nix b/delft/chef.nix
@@ -3,13 +3,9 @@
 {
   imports =
     [ ./common.nix
-      ./hydra.nix
-      ./hydra-proxy.nix
       ./datadog.nix
       ./fstrim.nix
-      ./provisioner.nix
       ../modules/wireguard.nix
-      ./packet-importer.nix
     ];
 
   deployment.targetEnv = "hetzner";
@@ -19,6 +15,8 @@
     enable = true;
     package = pkgs.postgresql95;
     extraConfig = ''
+      listen_addresses = '10.254.1.2'
+
       log_min_duration_statement = 5000
       log_duration = off
       log_statement = 'none'
@@ -35,32 +33,20 @@
 
       effective_cache_size = 16GB
     '';
+    # FIXME: don't use 'trust'.
+    authentication = ''
+      host hydra all 10.254.1.3/32 trust
+    '';
   };
 
   networking = {
-
-    firewall.allowedTCPPorts = [ 80 443 ];
+    firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
     firewall.allowPing = true;
     firewall.logRefusedConnections = true;
-
   };
 
-  nix.gc.automatic = true;
-  nix.gc.options = ''--max-freed "$((100 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
-  nix.gc.dates = "03,09,15,21:15";
-
-  nix.extraOptions = "gc-keep-outputs = false";
-
-  networking.defaultMailServer.directDelivery = lib.mkForce false;
-  #services.postfix.enable = true;
-  #services.postfix.hostname = "hydra.nixos.org";
-
-  # Don't rate-limit the journal.
-  services.journald.rateLimitBurst = 0;
-
   fileSystems."/data" =
     { device = "/dev/disk/by-label/data";
       fsType = "ext4";
     };
-
 }
diff --git a/delft/eris.nix b/delft/eris.nix
@@ -1,7 +1,7 @@
 { nodes, config, lib, pkgs, ... }:
-let
 
-in { deployment.targetEnv = "hetzner";
+{
+  deployment.targetEnv = "hetzner";
   deployment.hetzner.mainIPv4 = "138.201.32.77";
 
   networking.extraHosts = ''

diff --git a/delft/hydra-proxy.nix b/delft/hydra-proxy.nix
@@ -2,8 +2,6 @@
 
 let
 
-  hydraCacheDir = "/var/cache/hydra-binary-cache";
-
   hydraProxyConfig =
     ''
       TimeOut 900
@@ -21,11 +19,6 @@ let
       ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
       ProxyPassReverse  /       http://127.0.0.1:3000/
 
-      #CacheEnable disk /
-      #CacheRoot ${hydraCacheDir}
-      #CacheMaxFileSize 64000000
-      #CacheIgnoreHeaders Set-Cookie
-
       <Location />
         SetOutputFilter DEFLATE
         BrowserMatch ^Mozilla/4\.0[678] no-gzip\
@@ -81,22 +74,6 @@ in
 
   };
 
-  /*
-  system.activationScripts.createHydraCache =
-    ''
-      mkdir -p ${hydraCacheDir}
-      chown wwwrun ${hydraCacheDir}
-    '';
-
-  systemd.services.htcacheclean =
-    { description = "Clean httpd Cache";
-      serviceConfig.ExecStart =
-        "${config.services.httpd.package}/bin/htcacheclean " +
-        "-v -t -l 4G -p /var/cache/hydra-binary-cache";
-      startAt = "05:45";
-    };
-  */
-
   # Let's Encrypt configuration.
   security.acme.certs."hydra.nixos.org" =
     { email = "edolstra@gmail.com";

diff --git a/delft/hydra.nix b/delft/hydra.nix
@@ -38,11 +38,11 @@ in
       server_store_uri = https://cache.nixos.org?local-nar-cache=${narCache}
       binary_cache_public_uri = https://cache.nixos.org
 
-      <hipchat>
-        jobs = (hydra|nixops):.*:.*
-        room = 182482
-        token = ${builtins.readFile ./hipchat-lb-token}
-      </hipchat>
+      #<hipchat>
+      #  jobs = (hydra|nixops):.*:.*
+      #  room = 182482
+      #  token = ${builtins.readFile ./hipchat-lb-token}
+      #</hipchat>
 
       <Plugin::Session>
         cache_size = 32m
@@ -51,7 +51,7 @@ in
       # patchelf:master:3
       xxx-jobset-repeats = nixos:reproducibility:1
 
-      nar_buffer_size = 6442450944
+      nar_buffer_size = ${let gb = 10; in toString (gb * 1024 * 1024 * 1024)}
 
       upload_logs_to_binary_cache = true
 
@@ -60,10 +60,9 @@ in
 
       log_prefix = https://nix-cache.s3.amazonaws.com/
 
-      evaluator_initial_heap_size = ${let gb = 11; in toString (gb * 1000 * 1000 * 1000)}
-      evaluator_max_heap_size     = ${let gb = 12; in toString (gb * 1000 * 1000 * 1000)}
+      evaluator_initial_heap_size = ${let gb = 20; in toString (gb * 1024 * 1024 * 1024)}
 
-      max_concurrent_evals = 1
+      max_concurrent_evals = 2
     '';
 
   systemd.tmpfiles.rules =
@@ -84,52 +83,47 @@ in
       TCPKeepAlive yes
 
       Host mac1-guest
-      Hostname 37.153.215.191
-      Port 6001
+      Hostname 10.254.2.1
+      Port 2200
       Compression yes
 
       Host mac2-guest
-      Hostname 37.153.215.191
-      Port 6002
+      Hostname 10.254.2.2
+      Port 2200
       Compression yes
 
       Host mac3-guest
-      Hostname 37.153.215.191
-      Port 6003
+      Hostname 10.254.2.3
+      Port 2200
       Compression yes
 
       Host mac4-guest
-      Hostname 37.153.215.191
-      Port 6004
+      Hostname 10.254.2.4
+      Port 2200
       Compression yes
 
       Host mac5-guest
-      Hostname 37.153.215.191
-      Port 6005
+      Hostname 10.254.2.5
+      Port 2200
       Compression yes
 
       Host mac6-guest
-      Hostname 37.153.215.191
-      Port 6006
+      Hostname 10.254.2.6
+      Port 2200
       Compression yes
 
       Host mac7-guest
-      Hostname 37.153.215.191
-      Port 6007
+      Hostname 10.254.2.7
+      Port 2200
       Compression yes
 
       Host mac8-guest
-      Hostname 37.153.215.191
-      Port 6008
+      Hostname 10.254.2.8
+      Port 2200
       Compression yes
 
       Host mac9-guest
-      Hostname 37.153.215.191
-      Port 6009
-      Compression yes
-
-      Host bigmac-guest
-      Hostname 147.75.64.174
+      Hostname 10.254.2.9
       Port 2200
       Compression yes
 
@@ -171,14 +165,12 @@ in
       { hostNames = [ "83.87.124.39" ]; publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVTkY4tQ6V29XTW1aKtoFJoF4uyaEy0fms3HqmI56av8UCg3MN5G6CL6EDIvbe46mBsI3++V3uGiOr0pLPbM9fkWC92LYGk5f7fNvCoy9bvuZy5bHwFQ5b5S9IJ1o3yDlCToc9CppmPVbFMMMLgKF06pQiGBeMCUG/VoCfiUBq+UgEGhAifWcuWIOGmdua6clljH5Dcc+7S0HTLoVtrxmPPXBVZUvW+lgAJTM6FXYIZiIqMSC2uZHGVstY87nPcZFXIbzhlYQqxx5H0um2bL3mbS7vdKhSsIWWaUZeck9ghNyUV1fVRLUhuXkQHe/8Z58cAhTv5dDd42YLB0fgjETV"; }
 
       # (for i in $(seq 6000 6010); do ssh-keyscan -t ssh-ed25519 -p $i 37.153.215.191 2> /dev/null; done) | sed -e 's/^/      { hostNames = [ "/' -e 's/ ssh/" ]; publicKey = "ssh/' -e 's/$/"; }/'; echo
-      { hostNames = [ "[37.153.215.191]:6002" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKJSLxndBO2SqMklTT4dWsv54CtKVMHDOhXl7v1tC38v"; }
-      { hostNames = [ "[37.153.215.191]:6003" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdzP9scGjJ31wwDJdcXcO4+wayvrHkFAefP/0I7UB0r"; }
-      { hostNames = [ "[37.153.215.191]:6004" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAaK44azT14aWHTE2fMYJFeJBTg1hQg2n2KAEgu8qlII"; }
-      { hostNames = [ "[37.153.215.191]:6005" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEgJ2Mh0vkrBcIBcWgFoEPsYG9dyPOAUgL0GOhhBoRcD"; }
-      { hostNames = [ "[37.153.215.191]:6006" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyjTqJsNioHUP0HMmGVfDpej9dCIQH0Bdn5aNbe8oYU"; }
-      { hostNames = [ "[37.153.215.191]:6007" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjy5mXHUHqGO+lzeyj09iuADIDtcQtnBtwiPCmV/UBq"; }
-      { hostNames = [ "[37.153.215.191]:6008" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHJF0dXSTvZwWtPjN/iUBp2A5pHCPuOUVOvBkjXw+Eu"; }
-      { hostNames = [ "[139.178.82.19]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIyfeCptEJWPP2nsC5jBZ62Z13+Vl1I/5fFQ9acTbQp3"; }
+      { hostNames = [ "[10.254.2.2]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKJSLxndBO2SqMklTT4dWsv54CtKVMHDOhXl7v1tC38v"; }
+      { hostNames = [ "[10.254.2.3]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdzP9scGjJ31wwDJdcXcO4+wayvrHkFAefP/0I7UB0r"; }
+      { hostNames = [ "[10.254.2.4]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAaK44azT14aWHTE2fMYJFeJBTg1hQg2n2KAEgu8qlII"; }
+      { hostNames = [ "[10.254.2.5]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEgJ2Mh0vkrBcIBcWgFoEPsYG9dyPOAUgL0GOhhBoRcD"; }
+      { hostNames = [ "[10.254.2.6]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyjTqJsNioHUP0HMmGVfDpej9dCIQH0Bdn5aNbe8oYU"; }
+      { hostNames = [ "[10.254.2.7]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjy5mXHUHqGO+lzeyj09iuADIDtcQtnBtwiPCmV/UBq"; }
 
       # bigmac
       { hostNames = [ "[147.75.64.174]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINM+o3+x2cncVNdRTVZTOeckSzheAJRzVkYdu7PsgDkq"; }

diff --git a/delft/network.nix b/delft/network.nix
@@ -8,6 +8,7 @@ let
       network = {
         interiorNetworkPrefix = "10.172.170"; #172="n", 170="x"
         externalInterface = "enp3s0f0";
+        sshInterface = "wg0";
       };
 
       guest = {
@@ -36,6 +37,7 @@ in {
   packet-t2-4 = { deployment.targetHost = "147.75.98.145"; imports = [ ./packet/packet-t2-4.nix ./build-machines-common.nix ]; };
   chef = import ./chef.nix;
   eris = import ./eris.nix;
+  ceres = import ./ceres.nix;
 
   mac1 = makeMac {
     ip = "10.254.2.1";

diff --git a/macs/guest/apply.sh b/macs/guest/apply.sh
@@ -49,7 +49,7 @@ echo "%admin ALL = NOPASSWD: ALL" | tee /etc/sudoers.d/passwordless
     export HOME=~root
     export ALLOW_PREEXISTING_INSTALLATION=1
     env
-    curl https://nixos.org/releases/nix/nix-2.1.3/install > ~nixos/install-nix
+    curl https://nixos.org/releases/nix/nix-2.2.2/install > ~nixos/install-nix
     chmod +rwx ~nixos/install-nix
     cat /dev/null | sudo -i -H -u nixos -- sh ~nixos/install-nix --daemon
 )
@@ -65,12 +65,17 @@ echo "%admin ALL = NOPASSWD: ALL" | tee /etc/sudoers.d/passwordless
     ls -la /private/var || true
     ls -la /private/var/run || true
     ln -s /private/var/run /run || true
-    nix-channel --add https://nixos.org/channels/nixos-19.03 nixpkgs
+
+    # todo: clean up this channel business, which is complicated because
+    # channels on darwin are a bit ill defined and have a very bad UX.
+    # If me, Graham, the author of the multi-user darwin installer can't
+    # even figure this out, how can I possibly expect anybody else to know.
     nix-channel --add https://github.com/LnL7/nix-darwin/archive/master.tar.gz darwin
+    nix-channel --add https://nixos.org/channels/nixpkgs-19.03-darwin nixpkgs
     nix-channel --update
 
-    sudo -i -H -u nixos -- nix-channel --add https://nixos.org/channels/nixos-19.03 nixpkgs
     sudo -i -H -u nixos -- nix-channel --add https://github.com/LnL7/nix-darwin/archive/master.tar.gz darwin
+    sudo -i -H -u nixos -- nix-channel --add https://nixos.org/channels/nixpkgs-19.03-darwin nixpkgs
     sudo -i -H -u nixos -- nix-channel --update
 
     export NIX_PATH=$NIX_PATH:darwin=https://github.com/LnL7/nix-darwin/archive/master.tar.gz
@@ -91,5 +96,12 @@ echo "%admin ALL = NOPASSWD: ALL" | tee /etc/sudoers.d/passwordless
     ln -s /etc/static/bashrc /etc/bashrc
     . /etc/static/bashrc
     cat /Volumes/CONFIG/darwin-configuration.nix | sudo -u nixos -- tee ~nixos/.nixpkgs/darwin-configuration.nix
+
+    while ! sudo -i -H -u nixos -- nix ping-store; do
+        cat /var/log/nix-daemon.log
+        sleep 1
+    done
+
     sudo -i -H -u nixos -- darwin-rebuild switch
 )
+
diff --git a/macs/host/default.nix b/macs/host/default.nix
@@ -44,6 +44,13 @@ in {
           '';
         };
 
+	sshInterface = mkOption {
+          type = types.str;
+          description = ''
+            Public network interface to receive SSH connections on.
+          '';
+        };
+
         interiorNetworkPrefix = mkOption {
           type = types.str;
           description = ''