NixOS · Oct 4, 2019 · Oct 4, 2019 · Oct 4, 2019 · Oct 4, 2019 · Oct 4, 2019
diff --git a/bastion/network.nix b/bastion/network.nix
@@ -117,7 +117,7 @@ in
 
       imports =
         [ ../modules/common.nix
-	   ../modules/wireguard.nix
+          ../modules/wireguard.nix
           ../modules/tarball-mirror.nix
           ../modules/hydra-mirror.nix
         ];
@@ -147,7 +147,7 @@ in
       # Temporary hack until we have proper users/roles.
       services.openssh.extraConfig =
         ''
-          AcceptEnv AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY FASTLY_API_KEY
+          AcceptEnv AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY FASTLY_API_KEY GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL
         '';
 
       fileSystems."/scratch" =

diff --git a/delft/ceres.nix b/delft/ceres.nix
@@ -6,7 +6,6 @@
       ./hydra.nix
       ./hydra-proxy.nix
       ./fstrim.nix
-      ../modules/wireguard.nix
       ./packet-importer.nix
     ];
 
@@ -50,7 +49,7 @@
   services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
 
   nix.gc.automatic = true;
-  nix.gc.options = ''--max-freed "$((100 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
+  nix.gc.options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
   nix.gc.dates = "03,09,15,21:15";
 
   nix.extraOptions = "gc-keep-outputs = false";

diff --git a/delft/chef.nix b/delft/chef.nix
@@ -5,7 +5,6 @@
     [ ./common.nix
       ./datadog.nix
       ./fstrim.nix
-      ../modules/wireguard.nix
     ];
 
   deployment.targetEnv = "hetzner";

diff --git a/delft/common.nix b/delft/common.nix
@@ -12,7 +12,7 @@ with lib;
       rev = "531a2338101f0b6db2d9d512c3f98145f7b75397";
        }) + "/module.nix")
       ../modules/common.nix
-      ./prometheus/node.nix
+      ../modules/prometheus
     ];
 
   system.stateVersion = "14.12";

diff --git a/delft/eris.nix b/delft/eris.nix
@@ -1,28 +1,31 @@
-{ nodes, config, lib, pkgs, ... }:
+{ resources, config, lib, pkgs, ... }:
+let
+  inherit (lib) filterAttrs flip mapAttrsToList;
 
-{
+  macs = filterAttrs (_: v: (v.macosGuest or {}).enable or false) resources.machines;
+in {
+  imports =  [
+    ../modules/prometheus
+  ];
   deployment.targetEnv = "hetzner";
   deployment.hetzner.mainIPv4 = "138.201.32.77";
 
+
   networking.extraHosts = ''
-    46.4.67.10 chef
-    147.75.198.47 packet-epyc-1
-    147.75.98.145 packet-t2-4
-    147.75.79.198 packet-t2a-2
-    147.75.198.170 packet-t2a-3
-    147.75.74.238 packet-t2a5-qc-centriq-1
-    147.75.107.178 packet-t2a6-ampere-1
-    139.178.82.19 bigmac-host
-    139.178.82.19 bigmac-guest
-
-
-    '' + (let
-        nums = lib.lists.range 1 9;
-        name = num: ''
-          37.153.215.191 mac${toString num}-host
-          37.153.215.191 mac${toString num}-guest
-        '';
-      in lib.strings.concatMapStrings name nums);
+    10.254.1.1 bastion
+    10.254.1.2 chef
+    10.254.1.3 ceres
+
+    10.254.1.5 ike
+    10.254.1.6 hydra
+    10.254.1.7 lucifer
+    10.254.1.8 wendy
+
+    10.254.3.1 webserver
+
+    '' + (toString (flip mapAttrsToList macs (machine: v: ''
+    ${v.deployment.targetHost} ${machine}
+    '')));
 
   networking.firewall.allowedTCPPorts = [
     443 80 # nginx
@@ -35,7 +38,12 @@
     virtualHosts."status.nixos.org" = {
       enableACME = true;
       forceSSL = true;
-      root = pkgs.writeTextDir "index.html" "check out /grafana and /prometheus";
+      root = pkgs.writeTextDir "index.html" ''
+        <ul>
+          <li><a href="/grafana">Grafana</a></li>
+          <li><a href="/prometheus">Prometheus</a></li>
+        </ul>
+      '';
       locations."/grafana/".proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}/";
       locations."/prometheus".proxyPass = "http://${config.services.prometheus.listenAddress}";
     };
@@ -54,41 +62,79 @@
         job_name = "node";
         static_configs = [
           {
-            targets = [
-              "chef:9100"
-            ];
-            labels.role = "hydra";
+            targets = flip mapAttrsToList resources.machines (machine: v: "${v.networking.hostName}:9100");
+            labels.role = "unknown";
           }
           {
             targets = [
-              "packet-epyc-1:9100" "packet-t2-4:9100"
-              "packet-t2a-2:9100" "packet-t2a-3:9100"
-              "packet-t2a5-qc-centriq-1:9100" "packet-t2a6-ampere-1:9100"
               "chef:9100"
             ];
             labels.role = "builder";
           }
           {
-            targets = builtins.map (n: "mac${toString n}-host:6010") (lib.lists.range 1 9);
-            labels.mac = "host";
-            labels.role = "macos-hypervisor";
+            targets = [
+              "webserver:9100"
+            ];
+            labels.role = "webserver";
           }
           {
-            targets = builtins.map (n: "mac${toString n}-guest:6010") (lib.lists.range 1 9);
+            targets = flip mapAttrsToList macs (machine: v: "${machine}:9101");
             labels.mac = "guest";
             labels.role = "builder";
           }
           {
-            targets = [ "bigmac-host:9100" ];
+            targets = flip mapAttrsToList macs (machine: v: "${machine}:9100");
             labels.mac = "host";
-            labels.role = "macos-hypervisor";
+            labels.role = "builder";
           }
+        ];
+      }
+      {
+        job_name = "nixos";
+        static_configs = [
           {
-            targets = [ "bigmac-guest:9101" ];
-            labels.mac = "guest";
-            labels.role = "builder";
+            targets = flip mapAttrsToList resources.machines (machine: v: "${v.networking.hostName}:9300");
+          }
+          {
+            targets = [
+              "webserver:9300"
+            ];
+            labels.role = "webserver";
+          }
+        ];
+      }
+      {
+        job_name = "packet_nodes";
+        file_sd_configs = [
+          {
+            files = [ "/var/lib/packet-sd/packet-sd.json" ];
+            refresh_interval = "30s";
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__meta_packet_short_id" ];
+            target_label = "__address__";
+            replacement = "\${1}.packethost.net:9100";
+            action = "replace";
+          }
+          {
+            source_labels = [ "__meta_packet_facility" ];
+            target_label = "facility";
+          }
+          {
+            source_labels = [ "__meta_packet_short_id" ];
+            target_label = "instance";
+            replacement = "\${1}.packethost.net";
+            action = "replace";
+          }
+          {
+            source_labels = [ "__meta_packet_tags" ];
+            target_label = "role";
+            regex = ".*hydra.*";
+            replacement = "builder";
+            action = "replace";
           }
-
         ];
       }
 
@@ -103,6 +149,18 @@
           }
         ];
       }
+
+      {
+        job_name = "prometheus-packet-sd";
+        metrics_path = "/metrics";
+        static_configs = [
+          {
+            targets = [
+              "127.0.0.1:9465"
+            ];
+          }
+        ];
+      }
     ];
   };
 
@@ -132,4 +190,34 @@
       '';
     };
   };
+
+  deployment.keys."packet-sd-env" = {
+    keyFile = ../prometheus-packet-service-discovery;
+    user = "packet-sd";
+  };
+
+  users.extraUsers.packet-sd = {
+    description = "Prometheus Packet Service Discovery";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/packet-sd 0755 packet-sd - -"
+    "f /var/lib/packet-sd/packet-sd.json 0644 packet-sd - -"
+  ];
+
+  systemd.services.prometheus-packet-sd = let
+    sd = pkgs.callPackage ./prometheus/packet-sd.nix {};
+  in {
+    wantedBy = [ "multi-user.target" "prometheus.service" ];
+    after = [ "network.target" ];
+
+    serviceConfig = {
+      User = "packet-sd";
+      Group = "keys";
+      ExecStart = "${sd}/bin/prometheus-packet-sd --output.file=/var/lib/packet-sd/packet-sd.json";
+      EnvironmentFile = "/run/keys/packet-sd-env";
+      Restart = "always";
+      RestartSec = "60s";
+    };
+  };
 }
diff --git a/delft/hydra.nix b/delft/hydra.nix
@@ -134,47 +134,24 @@ in
       Host macstadium2
       Hostname 208.78.106.252
       Compression yes
-
-      Host packet-t2-4
-      Hostname 147.75.98.145
-      Compression yes
-
-      Host packet-t2a-2
-      Hostname 147.75.79.198
-      Compression yes
-
-      Host packet-t2a-3
-      Hostname 147.75.198.170
-      Compression yes
-
-      Host packet-epyc-1
-      Hostname 147.75.198.47
-      Compression yes
-
-      Host packet-t2a5-qc-centriq-1
-      Hostname 147.75.74.238
-      Compression yes
-
-      Host packet-c2-large-arm
-      Hostname 147.75.107.234
-      Compression yes
   '';
 
   services.openssh.knownHosts =
     [
       { hostNames = [ "83.87.124.39" ]; publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVTkY4tQ6V29XTW1aKtoFJoF4uyaEy0fms3HqmI56av8UCg3MN5G6CL6EDIvbe46mBsI3++V3uGiOr0pLPbM9fkWC92LYGk5f7fNvCoy9bvuZy5bHwFQ5b5S9IJ1o3yDlCToc9CppmPVbFMMMLgKF06pQiGBeMCUG/VoCfiUBq+UgEGhAifWcuWIOGmdua6clljH5Dcc+7S0HTLoVtrxmPPXBVZUvW+lgAJTM6FXYIZiIqMSC2uZHGVstY87nPcZFXIbzhlYQqxx5H0um2bL3mbS7vdKhSsIWWaUZeck9ghNyUV1fVRLUhuXkQHe/8Z58cAhTv5dDd42YLB0fgjETV"; }
 
-      # (for i in $(seq 6000 6010); do ssh-keyscan -t ssh-ed25519 -p $i 37.153.215.191 2> /dev/null; done) | sed -e 's/^/      { hostNames = [ "/' -e 's/ ssh/" ]; publicKey = "ssh/' -e 's/$/"; }/'; echo
-      { hostNames = [ "[10.254.2.2]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKJSLxndBO2SqMklTT4dWsv54CtKVMHDOhXl7v1tC38v"; }
-      { hostNames = [ "[10.254.2.3]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdzP9scGjJ31wwDJdcXcO4+wayvrHkFAefP/0I7UB0r"; }
-      { hostNames = [ "[10.254.2.4]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAaK44azT14aWHTE2fMYJFeJBTg1hQg2n2KAEgu8qlII"; }
-      { hostNames = [ "[10.254.2.5]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEgJ2Mh0vkrBcIBcWgFoEPsYG9dyPOAUgL0GOhhBoRcD"; }
-      { hostNames = [ "[10.254.2.6]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyjTqJsNioHUP0HMmGVfDpej9dCIQH0Bdn5aNbe8oYU"; }
-      { hostNames = [ "[10.254.2.7]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjy5mXHUHqGO+lzeyj09iuADIDtcQtnBtwiPCmV/UBq"; }
-      { hostNames = [ "[10.254.2.8]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHJF0dXSTvZwWtPjN/iUBp2A5pHCPuOUVOvBkjXw+Eu"; }
-
-      # bigmac
-      { hostNames = [ "[147.75.64.174]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINM+o3+x2cncVNdRTVZTOeckSzheAJRzVkYdu7PsgDkq"; }
+      # (for i in 10.254.2.{1,2,3,4,5,6,7,8,9}; do ssh-keyscan -t ssh-ed25519 -p 2200 $i 2> /dev/null; done) | sed -e 's/^/      { hostNames = [ "/' -e 's/ ssh/" ]; publicKey = "ssh/' -e 's/$/"; }/'; echo
+
+      { hostNames = [ "[10.254.2.1]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHAPtrCAuysFZCzDXlQMrxvybhMMBjpPE4c1vUoUyvvm"; }
+      { hostNames = [ "[10.254.2.2]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICIzkQ2sjmphJL2oo1FSA/3F7/G+YTraWuPYUXBdZJ/t"; }
+      { hostNames = [ "[10.254.2.3]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDnJQyL2LlWjIE+4wGBZyTapXlCgwTZ+uBh7eoaGPfL"; }
+      { hostNames = [ "[10.254.2.4]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq44jmyZD6fQY+WLPH3Dx9mQXzCK7ZBpfmYjATVvT7T"; }
+      { hostNames = [ "[10.254.2.5]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEKH3SXo9u27y105FLNz1PxrDMBZ0gAsBsC9t2ErHbx4"; }
+      { hostNames = [ "[10.254.2.6]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBNU5/bIy24ea7twM6j7QAKs1KWJADYNfov94N9YjlVz"; }
+      { hostNames = [ "[10.254.2.7]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN8AwqnYcZhj1jINB5HMAT+VBl+rPH9TCeHPtZMwMIUJ"; }
+      { hostNames = [ "[10.254.2.8]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAYH18PbDKKNmRaRYdMbbSqJSC+g5yB83LLSNemxhoCE"; }
+      { hostNames = [ "[10.254.2.9]:2200" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP4oOQBk3nRMKcPsDAL54jMAfSy9fwCyfH1qWwp1jwQt"; }
+
 
       { hostNames = [ "208.78.106.251" ]; publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKrwg2592DfDUG1U0LZxJcBaT35YfEsuKo4helEAzeoujvzOo4DIaBrTCX7+LxcYZlGoi4WvsnwxUG11GY12l2A="; }
       { hostNames = [ "208.78.106.252" ]; publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLV1p6f6Rk3rKwNJbcqvG68wjfT3wPcJfChc1LFU9A3tTFslUDr47FHLmT+FTr+ChkoqD6Gsl+jtSnvkYnTlpGY="; }
@@ -186,24 +163,6 @@ in
       { hostNames = [ "stan.ewi.tudelft.nl" "131.180.119.74" ]; publicKey = "ssh-dss AAAAB3NzaC1kc3MAAACBAL0SlYpGjDjPKrLIwoltYHHTYo/d6Ct2FQZKh4ltKOszWPYYAbs/YNSm2eFkvj0CGc3aastFuebz6+pRfvGMvqi4q6IoHwVvOkbWMadyuqrWIO+Z1YemZP/GAG69pLy+UyoydiSI83ycwPe4YARAU/cpBMNKJZbSxyrO80XatmtRAAAAFQC291WK+9M8+zI4KAtk6EqX0vqQ1QAAAIBd1YgRfdfRdu60BpR+3/YMbSYZMjRLFPyoSgmEQR2TtKfqsuKTsTREzB20iMgFlhEWb6C4r5y6jYDU85OOnvpf7zne22j6bKFDIiAbgsjUFHK1EB7+TBltf5yqq0FyNOy/PnLqVzOeGaUeCOc3Ris71Lxkm60oVF4mjut2d2UJ6AAAAIByuCH1bIIRb4za4yiiFQUz2CBX1XHhBn/h/LhNMLuyCTciG6tkppGBAgq5rWrNhjaEc7dIFgZR+E1wE5PQzWG/TBiXctwCOqOOErDB5b95jO2EntIhi8x5PO9Ef6jgis4QRsBIZiENDDeQHxFHCv4q+10TpyV+625O8TXkkcxl0g=="; }
       { hostNames = [ "wendy.ewi.tudelft.nl" "131.180.119.77" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBQOW9V+azndhOiWApltwo7Khnc5/MNEAW8Rf5J/NyBx"; }
 
-      # packet-t2-4
-      { hostNames = [ "147.75.98.145" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIABlJeKJ2B5mgK8TujC3FZ7geAOzQFUDJpay6a8Utdg6"; }
-
-      # packet-t2a-2
-      { hostNames = [ "147.75.79.198" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDCo+z5d8C6SpCyvC8KAPMAcMEtd5J74tRsk+7sm2KgD"; }
-
-      # packet-t2a-3
-      { hostNames = [ "147.75.198.170" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPieDw5aeaN88HMIye2t4tL6DBqgJwQbXH0PzPd0HOVl"; }
-
-      # packet-t2a5-qc-centriq-1
-      { hostNames = [ "147.75.74.238" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJTGvykBvR6l4d0ssReNLMV2qVV+TKNUrU22OOix//qE"; }
-
-      # packet-c2-large-arm
-      { hostNames = [ "147.75.107.234" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHsY9skBvQx3mMFrMyR2zwK50OMEu5Xzw4tMLBH4Nzum"; }
-
-      # packet-epyc-1
-      { hostNames = [ "147.75.198.47" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAt76Jw/xXqvYcrNZSa+BEfKfLCFyvY50LZJuGBWtJwx"; }
-
       { hostNames = [ "t2m.cunat.cz" ]; publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9351NRVeQYvNV1bBbC5MX0iSmrXhVcBYMcn6AMo11U2zlOYRqBPzGLPjz9u31t4FxHNovxCrkFTqJY9zbsmTs="; }
       { hostNames = [ "t2a.cunat.cz" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIu3itg4hn5e4KrnyoreAUN3RIbAcvqc7yWx5i6EWqAu"; }
       { hostNames = [ "t4a.cunat.cz" ]; publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILXgOInIZ+1DdWDeXBO1ILtlM53ZrYOtrBlfZ7dIzCyu"; }