NixOS · Apr 26, 2019 · Apr 27, 2019 · Apr 27, 2019 · Apr 27, 2019 · May 13, 2019
Showing with 195 additions and 0 deletions.

+1 −0 delft/chef.nix

+44 −0 delft/packet-importer.nix

+33 −0 hydra-packet-importer/config-example.json

+16 −0 hydra-packet-importer/default.nix

+100 −0 hydra-packet-importer/import.py

+1 −0 hydra-packet-importer/shell.nix
diff --git a/delft/chef.nix b/delft/chef.nix
@@ -9,6 +9,7 @@
       ./fstrim.nix
       ./provisioner.nix
       ../modules/wireguard.nix
+      ./packet-importer.nix
     ];
 
   deployment.targetEnv = "hetzner";

diff --git a/delft/packet-importer.nix b/delft/packet-importer.nix
@@ -0,0 +1,44 @@
+{ config, lib, pkgs, ... }:
+let
+  importer = pkgs.callPackage ../hydra-packet-importer { };
+in
+{
+  deployment.keys."hydra-packet-import.json" = {
+    keyFile = ../hydra-packet-import.json;
+    user = "hydra-packet";
+  };
+
+  users.extraUsers.hydra-packet = {
+    description = "Hydra Packet Machine Importer";
+    group = "hydra";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/hydra/packet-import 0755 hydra-packet hydra -"
+    "f /var/lib/hydra/packet-import/machines 0644 hydra-packet hydra -"
+  ];
+
+  services.hydra-dev.buildMachinesFiles = [
+    "/var/lib/hydra/packet-import/machines"
+  ];
+
+  systemd.services.hydra-packet-import = {
+    path = with pkgs; [ openssh moreutils ];
+    script = "${importer}/bin/hydra-packet-importer /run/keys/hydra-packet-import.json | sponge /var/lib/hydra/packet-import/machines";
+    serviceConfig = {
+      User = "hydra-packet";
+      Group = "keys";
+      SupplementaryGroups = [ "hydra" "keys" ];
+      Type = "oneshot";
+    };
+  };
+
+  systemd.timers.hydra-packet-import = {
+    enable = true;
+    description = "Update the list of Hydra machines from Packet.net";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*:0/5";
+    };
+  };
+}
diff --git a/hydra-packet-importer/config-example.json b/hydra-packet-importer/config-example.json
@@ -0,0 +1,33 @@
+{
+    "token": "You read only API token",
+    "project_id": "Your project's unique ID",
+    "skip_tags": [ "tags which will", "exclude a server from", "being imported" ],
+    "plans": {
+        "c2.large.arm": {
+            "user": "root",
+            "system_types": ["aarch64-linux"],
+            "ssh_key": "/path/to/hydras-ssh-key",
+            "max_jobs": 15,
+            "speed_factor": 1,
+            "features": ["kvm","nixos-test","big-parallel"],
+            "mandatory_features": []
+        },
+        "c2.medium.x86": {
+            "user": "root",
+            "system_types": ["x86_64-linux", "i686-linux"],
+            "ssh_key": "/var/lib/hydra/queue-runner/.ssh/id_buildfarm_rsa",
+            "max_jobs": 18,
+            "speed_factor": 1,
+            "features": [ "kvm","nixos-test","big-parallel" ]
+        }
+    },
+    "name_overrides": {
+        "machines-with-this-name-override-defaults-in-plan": {
+            "max_jobs": 20,
+            "system_types": [ "x86_64-linux" ],
+            "max_jobs": 1,
+            "speed_factor": 100,
+            "features": [ "kvm","nixos-test","big-parallel" ]
+        }
+    }
+}
diff --git a/hydra-packet-importer/default.nix b/hydra-packet-importer/default.nix
@@ -0,0 +1,16 @@
+{ python3 }:
+python3.pkgs.buildPythonApplication {
+  name = "hydra-packet-importer";
+  src = ./.;
+
+  format = "other";
+
+  propagatedBuildInputs = [
+    python3.pkgs.packet-python
+  ];
+
+  installPhase = ''
+    mkdir -p $out/bin
+    mv import.py $out/bin/hydra-packet-importer
+  '';
+}
diff --git a/hydra-packet-importer/import.py b/hydra-packet-importer/import.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import json
+import packet
+import base64
+from pprint import pprint
+import subprocess
+import sys
+
+def debug(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def get_devices(manager):
+    devices = []
+
+    page = 'projects/{}/devices?page={}'.format(config['project_id'], 1)
+    while page is not None:
+        debug(page)
+        data = manager.call_api(page)
+        if data['meta']['next'] is None:
+            page = None
+        else:
+            page = data['meta']['next']['href']
+
+        for device in data['devices']:
+            if device['state'] != 'active':
+                continue
+            if 'spot_instance' not in device:
+                continue
+            if device['spot_instance'] != True:
+                continue
+
+            if not set(device['tags']).isdisjoint(config['skip_tags']):
+                continue
+
+            devices.append({
+                "hostname": device['hostname'],
+                "address": "{}.packethost.net".format(device['short_id']),
+                "type": device['plan']['name']
+            })
+
+    return devices
+
+def main(config):
+    rows = []
+    manager = packet.Manager(auth_token=config['token'])
+    found = 0
+    for device in get_devices(manager):
+        found += 1
+        debug("# {} ({})".format(device['hostname'], device['address']))
+        if device['type'] not in config['plans']:
+            debug("# Skipping {} (type {}) as it has no configured plan".format(
+                device['hostname'],
+                device['type'])
+            )
+            continue
+
+        default_stats = config['plans'][device['type']]
+        if device['hostname'] in config['name_overrides']:
+            specific_stats = config['name_overrides'][device['hostname']]
+        else:
+            specific_stats = {}
+
+        lookup = lambda key: specific_stats.get(key, device.get(key, default_stats.get(key)))
+        lookup_default = lambda key, default: default if not lookup(key) else lookup(key)
+
+        r = subprocess.check_output([
+                                     "ssh-keyscan",
+                                     "-4", # force IPv4
+                                     "-T", "5", # Timeout 5 seconds
+                                     "-t", "ed25519", # Only ed25519 keys
+                                     lookup("address")
+                                     ]).decode("utf-8")
+
+        elems = r.split(" ", 1)
+        if len(elems) != 2:
+            debug("# Skipped due keyscan failed to split on ' '")
+            continue
+        key = elems[1]
+
+        # root@address system,list /var/lib/ssh.key maxJobs speedFactor feature,list mandatory,features public-host-key
+        rows.append(" ".join([
+               "{user}@{host}".format(user=lookup("user"),host=lookup("address")),
+               ",".join(lookup("system_types")),
+               str(lookup("ssh_key")),
+               str(lookup("max_jobs")),
+               str(lookup("speed_factor")),
+               ",".join(lookup_default("features", ["-"])),
+               ",".join(lookup_default("mandatory_features", ["-"])),
+               base64.b64encode(key.encode()).decode("utf-8")
+        ]))
+
+    debug("# {} / {}".format(len(rows),found))
+    print("\n".join(rows))
+
+if __name__ == "__main__":
+    with open(sys.argv[1]) as config_file:
+        config = json.load(config_file)
+        main(config)
diff --git a/hydra-packet-importer/shell.nix b/hydra-packet-importer/shell.nix
@@ -0,0 +1 @@
+(import <nixpkgs> {}).callPackage ./default.nix {}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		(import <nixpkgs> {}).callPackage ./default.nix {}