NixOS · Nov 6, 2019 · Nov 7, 2019 · Nov 7, 2019 · Nov 7, 2019 · Nov 7, 2019
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
@@ -48,7 +48,6 @@ in
   clickhouse = handleTest ./clickhouse.nix {};
   cloud-init = handleTest ./cloud-init.nix {};
   codimd = handleTest ./codimd.nix {};
-  colord = handleTest ./colord.nix {};
   containers-bridge = handleTest ./containers-bridge.nix {};
   containers-ephemeral = handleTest ./containers-ephemeral.nix {};
   containers-extra_veth = handleTest ./containers-extra_veth.nix {};
@@ -88,27 +87,21 @@ in
   firewall = handleTest ./firewall.nix {};
   fish = handleTest ./fish.nix {};
   flannel = handleTestOn ["x86_64-linux"] ./flannel.nix {};
-  flatpak = handleTest ./flatpak.nix {};
-  flatpak-builder = handleTest ./flatpak-builder.nix {};
   fluentd = handleTest ./fluentd.nix {};
   fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {};
   fsck = handleTest ./fsck.nix {};
-  fwupd = handleTestOn ["x86_64-linux"] ./fwupd.nix {}; # libsmbios is unsupported on aarch64
-  gdk-pixbuf = handleTest ./gdk-pixbuf.nix {};
   gotify-server = handleTest ./gotify-server.nix {};
   gitea = handleTest ./gitea.nix {};
   gitlab = handleTest ./gitlab.nix {};
   gitolite = handleTest ./gitolite.nix {};
   gjs = handleTest ./gjs.nix {};
-  glib-networking = handleTest ./glib-networking.nix {};
   glusterfs = handleTest ./glusterfs.nix {};
   gnome3-xorg = handleTest ./gnome3-xorg.nix {};
   gnome3 = handleTest ./gnome3.nix {};
-  gnome-photos = handleTest ./gnome-photos.nix {};
+  installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {});
   gocd-agent = handleTest ./gocd-agent.nix {};
   gocd-server = handleTest ./gocd-server.nix {};
   google-oslogin = handleTest ./google-oslogin {};
-  graphene = handleTest ./graphene.nix {};
   grafana = handleTest ./grafana.nix {};
   graphite = handleTest ./graphite.nix {};
   graylog = handleTest ./graylog.nix {};
@@ -216,7 +209,6 @@ in
   os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {};
   osquery = handleTest ./osquery.nix {};
   osrm-backend = handleTest ./osrm-backend.nix {};
-  ostree = handleTest ./ostree.nix {};
   overlayfs = handleTest ./overlayfs.nix {};
   packagekit = handleTest ./packagekit.nix {};
   pam-oath-login = handleTest ./pam-oath-login.nix {};
@@ -291,7 +283,6 @@ in
   wireguard-generated = handleTest ./wireguard/generated.nix {};
   wordpress = handleTest ./wordpress.nix {};
   xautolock = handleTest ./xautolock.nix {};
-  xdg-desktop-portal = handleTest ./xdg-desktop-portal.nix {};
   xfce = handleTest ./xfce.nix {};
   xfce4-14 = handleTest ./xfce4-14.nix {};
   xmonad = handleTest ./xmonad.nix {};

diff --git a/nixos/tests/colord.nix b/nixos/tests/colord.nix
diff --git a/nixos/tests/flatpak-builder.nix b/nixos/tests/flatpak-builder.nix
diff --git a/nixos/tests/flatpak.nix b/nixos/tests/flatpak.nix
diff --git a/nixos/tests/fontconfig-default-fonts.nix b/nixos/tests/fontconfig-default-fonts.nix
@@ -1,7 +1,12 @@
-import ./make-test.nix ({ lib, ... }:
+import ./make-test-python.nix ({ lib, ... }:
 {
   name = "fontconfig-default-fonts";
 
+  meta.maintainers = with lib.maintainers; [
+    jtojnar
+    worldofpeace
+  ];
+
   machine = { config, pkgs, ... }: {
     fonts.enableDefaultFonts = true; # Background fonts
     fonts.fonts = with pkgs; [
@@ -20,9 +25,9 @@ import ./make-test.nix ({ lib, ... }:
   };
 
   testScript = ''
-    $machine->succeed("fc-match serif | grep '\"Gentium Plus\"'");
-    $machine->succeed("fc-match sans-serif | grep '\"Cantarell\"'");
-    $machine->succeed("fc-match monospace | grep '\"Source Code Pro\"'");
-    $machine->succeed("fc-match emoji | grep '\"Twitter Color Emoji\"'");
+    machine.succeed("fc-match serif | grep '\"Gentium Plus\"'")
+    machine.succeed("fc-match sans-serif | grep '\"Cantarell\"'")
+    machine.succeed("fc-match monospace | grep '\"Source Code Pro\"'")
+    machine.succeed("fc-match emoji | grep '\"Twitter Color Emoji\"'")
   '';
 })
diff --git a/nixos/tests/fwupd.nix b/nixos/tests/fwupd.nix
diff --git a/nixos/tests/gdk-pixbuf.nix b/nixos/tests/gdk-pixbuf.nix
diff --git a/nixos/tests/gjs.nix b/nixos/tests/gjs.nix
diff --git a/nixos/tests/glib-networking.nix b/nixos/tests/glib-networking.nix
diff --git a/nixos/tests/gnome-photos.nix b/nixos/tests/gnome-photos.nix
diff --git a/nixos/tests/graphene.nix b/nixos/tests/graphene.nix
diff --git a/nixos/tests/installed-tests/colord.nix b/nixos/tests/installed-tests/colord.nix
@@ -0,0 +1,5 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.colord;
+}
diff --git a/nixos/tests/installed-tests/default.nix b/nixos/tests/installed-tests/default.nix
@@ -0,0 +1,77 @@
+# NixOS tests for gnome-desktop-testing-runner using software
+# See https://wiki.gnome.org/Initiatives/GnomeGoals/InstalledTests
+
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; }
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+let
+
+  callInstalledTest = pkgs.newScope { inherit makeInstalledTest; };
+
+  makeInstalledTest =
+    { # Package to test. Needs to have an installedTests output
+      tested
+
+      # Config to inject into machine
+    , testConfig ? {}
+
+      # Test script snippet to inject before gnome-desktop-testing-runner begins.
+      # This is useful for extra setup the environment may need before the runner begins.
+    , preTestScript ? ""
+
+      # Does test need X11?
+    , withX11 ? false
+
+      # Extra flags to pass to gnome-desktop-testing-runner.
+    , testRunnerFlags ? ""
+    }:
+    makeTest rec {
+      name = tested.name;
+
+      meta = {
+        maintainers = tested.meta.maintainers;
+      };
+
+      machine = { ... }: {
+        imports = [
+          testConfig
+        ] ++ optional withX11 ../common/x11.nix;
+
+        environment.systemPackages = with pkgs; [ gnome-desktop-testing ];
+
+      };
+
+      testScript =
+        optionalString withX11 ''
+          machine.wait_for_x()
+        '' +
+        optionalString (preTestScript != "") ''
+          ${preTestScript}
+        '' +
+        ''
+          machine.succeed(
+              "gnome-desktop-testing-runner ${testRunnerFlags} -d '${tested.installedTests}/share'"
+          )
+        '';
+    };
+
+in
+
+{
+  colord = callInstalledTest ./colord.nix {};
+  flatpak = callInstalledTest ./flatpak.nix {};
+  flatpak-builder = callInstalledTest ./flatpak-builder.nix {};
+  fwupd = callInstalledTest ./fwupd.nix {};
+  gdk-pixbuf = callInstalledTest ./gdk-pixbuf.nix {};
+  gjs = callInstalledTest ./gjs.nix {};
+  glib-networking = callInstalledTest ./glib-networking.nix {};
+  gnome-photos = callInstalledTest ./gnome-photos.nix {};
+  graphene = callInstalledTest ./graphene.nix {};
+  ostree = callInstalledTest ./ostree.nix {};
+  xdg-desktop-portal = callInstalledTest ./xdg-desktop-portal.nix {};
+}
diff --git a/nixos/tests/installed-tests/flatpak-builder.nix b/nixos/tests/installed-tests/flatpak-builder.nix
@@ -0,0 +1,14 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.flatpak-builder;
+
+  testConfig = {
+    services.flatpak.enable = true;
+    xdg.portal.enable = true;
+    environment.systemPackages = with pkgs; [ flatpak-builder ] ++ flatpak-builder.installedTestsDependencies;
+    virtualisation.diskSize = 2048;
+  };
+
+  testRunnerFlags = "--timeout 3600";
+}
diff --git a/nixos/tests/installed-tests/flatpak.nix b/nixos/tests/installed-tests/flatpak.nix
@@ -0,0 +1,19 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.flatpak;
+  withX11 = true;
+
+  testConfig = {
+    services.xserver.desktopManager.gnome3.enable = true; # TODO: figure out minimal environment where the tests work
+    # common/x11.nix enables the auto display manager (lightdm)
+    services.xserver.displayManager.gdm.enable = false;
+    services.gnome3.core-utilities.enable = false;
+    services.flatpak.enable = true;
+    environment.systemPackages = with pkgs; [ gnupg ostree python2 ];
+    virtualisation.memorySize = 2047;
+    virtualisation.diskSize = 1024;
+  };
+
+  testRunnerFlags = "--timeout 3600";
+}
diff --git a/nixos/tests/installed-tests/fwupd.nix b/nixos/tests/installed-tests/fwupd.nix
@@ -0,0 +1,12 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.fwupd;
+
+  testConfig = {
+    services.fwupd.enable = true;
+    services.fwupd.blacklistPlugins = []; # don't blacklist test plugin
+    services.fwupd.enableTestRemote = true;
+    virtualisation.memorySize = 768;
+  };
+}
diff --git a/nixos/tests/installed-tests/gdk-pixbuf.nix b/nixos/tests/installed-tests/gdk-pixbuf.nix
@@ -0,0 +1,13 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.gdk-pixbuf;
+
+  testConfig = {
+    # Tests allocate a lot of memory trying to exploit a CVE
+    # but qemu-system-i386 has a 2047M memory limit
+    virtualisation.memorySize = if pkgs.stdenv.isi686 then 2047 else 4096;
+  };
+
+  testRunnerFlags = "--timeout 1800";
+}
diff --git a/nixos/tests/installed-tests/gjs.nix b/nixos/tests/installed-tests/gjs.nix
@@ -0,0 +1,6 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.gjs;
+  withX11 = true;
+}
diff --git a/nixos/tests/installed-tests/glib-networking.nix b/nixos/tests/installed-tests/glib-networking.nix
@@ -0,0 +1,5 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.glib-networking;
+}
diff --git a/nixos/tests/installed-tests/gnome-photos.nix b/nixos/tests/installed-tests/gnome-photos.nix
@@ -0,0 +1,35 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.gnome-photos;
+
+  withX11 = true;
+
+  testConfig = {
+    programs.dconf.enable = true;
+    services.gnome3.at-spi2-core.enable = true; # needed for dogtail
+    environment.systemPackages = with pkgs; [
+      # gsettings tool with access to gsettings-desktop-schemas
+      (stdenv.mkDerivation {
+        name = "desktop-gsettings";
+        dontUnpack = true;
+        nativeBuildInputs = [ glib wrapGAppsHook ];
+        buildInputs = [ gsettings-desktop-schemas ];
+        installPhase = ''
+          runHook preInstall
+          mkdir -p $out/bin
+          ln -s ${glib.bin}/bin/gsettings $out/bin/desktop-gsettings
+          runHook postInstall
+        '';
+      })
+    ];
+    services.dbus.packages = with pkgs; [ gnome-photos ];
+  };
+
+  preTestScript = ''
+    # dogtail needs accessibility enabled
+    machine.succeed(
+        "desktop-gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>&1"
+    )
+  '';
+}
diff --git a/nixos/tests/installed-tests/graphene.nix b/nixos/tests/installed-tests/graphene.nix
@@ -0,0 +1,5 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.graphene;
+}
diff --git a/nixos/tests/installed-tests/ostree.nix b/nixos/tests/installed-tests/ostree.nix
@@ -0,0 +1,23 @@
+{ pkgs, lib, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.ostree;
+
+  # TODO: Wrap/patch the tests directly in the package
+  testConfig = {
+    environment.systemPackages = with pkgs; [
+      (python3.withPackages (p: with p; [ pyyaml ]))
+      gnupg
+      ostree
+    ];
+
+    # for GJS tests
+    environment.variables.GI_TYPELIB_PATH = lib.makeSearchPath "lib/girepository-1.0" (with pkgs; [
+      gtk3
+      pango.out
+      ostree
+      gdk-pixbuf
+      atk
+    ]);
+  };
+}
diff --git a/nixos/tests/installed-tests/xdg-desktop-portal.nix b/nixos/tests/installed-tests/xdg-desktop-portal.nix
@@ -0,0 +1,5 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.xdg-desktop-portal;
+}
diff --git a/nixos/tests/ostree.nix b/nixos/tests/ostree.nix
diff --git a/nixos/tests/packagekit.nix b/nixos/tests/packagekit.nix
@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ... }: {
+import ./make-test-python.nix ({ pkgs, ... }: {
   name = "packagekit";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ peterhoeg ];
@@ -13,12 +13,14 @@ import ./make-test.nix ({ pkgs, ... }: {
   };
 
   testScript = ''
-    startAll;
+    start_all()
 
     # send a dbus message to activate the service
-    $machine->succeed("dbus-send --system --type=method_call --print-reply --dest=org.freedesktop.PackageKit /org/freedesktop/PackageKit org.freedesktop.DBus.Introspectable.Introspect");
+    machine.succeed(
+        "dbus-send --system --type=method_call --print-reply --dest=org.freedesktop.PackageKit /org/freedesktop/PackageKit org.freedesktop.DBus.Introspectable.Introspect"
+    )
 
     # so now it should be running
-    $machine->succeed("systemctl is-active packagekit.service");
+    machine.wait_for_unit("packagekit.service")
   '';
 })
diff --git a/nixos/tests/udisks2.nix b/nixos/tests/udisks2.nix
@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ... }:
+import ./make-test-python.nix ({ pkgs, ... }:
 
 let
 
@@ -30,32 +30,40 @@ in
 
   testScript =
     ''
-      my $stick = $machine->stateDir . "/usbstick.img";
-      system("xz -d < ${stick} > $stick") == 0 or die;
+      import lzma
 
-      $machine->succeed("udisksctl info -b /dev/vda >&2");
-      $machine->fail("udisksctl info -b /dev/sda1");
+      with lzma.open(
+          "${stick}"
+      ) as data, open(machine.state_dir + "/usbstick.img", "wb") as stick:
+          stick.write(data.read())
+
+      machine.succeed("udisksctl info -b /dev/vda >&2")
+      machine.fail("udisksctl info -b /dev/sda1")
 
       # Attach a USB stick and wait for it to show up.
-      $machine->sendMonitorCommand("drive_add 0 id=stick,if=none,file=$stick,format=raw");
-      $machine->sendMonitorCommand("device_add usb-storage,id=stick,drive=stick");
-      $machine->waitUntilSucceeds("udisksctl info -b /dev/sda1");
-      $machine->succeed("udisksctl info -b /dev/sda1 | grep 'IdLabel:.*USBSTICK'");
+      machine.send_monitor_command(
+          f"drive_add 0 id=stick,if=none,file={stick.name},format=raw"
+      )
+      machine.send_monitor_command("device_add usb-storage,id=stick,drive=stick")
+      machine.wait_until_succeeds("udisksctl info -b /dev/sda1")
+      machine.succeed("udisksctl info -b /dev/sda1 | grep 'IdLabel:.*USBSTICK'")
 
       # Mount the stick as a non-root user and do some stuff with it.
-      $machine->succeed("su - alice -c 'udisksctl info -b /dev/sda1'");
-      $machine->succeed("su - alice -c 'udisksctl mount -b /dev/sda1'");
-      $machine->succeed("su - alice -c 'cat /run/media/alice/USBSTICK/test.txt'") =~ /Hello World/ or die;
-      $machine->succeed("su - alice -c 'echo foo > /run/media/alice/USBSTICK/bar.txt'");
+      machine.succeed("su - alice -c 'udisksctl info -b /dev/sda1'")
+      machine.succeed("su - alice -c 'udisksctl mount -b /dev/sda1'")
+      machine.succeed(
+          "su - alice -c 'cat /run/media/alice/USBSTICK/test.txt' | grep -q 'Hello World'"
+      )
+      machine.succeed("su - alice -c 'echo foo > /run/media/alice/USBSTICK/bar.txt'")
 
       # Unmounting the stick should make the mountpoint disappear.
-      $machine->succeed("su - alice -c 'udisksctl unmount -b /dev/sda1'");
-      $machine->fail("[ -d /run/media/alice/USBSTICK ]");
+      machine.succeed("su - alice -c 'udisksctl unmount -b /dev/sda1'")
+      machine.fail("[ -d /run/media/alice/USBSTICK ]")
 
       # Remove the USB stick.
-      $machine->sendMonitorCommand("device_del stick");
-      $machine->waitUntilFails("udisksctl info -b /dev/sda1");
-      $machine->fail("[ -e /dev/sda ]");
+      machine.send_monitor_command("device_del stick")
+      machine.wait_until_fails("udisksctl info -b /dev/sda1")
+      machine.fail("[ -e /dev/sda ]")
     '';
 
 })
diff --git a/nixos/tests/xdg-desktop-portal.nix b/nixos/tests/xdg-desktop-portal.nix