NixOS · Mar 8, 2020 · Mar 8, 2020 · Mar 8, 2020 · Mar 8, 2020 · Mar 8, 2020
diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix
@@ -41,7 +41,7 @@ import ./make-test-python.nix {
       systemd.services.git-daemon = {
         description   = "Git daemon for the test";
         wantedBy      = [ "multi-user.target" ];
-        after         = [ "network.target" ];
+        after         = [ "network.target" "sshd.service" ];
 
         serviceConfig.Restart = "always";
         path = with pkgs; [ coreutils git openssh ];

diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
@@ -1,87 +1,158 @@
 # this test creates a simple GNU image with docker tools and sees if it executes
 
-import ./make-test.nix ({ pkgs, ... }: {
+import ./make-test-python.nix ({ pkgs, ... }: {
   name = "docker-tools";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ lnl7 ];
   };
 
   nodes = {
-    docker =
-      { ... }: {
-        virtualisation = {
-          diskSize = 2048;
-          docker.enable = true;
-        };
+    docker = { ... }: {
+      virtualisation = {
+        diskSize = 2048;
+        docker.enable = true;
       };
+    };
   };
 
-  testScript =
-    ''
-      $docker->waitForUnit("sockets.target");
-
-      # Ensure Docker images use a stable date by default
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.bash}'");
-      $docker->succeed("[ '1970-01-01T00:00:01Z' = \"\$(docker inspect ${pkgs.dockerTools.examples.bash.imageName} | ${pkgs.jq}/bin/jq -r .[].Created)\" ]");
-
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.bash.imageName} bash --version");
-      $docker->succeed("docker rmi ${pkgs.dockerTools.examples.bash.imageName}");
-
-      # Check if the nix store is correctly initialized by listing dependencies of the installed Nix binary
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nix}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.nix.imageName} nix-store -qR ${pkgs.nix}");
-      $docker->succeed("docker rmi ${pkgs.dockerTools.examples.nix.imageName}");
-
-      # To test the pullImage tool
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nixFromDockerHub}'");
-      $docker->succeed("docker run --rm nix:2.2.1 nix-store --version");
-      $docker->succeed("docker rmi nix:2.2.1");
-
-      # To test runAsRoot and entry point
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nginx}'");
-      $docker->succeed("docker run --name nginx -d -p 8000:80 ${pkgs.dockerTools.examples.nginx.imageName}");
-      $docker->waitUntilSucceeds('curl http://localhost:8000/');
-      $docker->succeed("docker rm --force nginx");
-      $docker->succeed("docker rmi '${pkgs.dockerTools.examples.nginx.imageName}'");
-
-      # An pulled image can be used as base image
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.onTopOfPulledImage}'");
-      $docker->succeed("docker run --rm ontopofpulledimage hello");
-      $docker->succeed("docker rmi ontopofpulledimage");
-
-      # Regression test for issue #34779
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.runAsRootExtraCommands}'");
-      $docker->succeed("docker run --rm runasrootextracommands cat extraCommands");
-      $docker->succeed("docker run --rm runasrootextracommands cat runAsRoot");
-      $docker->succeed("docker rmi '${pkgs.dockerTools.examples.runAsRootExtraCommands.imageName}'");
-
-      # Ensure Docker images can use an unstable date
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.bash}'");
-      $docker->succeed("[ '1970-01-01T00:00:01Z' != \"\$(docker inspect ${pkgs.dockerTools.examples.unstableDate.imageName} | ${pkgs.jq}/bin/jq -r .[].Created)\" ]");
-
-      # Ensure Layered Docker images work
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-image}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName}");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName} cat extraCommands");
-
-      # Ensure building an image on top of a layered Docker images work
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-on-top}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-on-top.imageName}");
-
-      # Ensure layers are shared between images
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.another-layered-image}'");
-      $docker->succeed("docker inspect ${pkgs.dockerTools.examples.layered-image.imageName} | ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]' | sort > layers1.sha256");
-      $docker->succeed("docker inspect ${pkgs.dockerTools.examples.another-layered-image.imageName} | ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]' | sort > layers2.sha256");
-      $docker->succeed('[ $(comm -1 -2 layers1.sha256 layers2.sha256 | wc -l) -ne 0 ]');
-
-      # Ensure order of layers is correct
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layersOrder}'");
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer1 | grep -q layer1");
-      # This is to be sure the order of layers of the parent image is preserved
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer2 | grep -q layer2");
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer3 | grep -q layer3");
-
-      # Ensure image with only 2 layers can be loaded
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.two-layered-image}'");
-    '';
+  testScript = with pkgs.dockerTools; ''
+    unix_time_second1 = "1970-01-01T00:00:01Z"
+
+    docker.wait_for_unit("sockets.target")
+
+    with subtest("Ensure Docker images use a stable date by default"):
+        docker.succeed(
+            "docker load --input='${examples.bash}'"
+        )
+        assert unix_time_second1 in docker.succeed(
+            "docker inspect ${examples.bash.imageName} "
+            + "| ${pkgs.jq}/bin/jq -r .[].Created",
+        )
+
+    docker.succeed("docker run --rm ${examples.bash.imageName} bash --version")
+    docker.succeed("docker rmi ${examples.bash.imageName}")
+
+    with subtest(
+        "Check if the nix store is correctly initialized by listing "
+        "dependencies of the installed Nix binary"
+    ):
+        docker.succeed(
+            "docker load --input='${examples.nix}'",
+            "docker run --rm ${examples.nix.imageName} nix-store -qR ${pkgs.nix}",
+            "docker rmi ${examples.nix.imageName}",
+        )
+
+    with subtest("The pullImage tool works"):
+        docker.succeed(
+            "docker load --input='${examples.nixFromDockerHub}'",
+            "docker run --rm nix:2.2.1 nix-store --version",
+            "docker rmi nix:2.2.1",
+        )
+
+    with subtest("runAsRoot and entry point work"):
+        docker.succeed(
+            "docker load --input='${examples.nginx}'",
+            "docker run --name nginx -d -p 8000:80 ${examples.nginx.imageName}",
+        )
+        docker.wait_until_succeeds("curl http://localhost:8000/")
+        docker.succeed(
+            "docker rm --force nginx", "docker rmi '${examples.nginx.imageName}'",
+        )
+
+    with subtest("A pulled image can be used as base image"):
+        docker.succeed(
+            "docker load --input='${examples.onTopOfPulledImage}'",
+            "docker run --rm ontopofpulledimage hello",
+            "docker rmi ontopofpulledimage",
+        )
+
+    with subtest("Regression test for issue #34779"):
+        docker.succeed(
+            "docker load --input='${examples.runAsRootExtraCommands}'",
+            "docker run --rm runasrootextracommands cat extraCommands",
+            "docker run --rm runasrootextracommands cat runAsRoot",
+            "docker rmi '${examples.runAsRootExtraCommands.imageName}'",
+        )
+
+    with subtest("Ensure Docker images can use an unstable date"):
+        docker.succeed(
+            "docker load --input='${examples.bash}'"
+        )
+        assert unix_time_second1 not in docker.succeed(
+            "docker inspect ${examples.unstableDate.imageName} "
+            + "| ${pkgs.jq}/bin/jq -r .[].Created"
+        )
+
+    with subtest("Ensure Layered Docker images work"):
+        docker.succeed(
+            "docker load --input='${examples.layered-image}'",
+            "docker run --rm ${examples.layered-image.imageName}",
+            "docker run --rm ${examples.layered-image.imageName} cat extraCommands",
+        )
+
+    with subtest("Ensure building an image on top of a layered Docker images work"):
+        docker.succeed(
+            "docker load --input='${examples.layered-on-top}'",
+            "docker run --rm ${examples.layered-on-top.imageName}",
+        )
+
+
+    def set_of_layers(image_name):
+        return set(
+            docker.succeed(
+                f"docker inspect {image_name} "
+                + "| ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]'"
+            ).split()
+        )
+
+
+    with subtest("Ensure layers are shared between images"):
+        docker.succeed(
+            "docker load --input='${examples.another-layered-image}'"
+        )
+        layers1 = set_of_layers("${examples.layered-image.imageName}")
+        layers2 = set_of_layers("${examples.another-layered-image.imageName}")
+        assert bool(layers1 & layers2)
+
+    with subtest("Ensure order of layers is correct"):
+        docker.succeed(
+            "docker load --input='${examples.layersOrder}'"
+        )
+
+        for index in 1, 2, 3:
+            assert f"layer{index}" in docker.succeed(
+                f"docker run --rm  ${examples.layersOrder.imageName} cat /tmp/layer{index}"
+            )
+
+    with subtest("Ensure image with only 2 layers can be loaded"):
+        docker.succeed(
+            "docker load --input='${examples.two-layered-image}'"
+        )
+
+    with subtest(
+        "Ensure the bulk layer doesn't miss store paths (regression test for #78744)"
+    ):
+        docker.succeed(
+            "docker load --input='${pkgs.dockerTools.examples.bulk-layer}'",
+            # Ensure the two output paths (ls and hello) are in the layer
+            "docker run bulk-layer ls /bin/hello",
+        )
+
+    with subtest("Ensure correct behavior when no store is needed"):
+        # This check tests two requirements simultaneously
+        #  1. buildLayeredImage can build images that don't need a store.
+        #  2. Layers of symlinks are eliminated by the customization layer.
+        #
+        docker.succeed(
+            "docker load --input='${pkgs.dockerTools.examples.no-store-paths}'"
+        )
+
+        # Busybox will not recognize argv[0] and print an error message with argv[0],
+        # but it confirms that the custom-true symlink is present.
+        docker.succeed("docker run --rm no-store-paths custom-true |& grep custom-true")
+
+        # This check may be loosened to allow an *empty* store rather than *no* store.
+        docker.succeed("docker run --rm no-store-paths ls /")
+        docker.fail("docker run --rm no-store-paths ls /nix/store")
+  '';
 })
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
@@ -319,6 +319,8 @@ rec {
       enableParallelBuilding = true;
     }
     ''
+      mkdir layers
+
       # Delete impurities for store path layers, so they don't get
       # shared and taint other projects.
       cat ${configJson} \
@@ -330,13 +332,12 @@ rec {
       # created, and that no paths are missed. If you change the
       # following head and tail call lines, double-check that your
       # code behaves properly when the number of layers equals:
-      #      maxLayers-1, maxLayers, and maxLayers+1
+      #      maxLayers-1, maxLayers, and maxLayers+1, 0
       paths() {
-        cat $paths ${lib.concatMapStringsSep " " (path: "| grep -v ${path}") (closures ++ [ overallClosure ])}
+        cat $paths ${lib.concatMapStringsSep " " (path: "| (grep -v ${path} || true)") (closures ++ [ overallClosure ])}
       }
 
-      # We need to sponge to avoid grep broken pipe error when maxLayers == 1
-      paths | sponge | head -n $((maxLayers - 1)) | cat -n | xargs -r -P$NIX_BUILD_CORES -n2 ${storePathToLayer}
+      paths | head -n $((maxLayers - 1)) | cat -n | xargs -r -P$NIX_BUILD_CORES -n2 ${storePathToLayer}
       if [ $(paths | wc -l) -ge $maxLayers ]; then
         paths | tail -n+$maxLayers | xargs ${storePathToLayer} $maxLayers
       fi

diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
@@ -246,4 +246,38 @@ rec {
     contents = [ pkgs.bash pkgs.hello ];
     maxLayers = 2;
   };
+
+  # 16. Create a layered image with more packages than max layers.
+  # coreutils and hello are part of the same layer
+  bulk-layer = pkgs.dockerTools.buildLayeredImage {
+    name = "bulk-layer";
+    tag = "latest";
+    contents = with pkgs; [
+      coreutils hello
+    ];
+    maxLayers = 2;
+  };
+
+  # 17. Create a "layered" image without nix store layers. This is not
+  # recommended, but can be useful for base images in rare cases.
+  no-store-paths = pkgs.dockerTools.buildLayeredImage {
+    name = "no-store-paths";
+    tag = "latest";
+    extraCommands = ''
+      chmod a+w bin
+
+      # This removes sharing of busybox and is not recommended. We do this
+      # to make the example suitable as a test case with working binaries.
+      cp -r ${pkgs.pkgsStatic.busybox}/* .
+    '';
+    contents = [
+      # This layer has no dependencies and its symlinks will be dereferenced
+      # when creating the customization layer.
+      (pkgs.runCommand "layer-to-flatten" {} ''
+        mkdir -p $out/bin
+        ln -s /bin/true $out/bin/custom-true
+      ''
+      )
+    ];
+  };
 }
diff --git a/pkgs/development/python-modules/buildbot/default.nix b/pkgs/development/python-modules/buildbot/default.nix
@@ -3,7 +3,7 @@
   sqlalchemy_migrate, dateutil, txaio, autobahn, pyjwt, pyyaml, treq,
   txrequests, pyjade, boto3, moto, mock, python-lz4, setuptoolsTrial,
   isort, pylint, flake8, buildbot-worker, buildbot-pkg, buildbot-plugins,
-  parameterized, git, openssh, glibcLocales }:
+  parameterized, git, openssh, glibcLocales, nixosTests }:
 
 let
   withPlugins = plugins: buildPythonPackage {
@@ -25,11 +25,11 @@ let
 
   package = buildPythonPackage rec {
     pname = "buildbot";
-    version = "2.6.0";
+    version = "2.7.0";
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "1l3ajhy68jddbgbizaa5hq65lgqkll6389hss4p2j36cbxbn7hiv";
+      sha256 = "0jj8fh611n7xc3vsfbgpqsllp38cfj3spkr2kz3ara2x7jvh3406";
     };
 
     propagatedBuildInputs = [
@@ -91,6 +91,7 @@ let
 
     passthru = {
       inherit withPlugins;
+      tests.buildbot = nixosTests.buildbot;
     };
 
     meta = with lib; {

diff --git a/pkgs/development/python-modules/buildbot/pkg.nix b/pkgs/development/python-modules/buildbot/pkg.nix
@@ -2,11 +2,11 @@
 
 buildPythonPackage rec {
   pname = "buildbot-pkg";
-  version = "2.6.0";
+  version = "2.7.0";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "07ynk46c9h47ibbdm93h15xbrzflsl4llm0jr4c8pm6krs6byb4z";
+    sha256 = "03zb09r8w8dvd9qas7h6gdwlqc7q482ikph6h3708lpnkn72xdkb";
   };
 
   postPatch = ''

diff --git a/pkgs/development/python-modules/buildbot/plugins.nix b/pkgs/development/python-modules/buildbot/plugins.nix
@@ -7,14 +7,12 @@
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "0njix8g6g1dgfviick55p9calm82nnkhhgkikh7a5yvqk524cprg";
+      sha256 = "1d8xdk4rq4p3fw03cvz7d1pmpjjbyrnzzjifzv46q88vk7jakgxi";
     };
 
     # Remove unneccessary circular dependency on buildbot
     postPatch = ''
-      sed -i setup.py \
-        -e "/import buildbot/d" \
-        -e "s/'buildbot',//"
+      sed -i "s/'buildbot'//" setup.py
     '';
 
     buildInputs = [ buildbot-pkg mock ];
@@ -36,7 +34,7 @@
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "1wmay9bbb1wwf0zh9rw95swmdc7bxabgvx2n6dyyzs4p8k92px10";
+      sha256 = "03pl75avi6cmmhjvfn0a0b4drn35yv37kvgn04zjdwa3m6p3haa8";
     };
 
     buildInputs = [ buildbot-pkg ];
@@ -58,7 +56,7 @@
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "14b28vpkk68yk6zmdd64n7izb1r3barb7y711n4g37skqd13jygb";
+      sha256 = "0rzjk3qmlvid8qag3r00zaszchncl1nl8l2yapvc1zqh2dqlln58";
     };
 
     buildInputs = [ buildbot-pkg ];
@@ -80,7 +78,7 @@
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "1m8kbycjq7jsl72xbrzjj48hb65r53lfxx78yvlnhc2f9ian5vnh";
+      sha256 = "1n4j73y9kwfqk7dz1fh1bpan68vlpnbz7idxpmsphyay8w8y9dd4";
     };
 
     buildInputs = [ buildbot-pkg ];
@@ -102,7 +100,7 @@
 
     src = fetchPypi {
       inherit pname version;
-      sha256 = "1sbp13qimv8kc7dp0xz1pmda1x1n4948zgrz2pc45rwlbbn5h69r";
+      sha256 = "1babkcgxczs6zfk2b6jmsy2vwbrgdydrp2px1mfwa3wmv8fwlssg";
     };
 
     buildInputs = [ buildbot-pkg ];

diff --git a/pkgs/development/python-modules/buildbot/worker.nix b/pkgs/development/python-modules/buildbot/worker.nix
@@ -3,11 +3,11 @@
 
 buildPythonPackage (rec {
   pname = "buildbot-worker";
-  version = "2.6.0";
+  version = "2.7.0";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "0hr42fp3sw6c59qahihm9440618z7prwsy4z0ax553zvw47pc22l";
+    sha256 = "1vwy46acvczgk1hhpsqdwpcw55j4hm5pkw6j01f92axiga8r5jk6";
   };
 
   propagatedBuildInputs = [ twisted future ];