NixOS · Mar 8, 2020 · Mar 8, 2020 · Mar 8, 2020
Showing with 185 additions and 79 deletions.

+146 −75 nixos/tests/docker-tools.nix

+5 −4 pkgs/build-support/docker/default.nix

+34 −0 pkgs/build-support/docker/examples.nix
diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
@@ -1,87 +1,158 @@
 # this test creates a simple GNU image with docker tools and sees if it executes
 
-import ./make-test.nix ({ pkgs, ... }: {
+import ./make-test-python.nix ({ pkgs, ... }: {
   name = "docker-tools";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ lnl7 ];
   };
 
   nodes = {
-    docker =
-      { ... }: {
-        virtualisation = {
-          diskSize = 2048;
-          docker.enable = true;
-        };
+    docker = { ... }: {
+      virtualisation = {
+        diskSize = 2048;
+        docker.enable = true;
       };
+    };
   };
 
-  testScript =
-    ''
-      $docker->waitForUnit("sockets.target");
-
-      # Ensure Docker images use a stable date by default
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.bash}'");
-      $docker->succeed("[ '1970-01-01T00:00:01Z' = \"\$(docker inspect ${pkgs.dockerTools.examples.bash.imageName} | ${pkgs.jq}/bin/jq -r .[].Created)\" ]");
-
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.bash.imageName} bash --version");
-      $docker->succeed("docker rmi ${pkgs.dockerTools.examples.bash.imageName}");
-
-      # Check if the nix store is correctly initialized by listing dependencies of the installed Nix binary
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nix}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.nix.imageName} nix-store -qR ${pkgs.nix}");
-      $docker->succeed("docker rmi ${pkgs.dockerTools.examples.nix.imageName}");
-
-      # To test the pullImage tool
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nixFromDockerHub}'");
-      $docker->succeed("docker run --rm nix:2.2.1 nix-store --version");
-      $docker->succeed("docker rmi nix:2.2.1");
-
-      # To test runAsRoot and entry point
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.nginx}'");
-      $docker->succeed("docker run --name nginx -d -p 8000:80 ${pkgs.dockerTools.examples.nginx.imageName}");
-      $docker->waitUntilSucceeds('curl http://localhost:8000/');
-      $docker->succeed("docker rm --force nginx");
-      $docker->succeed("docker rmi '${pkgs.dockerTools.examples.nginx.imageName}'");
-
-      # An pulled image can be used as base image
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.onTopOfPulledImage}'");
-      $docker->succeed("docker run --rm ontopofpulledimage hello");
-      $docker->succeed("docker rmi ontopofpulledimage");
-
-      # Regression test for issue #34779
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.runAsRootExtraCommands}'");
-      $docker->succeed("docker run --rm runasrootextracommands cat extraCommands");
-      $docker->succeed("docker run --rm runasrootextracommands cat runAsRoot");
-      $docker->succeed("docker rmi '${pkgs.dockerTools.examples.runAsRootExtraCommands.imageName}'");
-
-      # Ensure Docker images can use an unstable date
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.bash}'");
-      $docker->succeed("[ '1970-01-01T00:00:01Z' != \"\$(docker inspect ${pkgs.dockerTools.examples.unstableDate.imageName} | ${pkgs.jq}/bin/jq -r .[].Created)\" ]");
-
-      # Ensure Layered Docker images work
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-image}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName}");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName} cat extraCommands");
-
-      # Ensure building an image on top of a layered Docker images work
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-on-top}'");
-      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-on-top.imageName}");
-
-      # Ensure layers are shared between images
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.another-layered-image}'");
-      $docker->succeed("docker inspect ${pkgs.dockerTools.examples.layered-image.imageName} | ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]' | sort > layers1.sha256");
-      $docker->succeed("docker inspect ${pkgs.dockerTools.examples.another-layered-image.imageName} | ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]' | sort > layers2.sha256");
-      $docker->succeed('[ $(comm -1 -2 layers1.sha256 layers2.sha256 | wc -l) -ne 0 ]');
-
-      # Ensure order of layers is correct
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layersOrder}'");
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer1 | grep -q layer1");
-      # This is to be sure the order of layers of the parent image is preserved
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer2 | grep -q layer2");
-      $docker->succeed("docker run --rm  ${pkgs.dockerTools.examples.layersOrder.imageName} cat /tmp/layer3 | grep -q layer3");
-
-      # Ensure image with only 2 layers can be loaded
-      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.two-layered-image}'");
-    '';
+  testScript = with pkgs.dockerTools; ''
+    unix_time_second1 = "1970-01-01T00:00:01Z"
+
+    docker.wait_for_unit("sockets.target")
+
+    with subtest("Ensure Docker images use a stable date by default"):
+        docker.succeed(
+            "docker load --input='${examples.bash}'"
+        )
+        assert unix_time_second1 in docker.succeed(
+            "docker inspect ${examples.bash.imageName} "
+            + "| ${pkgs.jq}/bin/jq -r .[].Created",
+        )
+
+    docker.succeed("docker run --rm ${examples.bash.imageName} bash --version")
+    docker.succeed("docker rmi ${examples.bash.imageName}")
+
+    with subtest(
+        "Check if the nix store is correctly initialized by listing "
+        "dependencies of the installed Nix binary"
+    ):
+        docker.succeed(
+            "docker load --input='${examples.nix}'",
+            "docker run --rm ${examples.nix.imageName} nix-store -qR ${pkgs.nix}",
+            "docker rmi ${examples.nix.imageName}",
+        )
+
+    with subtest("The pullImage tool works"):
+        docker.succeed(
+            "docker load --input='${examples.nixFromDockerHub}'",
+            "docker run --rm nix:2.2.1 nix-store --version",
+            "docker rmi nix:2.2.1",
+        )
+
+    with subtest("runAsRoot and entry point work"):
+        docker.succeed(
+            "docker load --input='${examples.nginx}'",
+            "docker run --name nginx -d -p 8000:80 ${examples.nginx.imageName}",
+        )
+        docker.wait_until_succeeds("curl http://localhost:8000/")
+        docker.succeed(
+            "docker rm --force nginx", "docker rmi '${examples.nginx.imageName}'",
+        )
+
+    with subtest("A pulled image can be used as base image"):
+        docker.succeed(
+            "docker load --input='${examples.onTopOfPulledImage}'",
+            "docker run --rm ontopofpulledimage hello",
+            "docker rmi ontopofpulledimage",
+        )
+
+    with subtest("Regression test for issue #34779"):
+        docker.succeed(
+            "docker load --input='${examples.runAsRootExtraCommands}'",
+            "docker run --rm runasrootextracommands cat extraCommands",
+            "docker run --rm runasrootextracommands cat runAsRoot",
+            "docker rmi '${examples.runAsRootExtraCommands.imageName}'",
+        )
+
+    with subtest("Ensure Docker images can use an unstable date"):
+        docker.succeed(
+            "docker load --input='${examples.bash}'"
+        )
+        assert unix_time_second1 not in docker.succeed(
+            "docker inspect ${examples.unstableDate.imageName} "
+            + "| ${pkgs.jq}/bin/jq -r .[].Created"
+        )
+
+    with subtest("Ensure Layered Docker images work"):
+        docker.succeed(
+            "docker load --input='${examples.layered-image}'",
+            "docker run --rm ${examples.layered-image.imageName}",
+            "docker run --rm ${examples.layered-image.imageName} cat extraCommands",
+        )
+
+    with subtest("Ensure building an image on top of a layered Docker images work"):
+        docker.succeed(
+            "docker load --input='${examples.layered-on-top}'",
+            "docker run --rm ${examples.layered-on-top.imageName}",
+        )
+
+
+    def set_of_layers(image_name):
+        return set(
+            docker.succeed(
+                f"docker inspect {image_name} "
+                + "| ${pkgs.jq}/bin/jq -r '.[] | .RootFS.Layers | .[]'"
+            ).split()
+        )
+
+
+    with subtest("Ensure layers are shared between images"):
+        docker.succeed(
+            "docker load --input='${examples.another-layered-image}'"
+        )
+        layers1 = set_of_layers("${examples.layered-image.imageName}")
+        layers2 = set_of_layers("${examples.another-layered-image.imageName}")
+        assert bool(layers1 & layers2)
+
+    with subtest("Ensure order of layers is correct"):
+        docker.succeed(
+            "docker load --input='${examples.layersOrder}'"
+        )
+
+        for index in 1, 2, 3:
+            assert f"layer{index}" in docker.succeed(
+                f"docker run --rm  ${examples.layersOrder.imageName} cat /tmp/layer{index}"
+            )
+
+    with subtest("Ensure image with only 2 layers can be loaded"):
+        docker.succeed(
+            "docker load --input='${examples.two-layered-image}'"
+        )
+
+    with subtest(
+        "Ensure the bulk layer doesn't miss store paths (regression test for #78744)"
+    ):
+        docker.succeed(
+            "docker load --input='${pkgs.dockerTools.examples.bulk-layer}'",
+            # Ensure the two output paths (ls and hello) are in the layer
+            "docker run bulk-layer ls /bin/hello",
+        )
+
+    with subtest("Ensure correct behavior when no store is needed"):
+        # This check tests two requirements simultaneously
+        #  1. buildLayeredImage can build images that don't need a store.
+        #  2. Layers of symlinks are eliminated by the customization layer.
+        #
+        docker.succeed(
+            "docker load --input='${pkgs.dockerTools.examples.no-store-paths}'"
+        )
+
+        # Busybox will not recognize argv[0] and print an error message with argv[0],
+        # but it confirms that the custom-true symlink is present.
+        docker.succeed("docker run --rm no-store-paths custom-true |& grep custom-true")
+
+        # This check may be loosened to allow an *empty* store rather than *no* store.
+        docker.succeed("docker run --rm no-store-paths ls /")
+        docker.fail("docker run --rm no-store-paths ls /nix/store")
+  '';
 })
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
@@ -319,6 +319,8 @@ rec {
       enableParallelBuilding = true;
     }
     ''
+      mkdir layers
+
       # Delete impurities for store path layers, so they don't get
       # shared and taint other projects.
       cat ${configJson} \
@@ -330,13 +332,12 @@ rec {
       # created, and that no paths are missed. If you change the
       # following head and tail call lines, double-check that your
       # code behaves properly when the number of layers equals:
-      #      maxLayers-1, maxLayers, and maxLayers+1
+      #      maxLayers-1, maxLayers, and maxLayers+1, 0
       paths() {
-        cat $paths ${lib.concatMapStringsSep " " (path: "| grep -v ${path}") (closures ++ [ overallClosure ])}
+        cat $paths ${lib.concatMapStringsSep " " (path: "| (grep -v ${path} || true)") (closures ++ [ overallClosure ])}
       }
 
-      # We need to sponge to avoid grep broken pipe error when maxLayers == 1
-      paths | sponge | head -n $((maxLayers - 1)) | cat -n | xargs -r -P$NIX_BUILD_CORES -n2 ${storePathToLayer}
+      paths | head -n $((maxLayers - 1)) | cat -n | xargs -r -P$NIX_BUILD_CORES -n2 ${storePathToLayer}
       if [ $(paths | wc -l) -ge $maxLayers ]; then
         paths | tail -n+$maxLayers | xargs ${storePathToLayer} $maxLayers
       fi

diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
@@ -246,4 +246,38 @@ rec {
     contents = [ pkgs.bash pkgs.hello ];
     maxLayers = 2;
   };
+
+  # 16. Create a layered image with more packages than max layers.
+  # coreutils and hello are part of the same layer
+  bulk-layer = pkgs.dockerTools.buildLayeredImage {
+    name = "bulk-layer";
+    tag = "latest";
+    contents = with pkgs; [
+      coreutils hello
+    ];
+    maxLayers = 2;
+  };
+
+  # 17. Create a "layered" image without nix store layers. This is not
+  # recommended, but can be useful for base images in rare cases.
+  no-store-paths = pkgs.dockerTools.buildLayeredImage {
+    name = "no-store-paths";
+    tag = "latest";
+    extraCommands = ''
+      chmod a+w bin
+
+      # This removes sharing of busybox and is not recommended. We do this
+      # to make the example suitable as a test case with working binaries.
+      cp -r ${pkgs.pkgsStatic.busybox}/* .
+    '';
+    contents = [
+      # This layer has no dependencies and its symlinks will be dereferenced
+      # when creating the customization layer.
+      (pkgs.runCommand "layer-to-flatten" {} ''
+        mkdir -p $out/bin
+        ln -s /bin/true $out/bin/custom-true
+      ''
+      )
+    ];
+  };
 }