NixOS · Nov 18, 2020 · Nov 18, 2020 · Nov 19, 2020 · Nov 19, 2020 · Nov 19, 2020
diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix
@@ -11,6 +11,7 @@ with lib;
 let
   cfg = config.ec2;
   metadataFetcher = import ./ec2-metadata-fetcher.nix {
+    inherit (pkgs) curl;
     targetRoot = "$targetRoot/";
     wgetExtraOptions = "-q";
   };

diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix
@@ -1,23 +1,79 @@
-{ targetRoot, wgetExtraOptions }:
+{ curl, targetRoot, wgetExtraOptions }:
+# Note: be very cautious about dependencies, each dependency grows
+# the closure of the initrd. Ideally we would not even require curl,
+# but there is no reasonable way to send an HTTP PUT request without
+# it. Note: do not be fooled: the wget referenced in this script
+# is busybox's wget, not the fully featured one with --method support.
+#
+# Make sure that every package you depend on here is already listed as
+# a channel blocker for both the full-sized and small channels.
+# Otherwise, we risk breaking user deploys in released channels.
 ''
   metaDir=${targetRoot}etc/ec2-metadata
   mkdir -m 0755 -p "$metaDir"
 
+  get_imds_token() {
+    # retry-delay of 1 selected to give the system a second to get going,
+    # but not add a lot to the bootup time
+    ${curl}/bin/curl \
+      -v \
+      --retry 3 \
+      --retry-delay 1 \
+      --fail \
+      -X PUT \
+      --connect-timeout 1 \
+      -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \
+      http://169.254.169.254/latest/api/token
+  }
+
+  preflight_imds_token() {
+    # retry-delay of 1 selected to give the system a second to get going,
+    # but not add a lot to the bootup time
+    ${curl}/bin/curl \
+      -v \
+      --retry 3 \
+      --retry-delay 1 \
+      --fail \
+      --connect-timeout 1 \
+      -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \
+      http://169.254.169.254/1.0/meta-data/instance-id
+  }
+
+  try=1
+  while [ $try -le 3 ]; do
+    echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..."
+    IMDS_TOKEN=$(get_imds_token) && break
+    try=$((try + 1))
+    sleep 1
+  done
+
+  if [ "x$IMDS_TOKEN" == "x" ]; then
+    echo "failed to fetch an IMDS2v token."
+  fi
+
+  try=1
+  while [ $try -le 10 ]; do
+    echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..."
+    preflight_imds_token && break
+    try=$((try + 1))
+    sleep 1
+  done
+
   echo "getting EC2 instance metadata..."
 
   if ! [ -e "$metaDir/ami-manifest-path" ]; then
-    wget ${wgetExtraOptions} -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
+    wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
   fi
 
   if ! [ -e "$metaDir/user-data" ]; then
-    wget ${wgetExtraOptions} -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
+    wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
   fi
 
   if ! [ -e "$metaDir/hostname" ]; then
-    wget ${wgetExtraOptions} -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
+    wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
   fi
 
   if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
-    wget ${wgetExtraOptions} -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
+    wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
   fi
 ''
diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
@@ -3,7 +3,7 @@
 with lib;
 
 let
-  metadataFetcher = import ./ec2-metadata-fetcher.nix {
+  metadataFetcher = import ./openstack-metadata-fetcher.nix {
     targetRoot = "/";
     wgetExtraOptions = "--retry-connrefused";
   };

diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
@@ -0,0 +1,23 @@
+{ targetRoot, wgetExtraOptions }:
+''
+  metaDir=${targetRoot}etc/ec2-metadata
+  mkdir -m 0755 -p "$metaDir"
+
+  echo "getting EC2 instance metadata..."
+
+  if ! [ -e "$metaDir/ami-manifest-path" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
+  fi
+
+  if ! [ -e "$metaDir/user-data" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
+  fi
+
+  if ! [ -e "$metaDir/hostname" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
+  fi
+
+  if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
+  fi
+''
diff --git a/nixos/release-combined.nix b/nixos/release-combined.nix
@@ -49,6 +49,7 @@ in rec {
         [ "nixos.channel" ]
         (onFullSupported "nixos.dummy")
         (onAllSupported "nixos.iso_minimal")
+        (onAllSupported "nixos.amazonImage")
         (onSystems ["x86_64-linux"] "nixos.iso_plasma5")
         (onSystems ["x86_64-linux"] "nixos.iso_gnome")
         (onFullSupported "nixos.manual")

diff --git a/nixos/release-small.nix b/nixos/release-small.nix
@@ -28,7 +28,7 @@ let
 in rec {
 
   nixos = {
-    inherit (nixos') channel manual options iso_minimal dummy;
+    inherit (nixos') channel manual options iso_minimal amazonImage dummy;
     tests = {
       inherit (nixos'.tests)
         containers-imperative
@@ -92,6 +92,7 @@ in rec {
       [ "nixos.channel"
         "nixos.dummy.x86_64-linux"
         "nixos.iso_minimal.x86_64-linux"
+        "nixos.amazonImage.x86_64-linux"
         "nixos.manual.x86_64-linux"
         "nixos.tests.boot.biosCdrom.x86_64-linux"
         "nixos.tests.containers-imperative.x86_64-linux"

diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
@@ -234,5 +234,17 @@ import ./make-test-python.nix ({ pkgs, ... }: {
             "docker run --rm file-in-store nix-store --verify --check-contents",
             "docker run --rm file-in-store |& grep 'some data'",
         )
+
+    with subtest("Ensure cross compiled image can be loaded and has correct arch."):
+        docker.succeed(
+            "docker load --input='${pkgs.dockerTools.examples.cross-aarch64}'",
+        )
+        assert (
+            docker.succeed(
+                "docker inspect ${pkgs.dockerTools.examples.cross-aarch64.imageName} "
+                + "| ${pkgs.jq}/bin/jq -r .[].Architecture"
+            ).strip()
+            == "arm64v8"
+        )
   '';
 })
diff --git a/pkgs/applications/networking/instant-messengers/matrix-dl/default.nix b/pkgs/applications/networking/instant-messengers/matrix-dl/default.nix
@@ -1,14 +1,14 @@
 { lib, python3Packages, fetchFromGitHub }:
 
 python3Packages.buildPythonApplication rec {
-  pname = "matrix-dl-unstable";
-  version = "2019-09-22";
+  pname = "matrix-dl";
+  version = "unstable-2020-07-14";
 
   src = fetchFromGitHub {
     owner = "rubo77";
-    repo = "matrix-dl";
-    rev = "e91610f45b7b3b0aca34923309fc83ba377f8a69";
-    sha256 = "036xfdd21pcfjlilknc67z5jqpk0vz07853wwcsiac32iypc6f2q";
+    repo = pname;
+    rev = "b1a86d1421f39ee327284e1023f09dc165e3c8a5";
+    sha256 = "1l8nh8z7kz24v0wcy3ll3w6in2yxwa1yz8lyc3x0blz37d8ss4ql";
   };
 
   propagatedBuildInputs = with python3Packages; [

diff --git a/pkgs/applications/terminal-emulators/mlterm/default.nix b/pkgs/applications/terminal-emulators/mlterm/default.nix
@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchurl, pkgconfig, autoconf, makeDesktopItem
+{ stdenv, lib, fetchFromGitHub, pkgconfig, autoconf, makeDesktopItem
 , libX11, gdk-pixbuf, cairo, libXft, gtk3, vte
 , harfbuzz #substituting glyphs with opentype fonts
 , fribidi, m17n_lib #bidi and encoding
@@ -12,9 +12,11 @@ stdenv.mkDerivation rec {
   pname = "mlterm";
   version = "3.9.1";
 
-  src = fetchurl {
-    url = "mirror://sourceforge/project/mlterm/01release/${pname}-${version}/${pname}-${version}.tar.gz";
-    sha256 = "03fnynwv7d1aicwk2rp31sgncv5m65agvygqvsgn59v9di40gnnb";
+  src = fetchFromGitHub {
+    owner = "arakiken";
+    repo = pname;
+    rev = "rel-${lib.replaceStrings [ "." ] [ "_" ] version}"; # 3.9.1 -> rel-3_9_1
+    sha256 = "1hh196kz2n3asv8r8r2bdk5b2w93zq7rw4880ciiq1554h0ib7fj";
   };
 
   nativeBuildInputs = [ pkgconfig autoconf wrapGAppsHook ];

diff --git a/pkgs/applications/version-management/mercurial/default.nix b/pkgs/applications/version-management/mercurial/default.nix
@@ -4,26 +4,24 @@
 }:
 
 let
-  inherit (python3Packages) docutils dulwich python;
+  inherit (python3Packages) docutils python;
 
 in python3Packages.buildPythonApplication rec {
   pname = "mercurial";
-  version = "5.4.2";
+  version = "5.6";
 
   src = fetchurl {
     url = "https://mercurial-scm.org/release/mercurial-${version}.tar.gz";
-    sha256 = "0ls8nwx3nz26pibphw54fg8pxqb365zmmqx95lqrxqqyf3d972sw";
+    sha256 = "1hk2y30zzdnlv8f71kabvh0xi9c7qhp28ksh20vpd0r712sv79yz";
   };
 
   format = "other";
 
-  inherit python; # pass it so that the same version can be used in hg2git
+  passthru = { inherit python; }; # pass it so that the same version can be used in hg2git
 
   buildInputs = [ makeWrapper docutils unzip ]
     ++ stdenv.lib.optionals stdenv.isDarwin [ ApplicationServices ];
 
-  propagatedBuildInputs = [ dulwich ];
-
   makeFlags = [ "PREFIX=$(out)" ];
 
   postInstall = (stdenv.lib.optionalString guiSupport ''

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
@@ -31,6 +31,7 @@
   writeScript,
   writeText,
   writePython3,
+  system,  # Note: This is the cross system we're compiling for
 }:
 
 # WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
@@ -48,14 +49,24 @@ let
     # A user is required by nix
     # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478
     export USER=nobody
-    ${nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration
+    ${buildPackages.nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration
 
     mkdir -p nix/var/nix/gcroots/docker/
     for i in ${lib.concatStringsSep " " contentsList}; do
     ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
     done;
   '';
 
+  # Map nixpkgs architecture to Docker notation
+  # Reference: https://github.com/docker-library/official-images#architectures-other-than-amd64
+  getArch = nixSystem: {
+    aarch64-linux = "arm64v8";
+    armv7l-linux = "arm32v7";
+    x86_64-linux = "amd64";
+    powerpc64le-linux = "ppc64le";
+    i686-linux = "i386";
+  }.${nixSystem} or "Can't map Nix system ${nixSystem} to Docker architecture notation. Please check that your input and your requested build are correct or update the mapping in Nixpkgs.";
+
 in
 rec {
 
@@ -72,7 +83,7 @@ rec {
     , imageDigest
     , sha256
     , os ? "linux"
-    , arch ? buildPackages.go.GOARCH
+    , arch ? getArch system
 
       # This is used to set name to the pulled image
     , finalImageName ? imageName
@@ -443,7 +454,7 @@ rec {
       runCommand "${name}.tar.gz" {
         inherit (stream) imageName;
         passthru = { inherit (stream) imageTag; };
-        buildInputs = [ pigz ];
+        nativeBuildInputs = [ pigz ];
       } "${stream} | pigz -nT > $out";
 
   # 1. extract the base image
@@ -488,7 +499,7 @@ rec {
       baseJson = let
           pure = writeText "${baseName}-config.json" (builtins.toJSON {
             inherit created config;
-            architecture = buildPackages.go.GOARCH;
+            architecture = getArch system;
             os = "linux";
           });
           impure = runCommand "${baseName}-config.json"
@@ -715,7 +726,7 @@ rec {
       streamScript = writePython3 "stream" {} ./stream_layered_image.py;
       baseJson = writeText "${name}-base.json" (builtins.toJSON {
          inherit config;
-         architecture = buildPackages.go.GOARCH;
+         architecture = getArch system;
          os = "linux";
       });
 
@@ -762,7 +773,7 @@ rec {
             else
               lib.head (lib.strings.splitString "-" (baseNameOf conf.outPath));
         paths = referencesByPopularity overallClosure;
-        buildInputs = [ jq ];
+        nativeBuildInputs = [ jq ];
       } ''
         ${if (tag == null) then ''
           outName="$(basename "$out")"
@@ -826,7 +837,7 @@ rec {
           # take images can know in advance how the image is supposed to be used.
           isExe = true;
         };
-        buildInputs = [ makeWrapper ];
+        nativeBuildInputs = [ makeWrapper ];
       } ''
         makeWrapper ${streamScript} $out --add-flags ${conf}
       '';

diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
@@ -7,7 +7,7 @@
 #  $ nix-build '<nixpkgs>' -A dockerTools.examples.redis
 #  $ docker load < result
 
-{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }:
+{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb, pkgsCross }:
 
 rec {
   # 1. basic example
@@ -407,4 +407,11 @@ rec {
       contents = [ pkgs.bash pkgs.coreutils ] ++ nonRootShadowSetup { uid = 999; user = "somebody"; };
     };
 
+  # basic example, with cross compilation
+  cross-aarch64 = pkgsCross.aarch64-multiplatform.dockerTools.buildImage {
+    name = "hello-cross";
+    tag = "latest";
+    contents = pkgsCross.aarch64-multiplatform.hello;
+  };
+
 }