Skip to content

Commit

Permalink
dockerTools.buildImage: Switch to the format image generated by Skopeo
Browse files Browse the repository at this point in the history 
We were using 'Combined Image JSON + Filesystem Changeset Format' [1] to
unpack and pack image and this patch switches to the format used by the registry.

We used the 'repository' file which is not generated by Skopeo when it
pulls an image. Moreover, all information of this file are also in the
manifest.json file.
We then use the manifest.json file instead of 'repository' file. Note
also the manifest.json file is required to push an image with Skopeo.

Fix #29636

[1] https://github.com/moby/moby/blob/749d90e10f989802638ae542daf54257f3bf71f2/image/spec/v1.1.md#combined-image-json--filesystem-changeset-format
  • Loading branch information
@nlewo @globin
nlewo authored and globin committed Sep 23, 2017
1 parent f7d7c7b commit 35f205a
Showing 1 changed file with 34 additions and 77 deletions.
111 changes: 34 additions & 77 deletions pkgs/build-support/docker/default.nix
View file
Expand Up @@ -137,7 +137,7 @@ rec {
};
inherit fromImage fromImageName fromImageTag;

buildInputs = [ utillinux e2fsprogs jshon rsync ];
buildInputs = [ utillinux e2fsprogs jshon rsync jq ];
} ''
rm -rf $out
Expand All @@ -146,44 +146,29 @@ rec {
mount /dev/${vmTools.hd} disk
cd disk
layers=""
if [[ -n "$fromImage" ]]; then
echo "Unpacking base image..."
mkdir image
tar -C image -xpf "$fromImage"
# If the image name isn't set, read it from the image repository json.
if [[ -z "$fromImageName" ]]; then
fromImageName=$(jshon -k < image/repositories | head -n 1)
echo "From-image name wasn't set. Read $fromImageName."
fi
# If the tag isn't set, use the name as an index into the json
# and read the first key found.
if [[ -z "$fromImageTag" ]]; then
fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
| head -n1)
echo "From-image tag wasn't set. Read $fromImageTag."
fi
# Use the name and tag to get the parent ID field.
parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
< image/repositories)
layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
fi
# Unpack all of the parent layers into the image.
# Unpack all of the layers into the image.
# Layer list is ordered starting from the base image
lowerdir=""
while [[ -n "$parentID" ]]; do
echo "Unpacking layer $parentID"
mkdir -p image/$parentID/layer
tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
rm image/$parentID/layer.tar
for layer in $layers; do
echo "Unpacking layer $layer"
layerDir=image/$(echo $layer | cut -d':' -f2)"_unpacked"
mkdir -p $layerDir
tar -C $layerDir -xpf image/$layer
chmod a+w image/$layer
rm image/$layer
find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
find $layerDir -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
# Get the next lower directory and continue the loop.
lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
parentID=$(cat image/$parentID/json \
| (jshon -e parent -u 2>/dev/null || true))
lowerdir=$lowerdir''${lowerdir:+:}$layerDir
done
mkdir work
Expand Down Expand Up @@ -461,26 +446,17 @@ rec {
mkdir image
touch baseFiles
layers=""
if [[ -n "$fromImage" ]]; then
echo "Unpacking base image..."
tar -C image -xpf "$fromImage"
# Do not import the base image configuration and manifest
chmod a+w image image/*.json
rm -f image/*.json
if [[ -z "$fromImageName" ]]; then
fromImageName=$(jshon -k < image/repositories|head -n1)
fi
if [[ -z "$fromImageTag" ]]; then
fromImageTag=$(jshon -e $fromImageName -k \
< image/repositories|head -n1)
fi
parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
< image/repositories)
for l in image/*/layer.tar; do
ls_tar $l >> baseFiles
config=$(jq -r '.[0].Config' image/manifest.json)
layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
for l in $layers; do
ls_tar image/$l >> baseFiles
done
chmod u+w image image/$config
rm image/$config
fi
chmod -R ug+rw image
Expand All @@ -507,47 +483,28 @@ rec {
tar -rpf temp/layer.tar --mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 --no-recursion --files-from newFiles
echo "Adding meta..."
# If we have a parentID, add it to the json metadata.
if [[ -n "$parentID" ]]; then
cat temp/json | jshon -s "$parentID" -i parent > tmpjson
mv tmpjson temp/json
fi
# Take the sha256 sum of the generated json and use it as the layer ID.
# Compute the size and add it to the json under the 'Size' field.
layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
size=$(stat --printf="%s" temp/layer.tar)
cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
mv tmpjson temp/json
# Use the temp folder we've been working on to create a new image.
mv temp image/$layerID
gzip temp/layer.tar
layerID="sha256:$(sha256sum temp/layer.tar.gz | cut -d ' ' -f 1)"
mv temp/layer.tar.gz image/$layerID
# Create image json and image manifest
echo "Generating image configuration and manifest..."
imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
currentID=$layerID
while [[ -n "$currentID" ]]; do
layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
# The layer list is ordered starting from the base image
layers=$(echo $layers $layerID)
for i in $(echo $layers); do
imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
diffId=$(gzip -dc image/$i | sha256sum | cut -d" " -f1)
imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$diffId\"] + .")
manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$i\"] + .")
done
imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
echo "$imageJson" > "image/$imageJsonChecksum.json"
manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
echo "$imageJson" > "image/sha256:$imageJsonChecksum"
manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"sha256:$imageJsonChecksum\"")
echo "$manifestJson" > image/manifest.json
# Store the json under the name image/repositories.
jshon -n object \
-n object -s "$layerID" -i "$imageTag" \
-i "$imageName" > image/repositories
# Make the image read-only.
chmod -R a-w image
Expand Down

0 comments on commit 35f205a

Please sign in to comment.