From 0af5db511ed1ed5beab0feb09a2a96347a263410 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Tue, 12 Dec 2017 05:41:05 +0000 Subject: [PATCH] Download support of images with multi-arch manifest Currently we only support 'application/vnd.docker.distribution.manifest.v2+json' manifest images download, with more multi-arch images used, we need to support download images with 'application/vnd.docker.distribution.manifest.list.v2+json' format(aka "fat manifest"), else we will fail to download those multi-arch ones. This PR adds 'application/vnd.docker.distribution.manifest.list.v2+json' manifest support, thus we can download both multi-arch and legacy images. Signed-off-by: Dennis Chen --- contrib/download-frozen-image-v2.sh | 220 ++++++++++++++++------------ hack/make/.detect-daemon-osarch | 4 +- 2 files changed, 132 insertions(+), 92 deletions(-) diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh index 2dfe247b5a..22a4b78273 100755 --- a/contrib/download-frozen-image-v2.sh +++ b/contrib/download-frozen-image-v2.sh @@ -6,7 +6,6 @@ set -eo pipefail # debian latest f6fab3b798be 10 weeks ago 85.1 MB # debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB - if ! command -v curl &> /dev/null; then echo >&2 'error: "curl" not found!' exit 1 @@ -80,6 +79,109 @@ fetch_blob() { fi } +# handle 'application/vnd.docker.distribution.manifest.v2+json' manifest +handle_single_manifest_v2() { + local manifestJson="$1"; shift + + local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" + local imageId="${configDigest#*:}" # strip off "sha256:" + + local configFile="$imageId.json" + fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s + + local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" + local IFS="$newlineIFS" + local layers=( $layersFs ) + unset IFS + + echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." + local layerId= + local layerFiles=() + for i in "${!layers[@]}"; do + local layerMeta="${layers[$i]}" + + local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" + local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" + + # save the previous layer's ID + local parentId="$layerId" + # create a new fake layer ID based on this layer's digest and the previous layer's fake ID + layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" + # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) + + mkdir -p "$dir/$layerId" + echo '1.0' > "$dir/$layerId/VERSION" + + if [ ! -s "$dir/$layerId/json" ]; then + local parentJson="$(printf ', parent: "%s"' "$parentId")" + local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" + # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers + jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON' + { + "created": "0001-01-01T00:00:00Z", + "container_config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": null, + "Image": "", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + } + } + EOJSON + fi + + case "$layerMediaType" in + application/vnd.docker.image.rootfs.diff.tar.gzip) + local layerTar="$layerId/layer.tar" + layerFiles=( "${layerFiles[@]}" "$layerTar" ) + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$layerTar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${layerId:0:12}" + continue + fi + local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress + ;; + + *) + echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" + exit 1 + ;; + esac + done + + # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) + imageId="$layerId" + + # munge the top layer image manifest to have the appropriate image configuration for older daemons + local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" + jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" + + local manifestJsonEntry="$( + echo '{}' | jq --raw-output '. + { + Config: "'"$configFile"'", + RepoTags: ["'"${image#library\/}:$tag"'"], + Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' + }' + )" + manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" ) +} + while [ $# -gt 0 ]; do imageTag="$1" shift @@ -101,6 +203,7 @@ while [ $# -gt 0 ]; do curl -fsSL \ -H "Authorization: Bearer $token" \ -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ "$registryBase/v2/$image/manifests/$digest" )" @@ -119,105 +222,40 @@ while [ $# -gt 0 ]; do case "$mediaType" in application/vnd.docker.distribution.manifest.v2+json) - configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" - imageId="${configDigest#*:}" # strip off "sha256:" - - configFile="$imageId.json" - fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s - - layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" + handle_single_manifest_v2 "$manifestJson" + ;; + application/vnd.docker.distribution.manifest.list.v2+json) + layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')" IFS="$newlineIFS" layers=( $layersFs ) unset IFS - echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." - layerId= - layerFiles=() + found="" + # parse first level multi-arch manifest for i in "${!layers[@]}"; do layerMeta="${layers[$i]}" - - layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" - layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" - - # save the previous layer's ID - parentId="$layerId" - # create a new fake layer ID based on this layer's digest and the previous layer's fake ID - layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" - # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) - - mkdir -p "$dir/$layerId" - echo '1.0' > "$dir/$layerId/VERSION" - - if [ ! -s "$dir/$layerId/json" ]; then - parentJson="$(printf ', parent: "%s"' "$parentId")" - addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" - # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers - jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON' - { - "created": "0001-01-01T00:00:00Z", - "container_config": { - "Hostname": "", - "Domainname": "", - "User": "", - "AttachStdin": false, - "AttachStdout": false, - "AttachStderr": false, - "Tty": false, - "OpenStdin": false, - "StdinOnce": false, - "Env": null, - "Cmd": null, - "Image": "", - "Volumes": null, - "WorkingDir": "", - "Entrypoint": null, - "OnBuild": null, - "Labels": null - } - } - EOJSON + maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')" + if [ "$maniArch" = "$(go env GOARCH)" ]; then + digest="$(echo "$layerMeta" | jq --raw-output '.digest')" + # get second level single manifest + submanifestJson="$( + curl -fsSL \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ + "$registryBase/v2/$image/manifests/$digest" + )" + handle_single_manifest_v2 "$submanifestJson" + found="found" + break fi - - case "$layerMediaType" in - application/vnd.docker.image.rootfs.diff.tar.gzip) - layerTar="$layerId/layer.tar" - layerFiles=( "${layerFiles[@]}" "$layerTar" ) - # TODO figure out why "-C -" doesn't work here - # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." - # "HTTP/1.1 416 Requested Range Not Satisfiable" - if [ -f "$dir/$layerTar" ]; then - # TODO hackpatch for no -C support :'( - echo "skipping existing ${layerId:0:12}" - continue - fi - token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" - fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress - ;; - - *) - echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" - exit 1 - ;; - esac done - - # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) - imageId="$layerId" - - # munge the top layer image manifest to have the appropriate image configuration for older daemons - imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" - jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" - - manifestJsonEntry="$( - echo '{}' | jq --raw-output '. + { - Config: "'"$configFile"'", - RepoTags: ["'"${image#library\/}:$tag"'"], - Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' - }' - )" - manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" ) + if [ -z "$found" ]; then + echo >&2 "error: manifest for $maniArch is not found" + exit 1 + fi ;; - *) echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'" exit 1 diff --git a/hack/make/.detect-daemon-osarch b/hack/make/.detect-daemon-osarch index 26cb30f4cf..7376c1d855 100644 --- a/hack/make/.detect-daemon-osarch +++ b/hack/make/.detect-daemon-osarch @@ -64,7 +64,9 @@ case "$PACKAGE_ARCH" in ;; *) DOCKERFILE="Dockerfile.$PACKAGE_ARCH" - TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH" + if [ "$PACKAGE_ARCH" != "aarch64" ]; then + TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH" + fi ;; esac export DOCKERFILE TEST_IMAGE_NAMESPACE