Merge pull request #18315 from jfrazelle/change-frozen-image-v2

update download-frozen-image.sh to v2 registry

Dockerfile

@@ -49,6 +49,7 @@ RUN apt-get update && apt-get install -y \
 	gcc-mingw-w64 \
 	git \
 	iptables \
+	jq \
 	libapparmor-dev \
 	libcap-dev \
 	libltdl-dev \
@@ -175,11 +176,11 @@ RUN ln -sfv $PWD/.bashrc ~/.bashrc
 RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
-COPY contrib/download-frozen-image.sh /go/src/github.com/docker/docker/contrib/
+COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
-RUN ./contrib/download-frozen-image.sh /docker-frozen-images \
+RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
-	busybox:latest@d7057cb020844f245031d27b76cb18af05db1cc3a96a29fa7777af75f5ac91a3 \
+	busybox:latest@sha256:eb3c0d4680f9213ee5f348ea6d39489a1f85a318a2ae09e012c426f78252a6d2 \
-	hello-world:frozen@91c95931e552b11604fea91c2f537284149ec32fff0f700a4769cfd31d7696ae \
+	hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7 \
-	jess/unshare@5c9f6ea50341a2a8eb6677527f2bdedbf331ae894a41714fda770fb130f3314d
+	jess/unshare:latest@sha256:2e3a8c0591c4690b82d4eba7e5ef8f49f2ddfe9f867f3e865198db9bd1436c5b
 # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)
 
 # Download man page generator

contrib/download-frozen-image-v2.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+set -e
+
+# hello-world                      latest              ef872312fe1b        3 months ago        910 B
+# hello-world                      latest              ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9   3 months ago        910 B
+
+# debian                           latest              f6fab3b798be        10 weeks ago        85.1 MB
+# debian                           latest              f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd   10 weeks ago        85.1 MB
+
+if ! command -v curl &> /dev/null; then
+	echo >&2 'error: "curl" not found!'
+	exit 1
+fi
+
+usage() {
+	echo "usage: $0 dir image[:tag][@digest] ..."
+	echo "       $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7"
+	[ -z "$1" ] || exit "$1"
+}
+
+dir="$1" # dir for building tar in
+shift || usage 1 >&2
+
+[ $# -gt 0 -a "$dir" ] || usage 2 >&2
+mkdir -p "$dir"
+
+# hacky workarounds for Bash 3 support (no associative arrays)
+images=()
+rm -f "$dir"/tags-*.tmp
+# repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."'
+
+while [ $# -gt 0 ]; do
+	imageTag="$1"
+	shift
+	image="${imageTag%%[:@]*}"
+	imageTag="${imageTag#*:}"
+	digest="${imageTag##*@}"
+	tag="${imageTag%%@*}"
+
+	# add prefix library if passed official image
+	if [[ "$image" != *"/"* ]]; then
+		image="library/$image"
+	fi
+
+	imageFile="${image//\//_}" # "/" can't be in filenames :)
+
+	token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)"
+
+	manifestJson="$(curl -sSL -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/manifests/$digest")"
+	if [ "${manifestJson:0:1}" != '{' ]; then
+		echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
+		echo >&2 "  $manifestJson"
+		exit 1
+	fi
+
+	layersFs=$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')
+
+	IFS=$'\n'
+	layers=( ${layersFs} )
+	unset IFS
+
+	history=$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')
+	imageId=$(echo "$history" | jq --raw-output .[0] | jq --raw-output .id)
+
+	if [ -s "$dir/tags-$imageFile.tmp" ]; then
+		echo -n ', ' >> "$dir/tags-$imageFile.tmp"
+	else
+		images=( "${images[@]}" "$image" )
+	fi
+	echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp"
+
+	echo "Downloading '${image}:${tag}@${digest}' (${#layers[@]} layers)..."
+	for i in "${!layers[@]}"; do
+		imageJson=$(echo "$history" | jq --raw-output .[${i}])
+		imageId=$(echo "$imageJson" | jq --raw-output .id)
+		imageLayer=${layers[$i]}
+
+		mkdir -p "$dir/$imageId"
+		echo '1.0' > "$dir/$imageId/VERSION"
+
+		echo "$imageJson" > "$dir/$imageId/json"
+
+		# TODO figure out why "-C -" doesn't work here
+		# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
+		# "HTTP/1.1 416 Requested Range Not Satisfiable"
+		if [ -f "$dir/$imageId/layer.tar" ]; then
+			# TODO hackpatch for no -C support :'(
+			echo "skipping existing ${imageId:0:12}"
+			continue
+		fi
+		curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C -
+	done
+	echo
+done
+
+echo -n '{' > "$dir/repositories"
+firstImage=1
+for image in "${images[@]}"; do
+	imageFile="${image//\//_}" # "/" can't be in filenames :)
+	image="${image#library\/}"
+
+	[ "$firstImage" ] || echo -n ',' >> "$dir/repositories"
+	firstImage=
+	echo -n $'\n\t' >> "$dir/repositories"
+	echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories"
+done
+echo -n $'\n}\n' >> "$dir/repositories"
+
+rm -f "$dir"/tags-*.tmp
+
+echo "Download of images into '$dir' complete."
+echo "Use something like the following to load the result into a Docker daemon:"
+echo "  tar -cC '$dir' . | docker load"

hack/make/.ensure-frozen-images

@@ -4,17 +4,17 @@ set -e
 # this list should match roughly what's in the Dockerfile (minus the explicit image IDs, of course)
 images=(
 	busybox:latest
-	hello-world:frozen
+	hello-world:latest
 	jess/unshare:latest
 )
 
 # on ARM we need images that work for the ARM architecture
-if [ -v DOCKER_ENGINE_OSARCH ] && [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then
+if [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then
-  images=(
+	images=(
-    hypriot/armhf-busybox@ea0800bb83571c585c5652b53668e76b29c7c0eef719892f9d0a48607984f9e1
+		hypriot/armhf-busybox@ea0800bb83571c585c5652b53668e76b29c7c0eef719892f9d0a48607984f9e1
-    hypriot/armhf-hello-world@508c59a4f8b23c77bbcf43296c3f580873dc7eecb1f0d680cea3067e221fd4c2
+		hypriot/armhf-hello-world@508c59a4f8b23c77bbcf43296c3f580873dc7eecb1f0d680cea3067e221fd4c2
-    hypriot/armhf-unshare@3f1db65f8bbabc743fd739cf7145a56c35b2a0979ae3174e9d79b7fa4b00fca1
+		hypriot/armhf-unshare@3f1db65f8bbabc743fd739cf7145a56c35b2a0979ae3174e9d79b7fa4b00fca1
-  )
+	)
 fi
 
 if ! docker inspect "${images[@]}" &> /dev/null; then
@@ -23,10 +23,10 @@ if ! docker inspect "${images[@]}" &> /dev/null; then
 		( set -x; tar -cC "$hardCodedDir" . | docker load )
 	else
 		dir="$DEST/frozen-images"
-		# extract the exact "RUN download-frozen-image.sh" line from the Dockerfile itself for consistency
+		# extract the exact "RUN download-frozen-image-v2.sh" line from the Dockerfile itself for consistency
-		# NOTE: this will fail if either "curl" is not installed or if the Dockerfile is not available/readable
+		# NOTE: this will fail if either "curl" or "jq" is not installed or if the Dockerfile is not available/readable
 		awk '
-			$1 == "RUN" && $2 == "./contrib/download-frozen-image.sh" {
+			$1 == "RUN" && $2 == "./contrib/download-frozen-image-v2.sh" {
 				for (i = 2; i < NF; i++)
 					printf ( $i == "'"$hardCodedDir"'" ? "'"$dir"'" : $i ) " ";
 				print $NF;
@@ -46,14 +46,16 @@ if ! docker inspect "${images[@]}" &> /dev/null; then
 	fi
 fi
 
-if [ -v DOCKER_ENGINE_OSARCH ] && [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then
+if [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then
-  # tag images to ensure that all integrations work with the defined image names
+	# tag images to ensure that all integrations work with the defined image names
-  docker tag hypriot/armhf-busybox:latest busybox:latest
+	docker tag hypriot/armhf-busybox:latest busybox:latest
-  docker tag hypriot/armhf-hello-world:latest hello-world:frozen
+	docker tag hypriot/armhf-hello-world:latest hello-world:frozen
-  docker tag hypriot/armhf-unshare:latest jess/unshare:latest
+	docker tag hypriot/armhf-unshare:latest jess/unshare:latest
 
-  # remove orignal tags as these make problems with later tests: TestInspectApiImageResponse
+	# remove orignal tags as these make problems with later tests: TestInspectApiImageResponse
-  docker rmi hypriot/armhf-busybox:latest
+	docker rmi hypriot/armhf-busybox:latest
-  docker rmi hypriot/armhf-hello-world:latest
+	docker rmi hypriot/armhf-hello-world:latest
-  docker rmi hypriot/armhf-unshare:latest
+	docker rmi hypriot/armhf-unshare:latest
+else
+	docker tag hello-world:latest hello-world:frozen
 fi

integration-cli/docker_cli_run_test.go

@@ -2868,7 +2868,7 @@ func (s *DockerSuite) TestRunUnshareProc(c *check.C) {
 
 	/* Ensure still fails if running privileged with the default policy */
 	name = "crashoverride"
-	if out, _, err := dockerCmdWithError("run", "--privileged", "--security-opt", "apparmor:docker-default", "--name", name, "jess/unshare", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc"); err == nil || !(strings.Contains(out, "Permission denied") || strings.Contains(out, "Operation not permitted")) {
+	if out, _, err := dockerCmdWithError("run", "--privileged", "--security-opt", "apparmor:docker-default", "--name", name, "jess/unshare", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc"); err == nil || !(strings.Contains(strings.ToLower(out), "permission denied") || strings.Contains(strings.ToLower(out), "operation not permitted")) {
 		c.Fatalf("unshare should have failed with permission denied, got: %s, %v", out, err)
 	}
 }