Refactor multi-arch support (especially for new mostly-official multi-arch official images)

See 99433d2ca2/README.md (architectures-other-than-amd64) for some context of where these images come from.

Signed-off-by: Andrew "Tianon" Page <admwiggin@gmail.com>

Dockerfile

@@ -23,15 +23,14 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-FROM ubuntu:14.04
+FROM ubuntu:trusty
-MAINTAINER Tianon Gravi <admwiggin@gmail.com> (@tianon)
 
 # add zfs ppa
 RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61
 RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list
 
 # add llvm repo
-RUN	apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 15CF4D18AF4F7421
+RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421
 RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list
 
 # Packaged dependencies
@@ -72,19 +71,25 @@ RUN apt-get update && apt-get install -y \
 	&& ln -snf /usr/bin/clang++-3.8 /usr/local/bin/clang++
 
 # Get lvm2 source for compiling statically
-RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2
+ENV LVM2_VERSION 2.02.103
+RUN mkdir -p /usr/local/lvm2 \
+	&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
+		| tar -xzC /usr/local/lvm2 --strip-components=1
 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
 
 # Compile and install lvm2
 RUN cd /usr/local/lvm2 \
-	&& ./configure --enable-static_link \
+	&& ./configure \
+		--build="$(gcc -print-multiarch)" \
+		--enable-static_link \
 	&& make device-mapper \
 	&& make install_device-mapper
 # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
 
 # Install Go
 ENV GO_VERSION 1.5.2
-RUN curl -sSL  "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar -v -C /usr/local -xz
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" \
+	| tar -xzC /usr/local
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go:/go/src/github.com/docker/docker/vendor
 
@@ -128,17 +133,17 @@ ENV PATH /osxcross/target/bin:$PATH
 # install seccomp
 # this can be changed to the ubuntu package libseccomp-dev if dockerinit is removed,
 # we need libseccomp.a (which the package does not provide) for dockerinit
-ENV SECCOMP_VERSION v2.2.3
+ENV SECCOMP_VERSION 2.2.3
 RUN set -x \
-	&& export SECCOMP_PATH=$(mktemp -d) \
+	&& export SECCOMP_PATH="$(mktemp -d)" \
-	&& git clone https://github.com/seccomp/libseccomp.git "$SECCOMP_PATH" \
+	&& curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \
+		| tar -xzC "$SECCOMP_PATH" --strip-components=1 \
 	&& ( \
 		cd "$SECCOMP_PATH" \
-		&& git checkout "$SECCOMP_VERSION" \
+		&& ./configure --prefix=/usr/local \
-		&& ./autogen.sh \
-		&& ./configure --prefix=/usr \
 		&& make \
 		&& make install \
+		&& ldconfig \
 	) \
 	&& rm -rf "$SECCOMP_PATH"
 
@@ -196,7 +201,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
-	busybox:latest@sha256:eb3c0d4680f9213ee5f348ea6d39489a1f85a318a2ae09e012c426f78252a6d2 \
+	busybox:latest@sha256:e4f93f6ed15a0cdd342f5aae387886fba0ab98af0a102da6276eaf24d6e6ade0 \
 	debian:jessie@sha256:24a900d1671b269d6640b4224e7b63801880d8e3cb2bcbfaa10a5dddcf4469ed \
 	hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7
 # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)
@@ -204,8 +209,8 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 # Download man page generator
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
-	&& git clone -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
+	&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
-	&& git clone -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
+	&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
 	&& go get -v -d github.com/cpuguy83/go-md2man \
 	&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
 	&& rm -rf "$GOPATH"
@@ -220,12 +225,12 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Build/install the tool for embedding resources in Windows binaries
-ENV RSRC_COMMIT e48dbf1b7fc464a9e85fcec450dddf80816b76e0
+ENV RSRC_VERSION v2
 RUN set -x \
-	&& git clone https://github.com/akavel/rsrc.git /go/src/github.com/akavel/rsrc \
+	&& export GOPATH="$(mktemp -d)" \
-	&& cd /go/src/github.com/akavel/rsrc \
+	&& git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
-	&& git checkout -q $RSRC_COMMIT \
+	&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
-	&& go install -v
+	&& rm -rf "$GOPATH"
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]

Dockerfile.armhf

@@ -23,8 +23,7 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-FROM ioft/armhf-ubuntu:14.04
+FROM armhf/ubuntu:trusty
-MAINTAINER Govinda Fichtner <govinda.fichtner@googlemail.com> (@_beagile_)
 
 # Packaged dependencies
 RUN apt-get update && apt-get install -y \
@@ -40,6 +39,7 @@ RUN apt-get update && apt-get install -y \
 	git \
 	iptables \
 	jq \
+	net-tools \
 	libapparmor-dev \
 	libcap-dev \
 	libltdl-dev \
@@ -52,38 +52,47 @@ RUN apt-get update && apt-get install -y \
 	python-mock \
 	python-pip \
 	python-websocket \
-	s3cmd=1.1.0* \
 	xfsprogs \
 	tar \
 	--no-install-recommends
 
 # Get lvm2 source for compiling statically
-RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2
+ENV LVM2_VERSION 2.02.103
+RUN mkdir -p /usr/local/lvm2 \
+	&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
+		| tar -xzC /usr/local/lvm2 --strip-components=1
 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
 
 # Compile and install lvm2
 RUN cd /usr/local/lvm2 \
-	&& ./configure --enable-static_link \
+	&& ./configure \
+		--build="$(gcc -print-multiarch)" \
+		--enable-static_link \
 	&& make device-mapper \
 	&& make install_device-mapper
 # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
 
 # Install Go
-ENV GO_VERSION 1.4.3
+#ENV GO_VERSION 1.5.2
 # TODO update GO_TOOLS_COMMIT below when this updates to 1.5+
-RUN curl -fsSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz
+ENV GO_VERSION 1.4.3
+RUN curl -fsSL "https://github.com/hypriot/golang-armbuilds/releases/download/v${GO_VERSION}/go${GO_VERSION}.linux-armv7.tar.gz" \
+	| tar -xzC /usr/local
+# temporarily using Hypriot's tarballs while we wait for official 1.6+
+#RUN curl -fsSL https://golang.org/dl/go${GO_VERSION}.linux-arm6.tar.gz \
+#		| tar -xzC /usr/local
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go:/go/src/github.com/docker/docker/vendor
 
 # we're building for armhf, which is ARMv7, so let's be explicit about that
+ENV GOARCH arm
 ENV GOARM 7
 
-RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1
-
 # This has been commented out and kept as reference because we don't support compiling with older Go anymore.
 # ENV GOFMT_VERSION 1.3.3
 # RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt
 
+#ENV GO_TOOLS_COMMIT 823804e1ae08dbb14eb807afc7db9993bc9e3cc3
 # TODO update this sha when we upgrade to Go 1.5+
 ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9
 # Grab Go's cover tool for dead-simple code coverage testing
@@ -94,6 +103,8 @@ RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
 	&& go install -v golang.org/x/tools/cmd/cover \
 	&& go install -v golang.org/x/tools/cmd/vet
 # Grab Go's lint tool
+#ENV GO_LINT_COMMIT 32a87160691b3c96046c0c678fe57c5bef761456
+# TODO update this sha when we upgrade to Go 1.5+
 ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f
 RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \
 	&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
@@ -102,17 +113,17 @@ RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint
 # install seccomp
 # this can be changed to the ubuntu package libseccomp-dev if dockerinit is removed,
 # we need libseccomp.a (which the package does not provide) for dockerinit
-ENV SECCOMP_VERSION v2.2.3
+ENV SECCOMP_VERSION 2.2.3
 RUN set -x \
-	&& export SECCOMP_PATH=$(mktemp -d) \
+	&& export SECCOMP_PATH="$(mktemp -d)" \
-	&& git clone https://github.com/seccomp/libseccomp.git "$SECCOMP_PATH" \
+	&& curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \
+		| tar -xzC "$SECCOMP_PATH" --strip-components=1 \
 	&& ( \
 		cd "$SECCOMP_PATH" \
-		&& git checkout "$SECCOMP_VERSION" \
+		&& ./configure --prefix=/usr/local \
-		&& ./autogen.sh \
-		&& ./configure --prefix=/usr \
 		&& make \
 		&& make install \
+		&& ldconfig \
 	) \
 	&& rm -rf "$SECCOMP_PATH"
 
@@ -127,29 +138,21 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Install notary server
-# commented Notary temporary as we are waiting for an update of jose2go: https://github.com/docker/notary/issues/239
+ENV NOTARY_COMMIT f211b1826dde5fc8c117ccff9bb04ae458a8e3d0
-#
+RUN set -x \
-# ENV NOTARY_COMMIT 8e8122eb5528f621afcd4e2854c47302f17392f7
+	&& export GOPATH="$(mktemp -d)" \
-# RUN set -x \
+	&& git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \
-# 	&& export GOPATH="$(mktemp -d)" \
+	&& (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_COMMIT") \
-# 	&& git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \
+	&& GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \
-# 	&& (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_COMMIT") \
+		go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \
-# 	&& GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \
+	&& rm -rf "$GOPATH"
-# 		go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \
-# 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 139850f3f3b17357bab5ba3edfb745fb14043764
+ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
-	&& git checkout -q $DOCKER_PY_COMMIT
+	&& git checkout -q $DOCKER_PY_COMMIT \
-
+	&& pip install -r test-requirements.txt
-# Setup s3cmd config
-RUN { \
-		echo '[default]'; \
-		echo 'access_key=$AWS_ACCESS_KEY'; \
-		echo 'secret_key=$AWS_SECRET_KEY'; \
-	} > ~/.s3cfg
 
 # Set user.email so crosbymichael's in-container merge commits go smoothly
 RUN git config --global user.email 'docker-dummy@example.com'
@@ -171,16 +174,16 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
-	hypriot/armhf-busybox:latest@sha256:b0fc94dac9793ce3c35607b15012b4c7deca300963a7cc38ab440189ec81e2e7 \
+	armhf/busybox:latest@sha256:d98a7343ac750ffe387e3d514f8521ba69846c216778919b01414b8617cfb3d4 \
-	hypriot/armhf-hello-world:latest@sha256:b618ec0cc3acf683e8d77ad6c5ec81546cddde2036eda9a78f628effdeca74cd \
+	armhf/debian:jessie@sha256:094687129906d2a43cb4e5946ea379b5619c9ca8e4e27b3ba28b40f237a4150c \
-	hypriot/armhf-unshare:latest@sha256:8fede091760d2fb8b2d14cedffdd681c4575b02b1abeeb18dd79b754c62327db
+	armhf/hello-world:latest@sha256:161dcecea0225975b2ad5f768058212c1e0d39e8211098666ffa1ac74cfb7791
 # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)
 
 # Download man page generator
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
-	&& git clone -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
+	&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
-	&& git clone -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
+	&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
 	&& go get -v -d github.com/cpuguy83/go-md2man \
 	&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
 	&& rm -rf "$GOPATH"
@@ -195,12 +198,12 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Build/install the tool for embedding resources in Windows binaries
-ENV RSRC_COMMIT e48dbf1b7fc464a9e85fcec450dddf80816b76e0
+ENV RSRC_VERSION v2
 RUN set -x \
-	&& git clone https://github.com/akavel/rsrc.git /go/src/github.com/akavel/rsrc \
+	&& export GOPATH="$(mktemp -d)" \
-	&& cd /go/src/github.com/akavel/rsrc \
+	&& git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
-	&& git checkout -q $RSRC_COMMIT \
+	&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
-	&& go install -v
+	&& rm -rf "$GOPATH"
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]

Dockerfile.ppc64le

@@ -1,10 +1,27 @@
-# This file describes the standard way to build Docker, using docker
+# This file describes the standard way to build Docker on ppc64le, using docker
 #
 # Usage:
 #
 # # Assemble the full dev environment. This is slow the first time.
 # docker build -t docker -f Dockerfile.ppc64le .
 #
+# # Mount your source in an interactive container for quick testing:
+# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
+#
+# # Run the test suite:
+# docker run --privileged docker hack/make.sh test
+#
+# # Publish a release:
+# docker run --privileged \
+#  -e AWS_S3_BUCKET=baz \
+#  -e AWS_ACCESS_KEY=foo \
+#  -e AWS_SECRET_KEY=bar \
+#  -e GPG_PASSPHRASE=gloubiboulga \
+#  docker hack/release.sh
+#
+# Note: AppArmor used to mess with privileged mode, but this is no longer
+# the case. Therefore, you don't have to disable it anymore.
+#
 
 FROM ppc64le/gcc:5.3
 
@@ -12,37 +29,83 @@ FROM ppc64le/gcc:5.3
 RUN apt-get update && apt-get install -y \
 	apparmor \
 	aufs-tools \
+	automake \
+	bash-completion \
 	btrfs-tools \
 	build-essential \
+	createrepo \
 	curl \
+	dpkg-sig \
 	git \
 	iptables \
 	jq \
 	net-tools \
 	libapparmor-dev \
 	libcap-dev \
+	libltdl-dev \
 	libsqlite3-dev \
+	libsystemd-journal-dev \
+	libtool \
 	mercurial \
-	parallel \
+	pkg-config \
 	python-dev \
 	python-mock \
 	python-pip \
 	python-websocket \
+	xfsprogs \
+	tar \
 	--no-install-recommends
 
-RUN rm -rf /usr/local/lvm2
+# Get lvm2 source for compiling statically
-RUN git clone --no-checkout git://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout -q v2_02_103
+ENV LVM2_VERSION 2.02.103
-RUN curl -o /usr/local/lvm2/autoconf/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'
+RUN mkdir -p /usr/local/lvm2 \
-RUN curl -o /usr/local/lvm2/autoconf/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'
+	&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
+		| tar -xzC /usr/local/lvm2 --strip-components=1
+# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
+
+# fix platform enablement in lvm2 to support ppc64le properly
+RUN set -e \
+	&& for f in config.guess config.sub; do \
+		curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \
+	done
+# "arch.c:78:2: error: #error the arch code needs to know about your machine type"
 
 # Compile and install lvm2
 RUN cd /usr/local/lvm2 \
-	&& ./configure --enable-static_link \
+	&& ./configure \
+		--build="$(gcc -print-multiarch)" \
+		--enable-static_link \
 	&& make device-mapper \
 	&& make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
 
+# TODO install Go, using gccgo as GOROOT_BOOTSTRAP (Go 1.5+ supports ppc64le properly)
+# possibly a ppc64le/golang image?
+
+ENV PATH /go/bin:$PATH
 ENV GOPATH /go:/go/src/github.com/docker/docker/vendor
 
+# This has been commented out and kept as reference because we don't support compiling with older Go anymore.
+# ENV GOFMT_VERSION 1.3.3
+# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt
+
+# TODO update this sha when we upgrade to Go 1.5+
+ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9
+# Grab Go's cover tool for dead-simple code coverage testing
+# Grab Go's vet tool for examining go code to find suspicious constructs
+# and help prevent errors that the compiler might not catch
+RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
+	&& (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \
+	&& go install -v golang.org/x/tools/cmd/cover \
+	&& go install -v golang.org/x/tools/cmd/vet
+# Grab Go's lint tool
+ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f
+RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \
+	&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
+	&& go install -v github.com/golang/lint/golint
+
+
+# Install registry
 ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
@@ -63,12 +126,15 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 47ab89ec2bd3bddf1221b856ffbaff333edeabb4
+ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
 	&& git checkout -q $DOCKER_PY_COMMIT \
 	&& pip install -r test-requirements.txt
 
+# Set user.email so crosbymichael's in-container merge commits go smoothly
+RUN git config --global user.email 'docker-dummy@example.com'
+
 # Add an unprivileged user to be used for tests which need it
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser
@@ -77,12 +143,45 @@ VOLUME /var/lib/docker
 WORKDIR /go/src/github.com/docker/docker
 ENV DOCKER_BUILDTAGS apparmor selinux
 
-ENV IMAGEREPO ppc64le
+# Let us use a .bashrc file
+RUN ln -sfv $PWD/.bashrc ~/.bashrc
+
+# Register Docker's bash completion.
+RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
+
+# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
-	$IMAGEREPO/busybox:latest \
+	ppc64le/busybox:latest@sha256:38bb82085248d5a3c24bd7a5dc146f2f2c191e189da0441f1c2ca560e3fc6f1b \
-	$IMAGEREPO/hello-world:frozen \ 
+	ppc64le/debian:jessie@sha256:74e06e6506b23cf8abd00250782838b2d19910824d8e7eab3d14dc1845ea10c6 \
-	$IMAGEREPO/unshare:latest	
+	ppc64le/hello-world:latest@sha256:186a40a9a02ca26df0b6c8acdfb8ac2f3ae6678996a838f977e57fac9d963974
+# see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)
+
+# Download man page generator
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
+	&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
+	&& go get -v -d github.com/cpuguy83/go-md2man \
+	&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
+	&& rm -rf "$GOPATH"
+
+# Download toml validator
+ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \
+	&& (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \
+	&& go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \
+	&& rm -rf "$GOPATH"
+
+# Build/install the tool for embedding resources in Windows binaries
+ENV RSRC_VERSION v2
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
+	&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
+	&& rm -rf "$GOPATH"
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]

Dockerfile.s390x

@@ -1,47 +1,111 @@
-# This file describes the standard way to build Docker, using docker
+# This file describes the standard way to build Docker on s390x, using docker
 #
 # Usage:
 #
 # # Assemble the full dev environment. This is slow the first time.
 # docker build -t docker -f Dockerfile.s390x .
 #
+# # Mount your source in an interactive container for quick testing:
+# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
+#
+# # Run the test suite:
+# docker run --privileged docker hack/make.sh test
+#
+# # Publish a release:
+# docker run --privileged \
+#  -e AWS_S3_BUCKET=baz \
+#  -e AWS_ACCESS_KEY=foo \
+#  -e AWS_SECRET_KEY=bar \
+#  -e GPG_PASSPHRASE=gloubiboulga \
+#  docker hack/release.sh
+#
+# Note: AppArmor used to mess with privileged mode, but this is no longer
+# the case. Therefore, you don't have to disable it anymore.
+#
 
-FROM s390x/gcc:5.2
+FROM s390x/gcc:5.3
 
 # Packaged dependencies
 RUN apt-get update && apt-get install -y \
 	apparmor \
 	aufs-tools \
+	automake \
+	bash-completion \
 	btrfs-tools \
 	build-essential \
+	createrepo \
 	curl \
+	dpkg-sig \
 	git \
 	iptables \
 	jq \
 	net-tools \
 	libapparmor-dev \
 	libcap-dev \
+	libltdl-dev \
 	libsqlite3-dev \
+	libsystemd-journal-dev \
+	libtool \
 	mercurial \
-	parallel \
+	pkg-config \
 	python-dev \
 	python-mock \
 	python-pip \
 	python-websocket \
+	xfsprogs \
+	tar \
 	--no-install-recommends
 
 # Get lvm2 source for compiling statically
-RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2
+ENV LVM2_VERSION 2.02.103
+RUN mkdir -p /usr/local/lvm2 \
+	&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
+		| tar -xzC /usr/local/lvm2 --strip-components=1
 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
 
+# fix platform enablement in lvm2 to support s390x properly
+RUN set -e \
+	&& for f in config.guess config.sub; do \
+		curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \
+	done
+# "arch.c:78:2: error: #error the arch code needs to know about your machine type"
+
 # Compile and install lvm2
 RUN cd /usr/local/lvm2 \
-	&& ./configure --enable-static_link \
+	&& ./configure \
+		--build="$(gcc -print-multiarch)" \
+		--enable-static_link \
 	&& make device-mapper \
 	&& make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
 
+# Note: Go comes from the base image (gccgo, specifically)
+# We can't compile Go proper because s390x isn't an officially supported architecture yet.
+
+ENV PATH /go/bin:$PATH
 ENV GOPATH /go:/go/src/github.com/docker/docker/vendor
 
+# This has been commented out and kept as reference because we don't support compiling with older Go anymore.
+# ENV GOFMT_VERSION 1.3.3
+# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt
+
+# TODO update this sha when we upgrade to Go 1.5+
+ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9
+# Grab Go's cover tool for dead-simple code coverage testing
+# Grab Go's vet tool for examining go code to find suspicious constructs
+# and help prevent errors that the compiler might not catch
+RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
+	&& (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \
+	&& go install -v golang.org/x/tools/cmd/cover \
+	&& go install -v golang.org/x/tools/cmd/vet
+# Grab Go's lint tool
+ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f
+RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \
+	&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
+	&& go install -v github.com/golang/lint/golint
+
+
+# Install registry
 ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
@@ -62,12 +126,15 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 47ab89ec2bd3bddf1221b856ffbaff333edeabb4
+ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
 	&& git checkout -q $DOCKER_PY_COMMIT \
 	&& pip install -r test-requirements.txt
 
+# Set user.email so crosbymichael's in-container merge commits go smoothly
+RUN git config --global user.email 'docker-dummy@example.com'
+
 # Add an unprivileged user to be used for tests which need it
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser
@@ -76,12 +143,45 @@ VOLUME /var/lib/docker
 WORKDIR /go/src/github.com/docker/docker
 ENV DOCKER_BUILDTAGS apparmor selinux
 
-ENV IMAGEREPO s390x
+# Let us use a .bashrc file
+RUN ln -sfv $PWD/.bashrc ~/.bashrc
+
+# Register Docker's bash completion.
+RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
+
+# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
-	$IMAGEREPO/busybox:latest \
+	s390x/busybox:latest@sha256:dd61522c983884a66ed72d60301925889028c6d2d5e0220a8fe1d9b4c6a4f01b \
-	$IMAGEREPO/hello-world:frozen \
+	s390x/debian:jessie@sha256:3c478e199f60c877c00306356267798d32727dc3cd38512cdb4b060659ea9d20 \
-	$IMAGEREPO/unshare:latest 
+	s390x/hello-world:latest@sha256:780d80b3a7677c3788c0d5cd9168281320c8d4a6d9183892d8ee5cdd610f5699
+# see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)
+
+# Download man page generator
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
+	&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
+	&& go get -v -d github.com/cpuguy83/go-md2man \
+	&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
+	&& rm -rf "$GOPATH"
+
+# Download toml validator
+ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \
+	&& (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \
+	&& go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \
+	&& rm -rf "$GOPATH"
+
+# Build/install the tool for embedding resources in Windows binaries
+ENV RSRC_VERSION v2
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
+	&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
+	&& rm -rf "$GOPATH"
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]

Makefile

@@ -1,14 +1,27 @@
 .PHONY: all binary build cross default docs docs-build docs-shell shell test test-docker-py test-integration-cli test-unit validate
 
 # get OS/Arch of docker engine
-DOCKER_ENGINE_OSARCH = $(shell docker version | grep 'OS/Arch' | tail -1 | cut -d':' -f2 | tr -d '[[:space:]]')
+DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH:+$$DOCKER_CLIENT_OSARCH}')
 # default for linux/amd64 and others
-DOCKER_FILE = Dockerfile
+DOCKERFILE := Dockerfile
 # switch to different Dockerfile for linux/arm
-ifeq ($(DOCKER_ENGINE_OSARCH),linux/arm)
+ifeq ($(DOCKER_OSARCH), linux/arm)
-	DOCKER_FILE = Dockerfile.armhf
+	DOCKERFILE := Dockerfile.armhf
+else
+ifeq ($(DOCKER_OSARCH), linux/arm64)
+	# TODO .arm64
+	DOCKERFILE := Dockerfile.armhf
+else
+ifeq ($(DOCKER_OSARCH), linux/ppc64le)
+	DOCKERFILE := Dockerfile.ppc64le
+else
+ifeq ($(DOCKER_OSARCH), linux/s390x)
+	DOCKERFILE := Dockerfile.s390x
 endif
-export DOCKER_FILE
+endif
+endif
+endif
+export DOCKERFILE
 
 # env vars passed through directly to Docker's build scripts
 # to allow things like `make DOCKER_CLIENTONLY=1 binary` easily
@@ -18,7 +31,7 @@ DOCKER_ENVS := \
 	-e DOCKER_CLIENTONLY \
 	-e DOCKER_DEBUG \
 	-e DOCKER_EXPERIMENTAL \
-	-e DOCKER_FILE \
+	-e DOCKERFILE \
 	-e DOCKER_GRAPHDRIVER \
 	-e DOCKER_REMAP_ROOT \
 	-e DOCKER_STORAGE_OPTS \
@@ -60,7 +73,7 @@ binary: build
 	$(DOCKER_RUN_DOCKER) hack/make.sh binary
 
 build: bundles
-	docker build -t "$(DOCKER_IMAGE)" -f $(DOCKER_FILE) .
+	docker build -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
 
 bundles:
 	mkdir bundles

hack/make/.detect-daemon-osarch

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Retrieve OS/ARCH of docker daemon, eg. linux/amd64
+export DOCKER_ENGINE_OSARCH="$(docker version |tac|tac| awk '
+	$1 == "Client:" { server = 0; next }
+	$1 == "Server:" { server = 1; next }
+	server && $1 == "OS/Arch:" { print $2; exit }
+')"
+export DOCKER_ENGINE_GOOS="${DOCKER_ENGINE_OSARCH%/*}"
+export DOCKER_ENGINE_GOARCH="${DOCKER_ENGINE_OSARCH##*/}"
+
+# and the client, just in case
+export DOCKER_CLIENT_OSARCH="$(docker version |tac|tac| awk '
+	$1 == "Client:" { client = 1; next }
+	$1 == "Server:" { client = 0; next }
+	client && $1 == "OS/Arch:" { print $2; exit }
+')"

hack/make/.ensure-frozen-images

@@ -1,38 +1,32 @@
 #!/bin/bash
 set -e
 
-# image lists for different archs that should match what's in the Dockerfile (minus the explicit images IDs)
+# image list should match what's in the Dockerfile (minus the explicit images IDs)
-case "$DOCKER_ENGINE_OSARCH" in
+images=(
-	linux/arm)
-		images=(
-			hypriot/armhf-busybox:latest
-			hypriot/armhf-hello-world:latest
-			hypriot/armhf-unshare:latest
-		)
-		;;
-	linux/ppc64le)
-		images=(
-			ppc64le/busybox:latest
-			ppc64le/hello-world:frozen
-			ppc64le/unshare:latest
-		)
-		;;
-	linux/s390x)
-		images=(
-			s390x/busybox:latest
-			s390x/hello-world:frozen
-			s390x/unshare:latest
-		)
-		;;
-	*)
-		images=(
 	busybox:latest
 	debian:jessie
 	hello-world:latest
-		)
+)
+
+imagePrefix=
+case "$DOCKER_ENGINE_OSARCH" in
+	linux/arm)
+		imagePrefix='armhf'
+		;;
+	linux/ppc64le)
+		imagePrefix='ppc64le'
+		;;
+	linux/s390x)
+		imagePrefix='s390x'
 		;;
 esac
 
+if [ "$imagePrefix" ]; then
+	for (( i = 0; i < ${#images[@]}; i++ )); do
+		images[$i]="$imagePrefix/${images[$i]}"
+	done
+fi
+
 if ! docker inspect "${images[@]}" &> /dev/null; then
 	hardCodedDir='/docker-frozen-images'
 	if [ -d "$hardCodedDir" ]; then
@@ -60,7 +54,7 @@ if ! docker inspect "${images[@]}" &> /dev/null; then
 					inCont = 0;
 				}
 			}
-		' ${DOCKER_FILE:="Dockerfile"} | sh -x
+		' "${DOCKERFILE:=Dockerfile}" | sh -x
 		# Do not use a subshell for the following command. Windows CI
 		# runs bash 3.x so will not trap an error in a subshell.
 		# http://stackoverflow.com/questions/22630363/how-does-set-e-work-with-subshells
@@ -68,35 +62,18 @@ if ! docker inspect "${images[@]}" &> /dev/null; then
 	fi
 fi
 
-# tag images to ensure that all integrations work with the defined image names
+if [ "$imagePrefix" ]; then
-# then remove original tags as these make problems with later tests (e.g., TestInspectApiImageResponse)
+	for image in "${images[@]}"; do
-case "$DOCKER_ENGINE_OSARCH" in
+		target="${image#$imagePrefix/}"
-	linux/arm)
+		if [ "$target" != "$image" ]; then
-		docker tag hypriot/armhf-busybox:latest busybox:latest
+			# tag images to ensure that all integrations work with the defined image names
-		docker tag hypriot/armhf-hello-world:latest hello-world:frozen
+			docker tag "$image" "$target"
-		docker tag hypriot/armhf-unshare:latest jess/unshare:latest
+			# then remove original tags as these make problems with later tests (e.g., TestInspectApiImageResponse)
-		docker rmi hypriot/armhf-busybox:latest
+			docker rmi "$image"
-		docker rmi hypriot/armhf-hello-world:latest
+		fi
-		docker rmi hypriot/armhf-unshare:latest
+	done
-		;;
+fi
-	linux/ppc64le)
+
-		docker tag ppc64le/busybox:latest busybox:latest
+# explicitly rename "hello-world:latest" to ":frozen" for the test that uses it
-		docker tag ppc64le/hello-world:frozen hello-world:frozen
+docker tag hello-world:latest hello-world:frozen
-		docker tag ppc64le/unshare:latest jess/unshare:latest
+docker rmi hello-world:latest
-		docker rmi ppc64le/busybox:latest
-		docker rmi ppc64le/hello-world:frozen
-		docker rmi ppc64le/unshare:latest
-		;;
-	linux/s390x)
-		docker tag s390x/busybox:latest busybox:latest
-		docker tag s390x/hello-world:frozen hello-world:frozen
-		docker tag s390x/unshare:latest jess/unshare:latest
-		docker rmi s390x/busybox:latest
-		docker rmi s390x/hello-world:frozen
-		docker rmi s390x/unshare:latest
-		;;
-	*)
-		docker tag hello-world:latest hello-world:frozen
-		docker rmi hello-world:latest
-		;;
-esac

hack/make/.integration-daemon-setup

@@ -1,12 +1,7 @@
 #!/bin/bash
+set -e
 
-# Retrieve OS/ARCH of docker daemon, eg. linux/amd64
+bundle .detect-daemon-osarch
-export DOCKER_ENGINE_OSARCH=$(docker version | grep 'OS/Arch' | tail -1 | cut -d':' -f2 | tr -d '[[:space:]]')
-# Retrieve OS of docker daemon, eg. linux
-export DOCKER_ENGINE_GOOS=$(echo $DOCKER_ENGINE_OSARCH | cut -d'/' -f1)
-# Retrieve ARCH of docker daemon, eg. amd64
-export DOCKER_ENGINE_GOARCH=$(echo $DOCKER_ENGINE_OSARCH | cut -d'/' -f2)
-
 bundle .ensure-emptyfs
 bundle .ensure-frozen-images
 bundle .ensure-httpserver