Use buildx as Makefile builder

Since the dockerfile now requires buildkit, let's just use buildx can
bootstrap itself into even an old version of Docker which does not
support buildkit.

This also decouples the Dockerfile/build from the version of Docker
which is installed.

One major downside:

If buildx needs to setup a container driver (ie, docker does not support
buildkit), the `make shell` target (and others which call
`DOCKER_RUN_DOCKER`) must export the image from buildkit and into
docker. This added an extra 70s to a full build for me (agan only for targets
which call `DOCKER_RUN_DOCKER`) and 40s on a rebuild (with no changes).

Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is contained in:
Brian Goff 2019-06-07 10:10:31 -07:00
parent 82c56755ea
commit 833444c0d6
3 changed files with 61 additions and 291 deletions

View File

@ -1,278 +0,0 @@
# syntax=docker.io/docker/dockerfile:experimental@sha256:9022e911101f01b2854c7a4b2c77f524b998891941da55208e71c0335e6e82c3
ARG CROSS="false"
FROM golang:1.12.5 AS base
# allow replacing httpredir or deb mirror
ARG APT_MIRROR=deb.debian.org
RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
FROM base AS criu
# Install CRIU for checkpoint/restore support
ENV CRIU_VERSION 3.11
# Install dependency packages specific to criu
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
apt-get update && apt-get install -y \
libnet-dev \
libprotobuf-c0-dev \
libprotobuf-dev \
libnl-3-dev \
libcap-dev \
protobuf-compiler \
protobuf-c-compiler \
python-protobuf \
&& mkdir -p /usr/src/criu \
&& curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \
&& cd /usr/src/criu \
&& make \
&& make PREFIX=/build/ install-criu
FROM base AS registry
ENV REGISTRY_COMMIT 47a064d4195a9b56133891bbb13620c3ac83a827
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \
&& (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \
&& GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
go build -buildmode=pie -o /build/registry-v2 github.com/docker/distribution/cmd/registry \
&& rm -rf "$GOPATH"
FROM base AS docker-py
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT ac922192959870774ad8428344d9faa0555f7ba6
RUN git clone https://github.com/docker/docker-py.git /build \
&& cd /build \
&& git checkout -q $DOCKER_PY_COMMIT
FROM base AS swagger
# Install go-swagger for validating swagger.yaml
ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \
&& (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \
&& go build -o /build/swagger github.com/go-swagger/go-swagger/cmd/swagger \
&& rm -rf "$GOPATH"
FROM base AS frozen-images
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
apt-get update && apt-get install -y jq ca-certificates --no-install-recommends
# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
COPY contrib/download-frozen-image-v2.sh /
RUN /download-frozen-image-v2.sh /build \
buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \
debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
# See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
FROM base AS cross-false
FROM base AS cross-true
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
dpkg --add-architecture armhf \
&& dpkg --add-architecture arm64 \
&& dpkg --add-architecture armel
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
if [ "$(go env GOHOSTARCH)" = "amd64" ]; then \
apt-get update \
&& apt-get install -y --no-install-recommends \
crossbuild-essential-armhf \
crossbuild-essential-arm64 \
crossbuild-essential-armel; \
fi
FROM cross-${CROSS} as dev-base
FROM dev-base AS runtime-dev-cross-false
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
apt-get update && apt-get install -y \
libapparmor-dev \
libseccomp-dev
FROM cross-true AS runtime-dev-cross-true
# These crossbuild packages rely on gcc-<arch>, but this doesn't want to install
# on non-amd64 systems.
# Additionally, the crossbuild-amd64 is currently only on debian:buster, so
# other architectures cannnot crossbuild amd64.
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
if [ "$(go env GOHOSTARCH)" = "amd64" ]; then \
apt-get update \
&& apt-get install -y \
libseccomp-dev:armhf \
libseccomp-dev:arm64 \
libseccomp-dev:armel \
libapparmor-dev:armhf \
libapparmor-dev:arm64 \
libapparmor-dev:armel \
# install this arches seccomp here due to compat issues with the v0 builder
# This is as opposed to inheriting from runtime-dev-cross-false
libapparmor-dev \
libseccomp-dev; \
fi
FROM runtime-dev-cross-${CROSS} AS runtime-dev
FROM base AS tomlv
ENV INSTALL_BINARY_NAME=tomlv
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM base AS vndr
ENV INSTALL_BINARY_NAME=vndr
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM dev-base AS containerd
RUN apt-get update && apt-get install -y btrfs-tools
ENV INSTALL_BINARY_NAME=containerd
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM dev-base AS proxy
ENV INSTALL_BINARY_NAME=proxy
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM base AS gometalinter
ENV INSTALL_BINARY_NAME=gometalinter
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM dev-base AS dockercli
ENV INSTALL_BINARY_NAME=dockercli
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM runtime-dev AS runc
ENV INSTALL_BINARY_NAME=runc
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM dev-base AS tini
RUN apt-get update && apt-get install -y cmake vim-common
COPY hack/dockerfile/install/install.sh ./install.sh
ENV INSTALL_BINARY_NAME=tini
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
FROM dev-base AS rootlesskit
ENV INSTALL_BINARY_NAME=rootlesskit
COPY hack/dockerfile/install/install.sh ./install.sh
COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
PREFIX=/build/ ./install.sh $INSTALL_BINARY_NAME
COPY ./contrib/dockerd-rootless.sh /build
# TODO: Some of this is only really needed for testing, it would be nice to split this up
FROM runtime-dev AS dev
RUN groupadd -r docker
RUN useradd --create-home --gid docker unprivilegeduser
# Let us use a .bashrc file
RUN ln -sfv /go/src/github.com/docker/docker/.bashrc ~/.bashrc
# Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH
RUN echo "source /usr/share/bash-completion/bash_completion" >> /etc/bash.bashrc
RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker
RUN ldconfig
# This should only install packages that are specifically needed for the dev environment and nothing else
# Do you really need to add another package here? Can it be done in a different build stage?
RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=private \
--mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=private \
apt-get update && apt-get install -y \
apparmor \
aufs-tools \
bash-completion \
btrfs-tools \
iptables \
jq \
libcap2-bin \
libdevmapper-dev \
# libffi-dev and libssl-dev appear to be required for compiling paramiko on s390x/ppc64le
libffi-dev \
libssl-dev \
libudev-dev \
libsystemd-dev \
binutils-mingw-w64 \
g++-mingw-w64-x86-64 \
net-tools \
pigz \
python-backports.ssl-match-hostname \
python-dev \
# python-cffi appears to be required for compiling paramiko on s390x/ppc64le
python-cffi \
python-mock \
python-pip \
python-requests \
python-setuptools \
python-websocket \
python-wheel \
thin-provisioning-tools \
vim \
vim-common \
xfsprogs \
zip \
bzip2 \
xz-utils \
libprotobuf-c1 \
libnet1 \
libnl-3-200 \
--no-install-recommends
COPY --from=swagger /build/swagger* /usr/local/bin/
COPY --from=frozen-images /build/ /docker-frozen-images
COPY --from=gometalinter /build/ /usr/local/bin/
COPY --from=tomlv /build/ /usr/local/bin/
COPY --from=vndr /build/ /usr/local/bin/
COPY --from=tini /build/ /usr/local/bin/
COPY --from=runc /build/ /usr/local/bin/
COPY --from=containerd /build/ /usr/local/bin/
COPY --from=proxy /build/ /usr/local/bin/
COPY --from=dockercli /build/ /usr/local/cli
COPY --from=registry /build/registry* /usr/local/bin/
COPY --from=criu /build/ /usr/local/
COPY --from=docker-py /build/ /docker-py
COPY --from=rootlesskit /build/ /usr/local/bin/
COPY --from=djs55/vpnkit@sha256:e508a17cfacc8fd39261d5b4e397df2b953690da577e2c987a47630cd0c42f8e /vpnkit /usr/local/bin/vpnkit.x86_64
ENV PATH=/usr/local/cli:$PATH
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
# Options for hack/validate/gometalinter
ENV GOMETALINTER_OPTS="--deadline=2m"
WORKDIR /go/src/github.com/docker/docker
VOLUME /var/lib/docker
# Wrap all commands in the "docker-in-docker" script to allow nested containers
ENTRYPOINT ["hack/dind"]
FROM dev AS final
# Upload docker source
COPY . /go/src/github.com/docker/docker
ARG DOCKER_GITCOMMIT=HEAD
RUN --mount=type=cache,id=gocache,target=/root/.cache/go-build \
hack/make.sh binary
RUN hack/make.sh install-binary

14
Dockerfile.buildx Normal file
View File

@ -0,0 +1,14 @@
ARG GO_VERSION=1.12.10
FROM golang:${GO_VERSION}-stretch
ARG BUILDX_REPO=https://github.com/docker/buildx.git
RUN git clone "${BUILDX_REPO}" /buildx
WORKDIR /buildx
ARG BUILDX_COMMIT=master
RUN git fetch origin "${BUILDX_COMMIT}":build && git checkout build
RUN go mod download
ARG GOOS
ARG GOARCH
# Keep these essentially no-op var settings for debug purposes.
# It allows us to see what the GOOS/GOARCH that's being built for is.
RUN GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X github.com/docker/buildx/version.Version=${BUILDX_COMMIT} -X github.com/docker/buildx/version.Revision=${BUILDX_COMMIT} -X github.com/docker/buildx/version.Package=github.com/docker/buildx' -o /usr/bin/buildx ./cmd/buildx
ENTRYPOINT ["/usr/bin/buildx"]

View File

@ -1,5 +1,15 @@
.PHONY: all binary dynbinary build cross help install manpages run shell test test-docker-py test-integration test-unit validate win
ifdef USE_BUILDX
BUILDX ?= $(shell command -v buildx)
BUILDX ?= $(shell command -v docker-buildx)
DOCKER_BUILDX_CLI_PLUGIN_PATH ?= ~/.docker/cli-plugins/docker-buildx
BUILDX ?= $(shell if [ -x "$(DOCKER_BUILDX_CLI_PLUGIN_PATH)" ]; then echo $(DOCKER_BUILDX_CLI_PLUGIN_PATH); fi)
endif
BUILDX ?= bundles/buildx
DOCKER ?= docker
# set the graph driver as the current graphdriver if not set
DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //'))
export DOCKER_GRAPHDRIVER
@ -107,7 +117,7 @@ GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
DOCKER_IMAGE := docker-dev$(if $(GIT_BRANCH_CLEAN),:$(GIT_BRANCH_CLEAN))
DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)
DOCKER_FLAGS := docker run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)
DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)
BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))
export BUILD_APT_MIRROR
@ -133,29 +143,30 @@ default: binary
all: build ## validate all checks, build linux binaries, run all tests\ncross build non-linux binaries and generate archives
$(DOCKER_RUN_DOCKER) bash -c 'hack/validate/default && hack/make.sh'
binary: DOCKER_BUILD_ARGS += --output=bundles/ --target=binary
binary: build ## build the linux binaries
$(DOCKER_RUN_DOCKER) hack/make.sh binary
dynbinary: DOCKER_BUILD_ARGS += --output=bundles/ --target=dynbinary
dynbinary: build ## build the linux dynbinaries
$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary
cross: DOCKER_BUILD_ARGS += --output=bundles/ --target=cross --build-arg DOCKER_CROSSPLATFORMS=$(DOCKER_CROSSPLATFORMS)
cross: DOCKER_CROSS := true
cross: build ## cross build the binaries for darwin, freebsd and\nwindows
$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary binary cross
ifdef DOCKER_CROSSPLATFORMS
build: DOCKER_CROSS := true
endif
ifeq ($(BIND_DIR), .)
build: DOCKER_BUILD_OPTS += --target=dev
endif
build: DOCKER_BUILD_ARGS += --build-arg=CROSS=$(DOCKER_CROSS)
build: DOCKER_BUILDKIT ?= 1
ifdef GO_VERSION
build: DOCKER_BUILD_ARGS += --build-arg=GO_VERSION=$(GO_VERSION)
endif
ifdef USE_BUILDX
build: bundles buildx
$(BUILDX) build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" $(BUILDX_BUILD_EXTRA_OPTS) .
else
build: bundles
$(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n})
DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build --build-arg=GO_VERSION ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
$(DOCKER) build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
endif
bundles:
mkdir bundles
@ -176,7 +187,9 @@ install: ## install the linux binaries
run: build ## run the docker daemon in a container
$(DOCKER_RUN_DOCKER) sh -c "KEEPBUNDLE=1 hack/make.sh install-binary run"
shell: build ## start a shell inside the build env
shell: DOCKER_BUILD_ARGS += --target=dev
shell: BUILDX_BUILD_EXTRA_OPTS += --load
shell: build ## start a shell inside the build env
$(DOCKER_RUN_DOCKER) bash
test: build test-unit ## run the unit, integration and docker-py tests
@ -222,3 +235,24 @@ swagger-docs: ## preview the API documentation
-e 'REDOC_OPTIONS=hide-hostname="true" lazy-rendering' \
-p $(SWAGGER_DOCS_PORT):80 \
bfirsh/redoc:1.6.2
.PHONY: buildx
ifeq ($(BUILDX), bundles/buildx)
buildx: bundles/buildx # build buildx cli tool
else
buildx:
endif
bundles/buildx: BUILDX_DOCKERFILE ?= Dockerfile.buildx
bundles/buildx: BUILDX_COMMIT ?= v0.3.0
bundles/buildx: bundles ## build buildx CLI tool
# This intetionally is not using the `--output` flag from the docker CLI which is a buildkit option
# The idea here being that if buildx is being used, it's because buildkit is not supported natively
docker build -f $(BUILDX_DOCKERFILE) -t "moby-buildx:$(BUILDX_COMMIT)" \
--build-arg BUILDX_COMMIT \
--build-arg BUILDX_REPO \
--build-arg GOOS=$$(if [ -n "$(GOOS)" ]; then echo $(GOOS); else go env GOHOSTOS || uname | awk '{print tolower($$0)}' || true; fi) \
--build-arg GOARCH=$$(if [ -n "$(GOARCH)" ]; then echo $(GOARCH); else go env GOHOSTARCH || true; fi) \
. && \
id=$$(docker create moby-buildx:$(BUILDX_COMMIT)); \
if [ -n "$${id}" ]; then docker cp $${id}:/usr/bin/buildx $@ && touch $@; docker rm -f $${id}; fi