From ccb0b0a7cc4f00c0fe79af5e5f5d0a5fa71e7fcb Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 5 Nov 2019 12:11:49 -0800 Subject: [PATCH 1/5] binary targets do not need the dev environment This makes the binary build targets use a minimal build env instead of having to build all the stuff needed for the full dev enviornment. Signed-off-by: Brian Goff --- Dockerfile | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 031e82928a..6d13e77f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ ARG CROSS="false" ARG GO_VERSION=1.13.4 ARG DEBIAN_FRONTEND=noninteractive ARG VPNKIT_DIGEST=e508a17cfacc8fd39261d5b4e397df2b953690da577e2c987a47630cd0c42f8e +ARG DOCKER_BUILDTAGS="apparmor seccomp selinux" FROM golang:${GO_VERSION}-stretch AS base RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache @@ -111,10 +112,16 @@ ARG DEBIAN_FRONTEND RUN --mount=type=cache,sharing=locked,id=moby-cross-false-aptlib,target=/var/lib/apt \ --mount=type=cache,sharing=locked,id=moby-cross-false-aptcache,target=/var/cache/apt \ apt-get update && apt-get install -y --no-install-recommends \ + binutils-mingw-w64 \ + btrfs-tools \ + g++-mingw-w64-x86-64 \ libapparmor-dev \ - libseccomp-dev + libdevmapper-dev \ + libseccomp-dev \ + libsystemd-dev \ + libudev-dev -FROM --platform=linux/amd64 cross-true AS runtime-dev-cross-true +FROM --platform=linux/amd64 runtime-dev-cross-false AS runtime-dev-cross-true ARG DEBIAN_FRONTEND # These crossbuild packages rely on gcc-, but this doesn't want to install # on non-amd64 systems. @@ -128,11 +135,7 @@ RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/ libapparmor-dev:armhf \ libseccomp-dev:arm64 \ libseccomp-dev:armel \ - libseccomp-dev:armhf \ - # install this arches seccomp here due to compat issues with the v0 builder - # This is as opposed to inheriting from runtime-dev-cross-false - libapparmor-dev \ - libseccomp-dev + libseccomp-dev:armhf FROM runtime-dev-cross-${CROSS} AS runtime-dev @@ -261,19 +264,13 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \ apparmor \ aufs-tools \ bash-completion \ - binutils-mingw-w64 \ - btrfs-tools \ bzip2 \ - g++-mingw-w64-x86-64 \ iptables \ jq \ libcap2-bin \ - libdevmapper-dev \ libnet1 \ libnl-3-200 \ libprotobuf-c1 \ - libsystemd-dev \ - libudev-dev \ net-tools \ pigz \ python3-pip \ @@ -304,15 +301,17 @@ COPY --from=containerd /build/ /usr/local/bin/ COPY --from=rootlesskit /build/ /usr/local/bin/ COPY --from=vpnkit /vpnkit /usr/local/bin/vpnkit.x86_64 COPY --from=proxy /build/ /usr/local/bin/ - ENV PATH=/usr/local/cli:$PATH -ENV DOCKER_BUILDTAGS apparmor seccomp selinux +ARG DOCKER_BUILDTAGS +ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}" WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] -FROM dev AS src +FROM runtime-dev AS src +# Make arg inheritable +WORKDIR /go/src/github.com/docker/docker COPY . /go/src/github.com/docker/docker FROM src AS binary-base @@ -326,6 +325,16 @@ ARG PRODUCT ENV PRODUCT=${PRODUCT} ARG DEFAULT_PRODUCT_LICENSE ENV DEFAULT_PRODUCT_LICENSE=${DEFAULT_PRODUCT_LICENSE} +ARG DOCKER_BUILDTAGS +ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}" +# TODO: This is here because hack/make.sh binary copies these extras binaries +# from $PATH into the bundles dir. +# It would be nice to handle this in a different way. +COPY --from=tini /build/ /usr/local/bin/ +COPY --from=runc /build/ /usr/local/bin/ +COPY --from=containerd /build/ /usr/local/bin/ +COPY --from=rootlesskit /build/ /usr/local/bin/ +COPY --from=proxy /build/ /usr/local/bin/ FROM binary-base AS build-binary RUN --mount=type=cache,target=/root/.cache/go-build \ @@ -337,7 +346,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM binary-base AS build-cross ARG DOCKER_CROSSPLATFORMS -RUN --mount=type=cache,target=/root/.cache/go-build \ hack/make.sh cross FROM scratch AS binary From 675b414f56063826457cbc8f0c165af67198bfa8 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 16 Oct 2019 17:04:58 -0700 Subject: [PATCH 2/5] Use -X ldflags to set dockerversion package vars This eliminates the need to lay down an auto-generated file. IIRC this was originally hadded for gccgo which we no longer support. Signed-off-by: Brian Goff --- .gitignore | 2 -- dockerversion/version_lib.go | 4 ++-- hack/make.sh | 2 +- hack/make/.go-autogen | 45 ++++++++---------------------------- 4 files changed, 13 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index abad293ee0..1605fa8930 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,6 @@ autogen/ bundles/ cmd/dockerd/dockerd contrib/builder/rpm/*/changelog -dockerversion/version_autogen.go -dockerversion/version_autogen_unix.go vendor/pkg/ go-test-report.json profile.out diff --git a/dockerversion/version_lib.go b/dockerversion/version_lib.go index b7d4650444..e383aca843 100644 --- a/dockerversion/version_lib.go +++ b/dockerversion/version_lib.go @@ -4,8 +4,8 @@ package dockerversion // import "github.com/docker/docker/dockerversion" // Default build-time variable for library-import. -// This file is overridden on build with build-time information. -const ( +// These variables are overridden on build with build-time information. +var ( GitCommit = "library-import" Version = "library-import" BuildTime = "library-import" diff --git a/hack/make.sh b/hack/make.sh index 1bd37b02cd..4791211334 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -113,7 +113,7 @@ LDFLAGS_STATIC='' EXTLDFLAGS_STATIC='-static' # ORIG_BUILDFLAGS is necessary for the cross target which cannot always build # with options like -race. -ORIG_BUILDFLAGS=( -tags "autogen netgo osusergo static_build $DOCKER_BUILDTAGS" -installsuffix netgo ) +ORIG_BUILDFLAGS=( -tags "netgo osusergo static_build $DOCKER_BUILDTAGS" -installsuffix netgo ) # see https://github.com/golang/go/issues/9369#issuecomment-69864440 for why -installsuffix is necessary here BUILDFLAGS=( ${BUILDFLAGS} "${ORIG_BUILDFLAGS[@]}" ) diff --git a/hack/make/.go-autogen b/hack/make/.go-autogen index a56add8934..3340448541 100644 --- a/hack/make/.go-autogen +++ b/hack/make/.go-autogen @@ -6,41 +6,16 @@ source hack/dockerfile/install/runc.installer source hack/dockerfile/install/tini.installer source hack/dockerfile/install/containerd.installer -cat > dockerversion/version_autogen.go < dockerversion/version_autogen_unix.go < Date: Tue, 5 Nov 2019 13:07:47 -0800 Subject: [PATCH 3/5] Add support for outputing binaries to custom dir Signed-off-by: Brian Goff --- hack/make.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index 4791211334..357e6a17bb 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -140,12 +140,17 @@ bundle() { } main() { + bundle_dir="bundles" + if [ -n "${PREFIX}" ]; then + bundle_dir="${PREFIX}/${bundle_dir}" + fi + if [ -z "${KEEPBUNDLE-}" ]; then - echo "Removing bundles/" - rm -rf bundles/* + echo "Removing ${bundle_dir}/" + rm -rf "${bundle_dir}"/* echo fi - mkdir -p bundles + mkdir -p "${bundle_dir}" if [ $# -lt 1 ]; then bundles=(${DEFAULT_BUNDLES[@]}) @@ -153,7 +158,7 @@ main() { bundles=($@) fi for bundle in ${bundles[@]}; do - export DEST="bundles/$(basename "$bundle")" + export DEST="${bundle_dir}/$(basename "$bundle")" # Cygdrive paths don't play well with go build -o. if [[ "$(uname -s)" == CYGWIN* ]]; then export DEST="$(cygpath -mw "$DEST")" From 5f314c24fcbb3f6fc5b9244686e624ca8f13d024 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 5 Nov 2019 13:41:04 -0800 Subject: [PATCH 4/5] Bind-mount context for Dockerfile build targets Signed-off-by: Brian Goff --- Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6d13e77f7c..c163c95682 100644 --- a/Dockerfile +++ b/Dockerfile @@ -309,12 +309,7 @@ VOLUME /var/lib/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] -FROM runtime-dev AS src -# Make arg inheritable -WORKDIR /go/src/github.com/docker/docker -COPY . /go/src/github.com/docker/docker - -FROM src AS binary-base +FROM runtime-dev AS binary-base ARG DOCKER_GITCOMMIT=HEAD ENV DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT} ARG VERSION @@ -327,6 +322,7 @@ ARG DEFAULT_PRODUCT_LICENSE ENV DEFAULT_PRODUCT_LICENSE=${DEFAULT_PRODUCT_LICENSE} ARG DOCKER_BUILDTAGS ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}" +ENV PREFIX=/build # TODO: This is here because hack/make.sh binary copies these extras binaries # from $PATH into the bundles dir. # It would be nice to handle this in a different way. @@ -335,26 +331,32 @@ COPY --from=runc /build/ /usr/local/bin/ COPY --from=containerd /build/ /usr/local/bin/ COPY --from=rootlesskit /build/ /usr/local/bin/ COPY --from=proxy /build/ /usr/local/bin/ +WORKDIR /go/src/github.com/docker/docker FROM binary-base AS build-binary RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=bind,target=/go/src/github.com/docker/docker \ hack/make.sh binary FROM binary-base AS build-dynbinary RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=bind,target=/go/src/github.com/docker/docker \ hack/make.sh dynbinary FROM binary-base AS build-cross ARG DOCKER_CROSSPLATFORMS +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=bind,target=/go/src/github.com/docker/docker \ hack/make.sh cross FROM scratch AS binary -COPY --from=build-binary /go/src/github.com/docker/docker/bundles/ / +COPY --from=build-binary /build/bundles/ / FROM scratch AS dynbinary -COPY --from=build-dynbinary /go/src/github.com/docker/docker/bundles/ / +COPY --from=build-dynbinary /build/ / FROM scratch AS cross -COPY --from=build-cross /go/src/github.com/docker/docker/bundles/ / +COPY --from=build-cross /build/ / -FROM src AS final +FROM dev AS final +COPY . /go/src/github.com/docker/docker From e6d514dbab5c4b6192bd41275eecfe68cfd257f6 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 5 Nov 2019 13:42:46 -0800 Subject: [PATCH 5/5] Add Dockerfile to .dockerignore The Dockerfile is not needed in any of the build targets. The one exception may be the dev image, however in most cases the docker source will be bind mounted into the container anyway. Signed-off-by: Brian Goff --- .dockerignore | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8645f94037..b8f478b4c1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ -bundles -.gopath -vendor/pkg -.go-pkg-cache +.dockerignore .git +.go-pkg-cache +.gopath +bundles +Dockerfile +vendor/pkg