Merge pull request #12111 from tianon/builder-deb

Add "builder-deb" base images for building ".deb" packages properly

contrib/builder/deb/README.md

@@ -0,0 +1,5 @@
+# `dockercore/builder-deb`
+
+This image's tags contain the dependencies for building Docker `.deb`s for each of the Debian-based platforms Docker targets.
+
+To add new tags, see [`contrib/builder/deb` in https://github.com/docker/docker](https://github.com/docker/docker/tree/master/contrib/builder/deb), specifically the `generate.sh` script, whose usage is described in a comment at the top of the file.

contrib/builder/deb/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+set -x
+./generate.sh
+for d in */; do
+	docker build -t "dockercore/builder-deb:$(basename "$d")" "$d"
+done

contrib/builder/deb/debian-jessie/Dockerfile

@@ -0,0 +1,14 @@
+#
+# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+#
+
+FROM debian:jessie
+
+RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+ENV GO_VERSION 1.4.2
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local
+ENV PATH $PATH:/usr/local/go/bin
+
+ENV AUTO_GOPATH 1
+ENV DOCKER_BUILDTAGS apparmor selinux

contrib/builder/deb/debian-wheezy/Dockerfile

@@ -0,0 +1,15 @@
+#
+# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+#
+
+FROM debian:wheezy
+RUN echo deb http://http.debian.net/debian wheezy-backports main > /etc/apt/sources.list.d/wheezy-backports.list
+
+RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+ENV GO_VERSION 1.4.2
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local
+ENV PATH $PATH:/usr/local/go/bin
+
+ENV AUTO_GOPATH 1
+ENV DOCKER_BUILDTAGS apparmor selinux

contrib/builder/deb/generate.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -e
+
+# usage: ./generate.sh [versions]
+#    ie: ./generate.sh
+#        to update all Dockerfiles in this directory
+#    or: ./generate.sh debian-jessie
+#        to only update debian-jessie/Dockerfile
+#    or: ./generate.sh debian-newversion
+#        to create a new folder and a Dockerfile within it
+
+cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+versions=( "$@" )
+if [ ${#versions[@]} -eq 0 ]; then
+	versions=( */ )
+fi
+versions=( "${versions[@]%/}" )
+
+for version in "${versions[@]}"; do
+	distro="${version%-*}"
+	suite="${version##*-}"
+	from="${distro}:${suite}"
+
+	mkdir -p "$version"
+	echo "$version -> FROM $from"
+	cat > "$version/Dockerfile" <<-EOF
+		#
+		# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+		#
+
+		FROM $from
+	EOF
+
+	case "$from" in
+		debian:wheezy)
+			# add -backports, like our users have to
+			echo "RUN echo deb http://http.debian.net/debian $suite-backports main > /etc/apt/sources.list.d/$suite-backports.list" >> "$version/Dockerfile"
+			;;
+	esac
+
+	echo >> "$version/Dockerfile"
+
+	# this list is sorted alphabetically; please keep it that way
+	packages=(
+		bash-completion # for bash-completion debhelper integration
+		btrfs-tools # for "btrfs/ioctl.h" (and "version.h" if possible)
+		build-essential # "essential for building Debian packages"
+		curl ca-certificates # for downloading Go
+		debhelper # for easy ".deb" building
+		dh-systemd # for systemd debhelper integration
+		git # for "git commit" info in "docker -v"
+		libapparmor-dev # for "sys/apparmor.h"
+		libdevmapper-dev # for "libdevmapper.h"
+		libsqlite3-dev # for "sqlite3.h"
+	)
+	echo "RUN apt-get update && apt-get install -y ${packages[*]} --no-install-recommends && rm -rf /var/lib/apt/lists/*" >> "$version/Dockerfile"
+
+	echo >> "$version/Dockerfile"
+
+	awk '$1 == "ENV" && $2 == "GO_VERSION" { print; exit }' ../../../Dockerfile >> "$version/Dockerfile"
+	echo 'RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local' >> "$version/Dockerfile"
+	echo 'ENV PATH $PATH:/usr/local/go/bin' >> "$version/Dockerfile"
+
+	echo >> "$version/Dockerfile"
+
+	echo 'ENV AUTO_GOPATH 1' >> "$version/Dockerfile"
+	awk '$1 == "ENV" && $2 == "DOCKER_BUILDTAGS" { print; exit }' ../../../Dockerfile >> "$version/Dockerfile"
+done

contrib/builder/deb/ubuntu-debootstrap-trusty/Dockerfile

@@ -0,0 +1,14 @@
+#
+# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+#
+
+FROM ubuntu-debootstrap:trusty
+
+RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+ENV GO_VERSION 1.4.2
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local
+ENV PATH $PATH:/usr/local/go/bin
+
+ENV AUTO_GOPATH 1
+ENV DOCKER_BUILDTAGS apparmor selinux

contrib/builder/deb/ubuntu-debootstrap-utopic/Dockerfile

@@ -0,0 +1,14 @@
+#
+# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+#
+
+FROM ubuntu-debootstrap:utopic
+
+RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+ENV GO_VERSION 1.4.2
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local
+ENV PATH $PATH:/usr/local/go/bin
+
+ENV AUTO_GOPATH 1
+ENV DOCKER_BUILDTAGS apparmor selinux

contrib/builder/deb/ubuntu-debootstrap-vivid/Dockerfile

@@ -0,0 +1,14 @@
+#
+# THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
+#
+
+FROM ubuntu-debootstrap:vivid
+
+RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+ENV GO_VERSION 1.4.2
+RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar xvzC /usr/local
+ENV PATH $PATH:/usr/local/go/bin
+
+ENV AUTO_GOPATH 1
+ENV DOCKER_BUILDTAGS apparmor selinux

hack/make/.build-deb/compat

@@ -0,0 +1 @@
+9

hack/make/.build-deb/control

@@ -0,0 +1,27 @@
+Source: docker-core
+Maintainer: Docker <support@docker.com>
+Homepage: https://dockerproject.com
+Vcs-Browser: https://github.com/docker/docker
+Vcs-Git: git://github.com/docker/docker.git
+
+Package: docker-core
+Architecture: linux-any
+Depends: iptables, ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}
+Recommends: aufs-tools,
+            ca-certificates,
+            cgroupfs-mount | cgroup-lite,
+            git,
+            xz-utils,
+            ${apparmor:Recommends}
+Conflicts: docker (<< 1.5~), docker.io, lxc-docker, lxc-docker-virtual-package
+Description: Docker: the open-source application container engine
+ Docker is an open source project to pack, ship and run any application as a
+ lightweight container
+ .
+ Docker containers are both hardware-agnostic and platform-agnostic. This means
+ they can run anywhere, from your laptop to the largest EC2 compute instance and
+ they can run anywhere, from your laptop to the largest EC2 compute instance and
+ everything in between - and they don't require you to use a particular
+ language, framework or packaging system. That makes them great building blocks
+ for deploying and scaling web apps, databases, and backend services without
+ depending on a particular stack or provider.

hack/make/.build-deb/docker-core.bash-completion

@@ -0,0 +1 @@
+contrib/completion/bash/docker

hack/make/.build-deb/docker-core.docker.default

@@ -0,0 +1 @@
+../../../contrib/init/sysvinit-debian/docker.default

hack/make/.build-deb/docker-core.docker.init

@@ -0,0 +1 @@
+../../../contrib/init/sysvinit-debian/docker

hack/make/.build-deb/docker-core.docker.upstart

@@ -0,0 +1 @@
+../../../contrib/init/upstart/docker.conf

hack/make/.build-deb/docker-core.install

@@ -0,0 +1,10 @@
+#contrib/syntax/vim/doc/* /usr/share/vim/vimfiles/doc/
+#contrib/syntax/vim/ftdetect/* /usr/share/vim/vimfiles/ftdetect/
+#contrib/syntax/vim/syntax/* /usr/share/vim/vimfiles/syntax/
+contrib/*-integration usr/share/docker-core/contrib/
+contrib/check-config.sh usr/share/docker-core/contrib/
+contrib/completion/zsh/_docker usr/share/zsh/vendor-completions/
+contrib/init/systemd/docker.service lib/systemd/system/
+contrib/init/systemd/docker.socket lib/systemd/system/
+contrib/mk* usr/share/docker-core/contrib/
+contrib/nuke-graph-directory.sh usr/share/docker-core/contrib/

hack/make/.build-deb/docker-core.manpages

@@ -0,0 +1 @@
+docs/man/man*/*

hack/make/.build-deb/docker-core.postinst

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+case "$1" in
+	configure)
+		if [ -z "$2" ]; then
+			if ! getent group docker > /dev/null; then
+				groupadd --system docker
+			fi
+		fi
+		;;
+	abort-*)
+		# How'd we get here??
+		exit 1
+		;;
+	*)
+		;;
+esac
+
+#DEBHELPER#

hack/make/.build-deb/docker-core.udev

@@ -0,0 +1 @@
+../../../contrib/udev/80-docker.rules

hack/make/.build-deb/docs

@@ -0,0 +1 @@
+README.md

hack/make/.build-deb/rules

@@ -0,0 +1,36 @@
+#!/usr/bin/make -f
+
+VERSION = $(shell cat VERSION)
+
+override_dh_gencontrol:
+	# if we're on Ubuntu, we need to Recommends: apparmor
+	echo 'apparmor:Recommends=$(shell dpkg-vendor --is Ubuntu && echo apparmor)' >> debian/docker-core.substvars
+	dh_gencontrol
+
+override_dh_auto_build:
+	./hack/make.sh dynbinary
+	# ./docs/man/md2man-all.sh runs outside the build container (if at all), since we don't have go-md2man here
+
+override_dh_auto_test:
+	./bundles/$(VERSION)/dynbinary/docker -v
+
+override_dh_strip:
+	# the SHA1 of dockerinit is important: don't strip it
+	# also, Go has lots of problems with stripping, so just don't
+
+override_dh_auto_install:
+	mkdir -p debian/docker-core/usr/bin
+	cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary/docker)" debian/docker-core/usr/bin/docker
+	mkdir -p debian/docker-core/usr/libexec/docker
+	cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary/dockerinit)" debian/docker-core/usr/libexec/docker/dockerinit
+
+override_dh_installinit:
+	# use "docker" as our service name, not "docker-core"
+	dh_installinit --name=docker
+
+override_dh_installudev:
+	# match our existing priority
+	dh_installudev --priority=z80
+
+%:
+	dh $@ --with=systemd,bash-completion

hack/make/build-deb

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -e
+
+DEST=$1
+
+# subshell so that we can export PATH without breaking other things
+(
+	source "$(dirname "$BASH_SOURCE")/.integration-daemon-start"
+
+	# we need to wrap up everything in between integration-daemon-start and
+	# integration-daemon-stop to make sure we kill the daemon and don't hang,
+	# even and especially on test failures
+	didFail=
+	if ! {
+		set -e
+
+		# TODO consider using frozen images for the dockercore/builder-deb tags
+
+		debVersion="${VERSION//-/'~'}"
+		# if we have a "-dev" suffix or have change in Git, let's make this package version more complex so it works better
+		if [[ "$VERSION" == *-dev ]] || [ -n "$(git status --porcelain)" ]; then
+			gitUnix="$(git log -1 --pretty='%at')"
+			gitDate="$(date --date "@$gitUnix" +'%Y%m%d.%H%M%S')"
+			gitCommit="$(git log -1 --pretty='%h')"
+			gitVersion="git${gitDate}.0.${gitCommit}"
+			# gitVersion is now something like 'git20150128.112847.0.17e840a'
+			debVersion="$debVersion~$gitVersion"
+
+			# $ dpkg --compare-versions 1.5.0 gt 1.5.0~rc1 && echo true || echo false
+			# true
+			# $ dpkg --compare-versions 1.5.0~rc1 gt 1.5.0~git20150128.112847.17e840a && echo true || echo false
+			# true
+			# $ dpkg --compare-versions 1.5.0~git20150128.112847.17e840a gt 1.5.0~dev~git20150128.112847.17e840a && echo true || echo false
+			# true
+
+			# ie, 1.5.0 > 1.5.0~rc1 > 1.5.0~git20150128.112847.17e840a > 1.5.0~dev~git20150128.112847.17e840a
+		fi
+
+		debSource="$(awk -F ': ' '$1 == "Source" { print $2; exit }' hack/make/.build-deb/control)"
+		debMaintainer="$(awk -F ': ' '$1 == "Maintainer" { print $2; exit }' hack/make/.build-deb/control)"
+		debDate="$(date --rfc-2822)"
+
+		# if go-md2man is available, pre-generate the man pages
+		./docs/man/md2man-all.sh -q || true
+		# TODO decide if it's worth getting go-md2man in _each_ builder environment to avoid this
+
+		# TODO add a configurable knob for _which_ debs to build so we don't have to modify the file or build all of them every time we need to test
+		for dir in contrib/builder/deb/*/; do
+			version="$(basename "$dir")"
+			suite="${version##*-}"
+
+			image="dockercore/builder-deb:$version"
+			if ! docker inspect "$image" &> /dev/null; then
+				( set -x && docker build -t "$image" "$dir" )
+			fi
+
+			mkdir -p "$DEST/$version"
+			cat > "$DEST/$version/Dockerfile.build" <<-EOF
+				FROM $image
+				WORKDIR /usr/src/docker
+				COPY . /usr/src/docker
+				RUN ln -sfv hack/make/.build-deb debian
+				RUN { echo '$debSource (${debVersion}-0~${suite}) $suite; urgency=low'; echo; echo '  * Version: $VERSION'; echo; echo " -- $debMaintainer  $debDate"; } > debian/changelog && cat >&2 debian/changelog
+				RUN dpkg-buildpackage -uc -us
+			EOF
+			cp -a "$DEST/$version/Dockerfile.build" . # can't use $DEST because it's in .dockerignore...
+			tempImage="docker-temp/build-deb:$version"
+			( set -x && docker build -t "$tempImage" -f Dockerfile.build . )
+			docker run --rm "$tempImage" bash -c 'cd .. && tar -c *_*' | tar -xvC "$DEST/$version"
+			docker rmi "$tempImage"
+		done
+	}; then
+		didFail=1
+	fi
+
+	# clean up after ourselves
+	rm -f Dockerfile.build
+
+	source "$(dirname "$BASH_SOURCE")/.integration-daemon-stop"
+
+	[ -z "$didFail" ] # "set -e" ftw
+) 2>&1 | tee -a $DEST/test.log