Merge pull request #17699 from jfrazelle/update-release-script-to-new-process

update release script and release process

Dockerfile

@@ -53,9 +53,6 @@ RUN apt-get update && apt-get install -y \
 	python-mock \
 	python-pip \
 	python-websocket \
-	reprepro \
-	ruby1.9.1 \
-	ruby1.9.1-dev \
 	s3cmd=1.1.0* \
 	ubuntu-zfs \
 	xfsprogs \
@@ -107,9 +104,6 @@ RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint
 	&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
 	&& go install -v github.com/golang/lint/golint
 
-# TODO replace FPM with some very minimal debhelper stuff
-RUN gem install --no-rdoc --no-ri fpm --version 1.3.2
-
 # Install registry
 ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd
 RUN set -x \

hack/make.sh

@@ -63,7 +63,6 @@ DEFAULT_BUNDLES=(
 	cover
 	cross
 	tgz
-	ubuntu
 )
 
 VERSION=$(< ./VERSION)

hack/release.sh

@@ -22,16 +22,12 @@ To run, I need:
   environment variables AWS_S3_BUCKET and AWS_S3_BUCKET_PATH (default: '');
 - to be provided with AWS credentials for this S3 bucket, in environment
   variables AWS_ACCESS_KEY and AWS_SECRET_KEY;
-- the passphrase to unlock the GPG key specified by the optional environment
-  variable GPG_KEYID (default: releasedocker) which will sign the deb
-  packages (passed as environment variable GPG_PASSPHRASE);
 - a generous amount of good will and nice manners.
 The canonical way to run me is to run the image produced by the Dockerfile: e.g.:"
 
 docker run -e AWS_S3_BUCKET=test.docker.com \
            -e AWS_ACCESS_KEY=... \
            -e AWS_SECRET_KEY=... \
-           -e GPG_PASSPHRASE=... \
            -i -t --privileged \
            docker ./hack/release.sh
 EOF
@@ -41,8 +37,6 @@ EOF
 [ "$AWS_S3_BUCKET" ] || usage
 [ "$AWS_ACCESS_KEY" ] || usage
 [ "$AWS_SECRET_KEY" ] || usage
-[ "$GPG_PASSPHRASE" ] || usage
-: ${GPG_KEYID:=releasedocker}
 [ -d /go/src/github.com/docker/docker ] || usage
 cd /go/src/github.com/docker/docker
 [ -x hack/make.sh ] || usage
@@ -51,7 +45,6 @@ RELEASE_BUNDLES=(
 	binary
 	cross
 	tgz
-	ubuntu
 )
 
 if [ "$1" != '--release-regardless-of-test-failure' ]; then
@@ -261,69 +254,6 @@ release_build() {
 	upload_release_build "$tgzDir/$tgz" "$s3Dir/$tgz" "$latestTgz"
 }
 
-# Upload the 'ubuntu' bundle to S3:
-# 1. A full APT repository is published at $BUCKET/ubuntu/
-# 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/index
-release_ubuntu() {
-	echo "Releasing ubuntu"
-	[ -e "bundles/$VERSION/ubuntu" ] || {
-		echo >&2 './hack/make.sh must be run before release_ubuntu'
-		exit 1
-	}
-
-	local debfiles=( "bundles/$VERSION/ubuntu/"*.deb )
-
-	# Sign our packages
-	dpkg-sig -g "--passphrase $GPG_PASSPHRASE" -k "$GPG_KEYID" --sign builder "${debfiles[@]}"
-
-	# Setup the APT repo
-	APTDIR=bundles/$VERSION/ubuntu/apt
-	mkdir -p "$APTDIR/conf" "$APTDIR/db"
-	s3cmd sync "s3://$BUCKET/ubuntu/db/" "$APTDIR/db/" || true
-	cat > "$APTDIR/conf/distributions" <<EOF
-Codename: docker
-Components: main
-Architectures: amd64 i386
-EOF
-
-	# Add the DEB package to the APT repo
-	reprepro -b "$APTDIR" includedeb docker "${debfiles[@]}"
-
-	# Sign
-	for F in $(find $APTDIR -name Release); do
-		gpg -u "$GPG_KEYID" --passphrase "$GPG_PASSPHRASE" \
-			--armor --sign --detach-sign \
-			--output "$F.gpg" "$F"
-	done
-
-	# Upload keys
-	s3cmd sync "$HOME/.gnupg/" "s3://$BUCKET/ubuntu/.gnupg/"
-	gpg --armor --export "$GPG_KEYID" > "bundles/$VERSION/ubuntu/gpg"
-	s3cmd --acl-public put "bundles/$VERSION/ubuntu/gpg" "s3://$BUCKET/gpg"
-
-	local gpgFingerprint=36A1D7869245C8950F966E92D8576A8BA88D21E9
-	local s3Headers=
-	if [[ $BUCKET == test* ]]; then
-		gpgFingerprint=740B314AE3941731B942C66ADF4FD13717AAD7D6
-	elif [[ $BUCKET == experimental* ]]; then
-		gpgFingerprint=E33FF7BF5C91D50A6F91FFFD4CC38D40F9A96B49
-		s3Headers='--add-header=Cache-Control:no-cache'
-	fi
-
-	# Upload repo
-	s3cmd --acl-public $s3Headers sync "$APTDIR/" "s3://$BUCKET/ubuntu/"
-	cat <<EOF | write_to_s3 s3://$BUCKET/ubuntu/index
-echo "# WARNING! This script is deprecated. Please use the script"
-echo "# at https://get.docker.com/"
-EOF
-
-	# Add redirect at /ubuntu/info for URL-backwards-compatibility
-	rm -rf /tmp/emptyfile && touch /tmp/emptyfile
-	s3cmd --acl-public --add-header='x-amz-website-redirect-location:/ubuntu/' --mime-type='text/plain' put /tmp/emptyfile "s3://$BUCKET/ubuntu/info"
-
-	echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu"
-}
-
 # Upload binaries and tgz files to S3
 release_binaries() {
 	[ -e "bundles/$VERSION/cross/linux/amd64/docker-$VERSION" ] || {
@@ -369,31 +299,10 @@ release_test() {
 	fi
 }
 
-setup_gpg() {
-	echo "Setting up GPG"
-	# Make sure that we have our keys
-	mkdir -p "$HOME/.gnupg/"
-	s3cmd sync "s3://$BUCKET/ubuntu/.gnupg/" "$HOME/.gnupg/" || true
-	gpg --list-keys "$GPG_KEYID" >/dev/null || {
-		gpg --gen-key --batch <<EOF
-Key-Type: RSA
-Key-Length: 4096
-Passphrase: $GPG_PASSPHRASE
-Name-Real: Docker Release Tool
-Name-Email: docker@docker.com
-Name-Comment: $GPG_KEYID
-Expire-Date: 0
-%commit
-EOF
-	}
-}
-
 main() {
 	build_all
 	setup_s3
-	setup_gpg
 	release_binaries
-	release_ubuntu
 	release_index
 	release_test
 }
@@ -407,7 +316,6 @@ echo "Use the following text to announce the release:"
 echo
 echo "We have just pushed $VERSION to $(s3_url). You can download it with the following:"
 echo
-echo "Ubuntu/Debian: curl -sSL $(s3_url) | sh"
 echo "Linux 64bit binary: $(s3_url)/builds/Linux/x86_64/docker-$VERSION"
 echo "Darwin/OSX 64bit client binary: $(s3_url)/builds/Darwin/x86_64/docker-$VERSION"
 echo "Darwin/OSX 32bit client binary: $(s3_url)/builds/Darwin/i386/docker-$VERSION"

project/RELEASE-CHECKLIST.md

@@ -203,7 +203,18 @@ That last command will give you the proper link to visit to ensure that you
 open the PR against the "release" branch instead of accidentally against
 "master" (like so many brave souls before you already have).
 
-### 7. Publish release candidate binaries
+### 7. Build release candidate rpms and debs
+
+```bash
+docker build -t docker .
+docker run \
+    --rm -t --privileged \
+    -v $(pwd)/bundles:/go/src/github.com/docker/docker/bundles \
+    docker \
+    hack/make.sh binary build-deb build-rpm
+```
+
+### 8. Publish release candidate binaries
 
 To run this you will need access to the release credentials. Get them from the
 Core maintainers.
@@ -214,20 +225,19 @@ Replace "..." with the respective credentials:
 docker build -t docker .
 
 docker run \
-       -e AWS_S3_BUCKET=test.docker.com \
+    -e AWS_S3_BUCKET=test.docker.com \ # static binaries are still pushed to s3
-       -e AWS_ACCESS_KEY="..." \
+    -e AWS_ACCESS_KEY="..." \
-       -e AWS_SECRET_KEY="..." \
+    -e AWS_SECRET_KEY="..." \
-       -e GPG_PASSPHRASE="..." \
+    -i -t --privileged \
-       -i -t --privileged \
+    docker \
-       docker \
+    hack/release.sh
-       hack/release.sh
 ```
 
-It will run the test suite, build the binaries and packages, and upload to the
+It will run the test suite, build the binaries and upload to the specified bucket,
-specified bucket, so this is a good time to verify that you're running against
+so this is a good time to verify that you're running against **test**.docker.com.
-**test**.docker.com.
 
-After the binaries and packages are uploaded to test.docker.com, make sure
+After the binaries are uploaded to test.docker.com and the packages are on
+apt.dockerproject.org and yum.dockerproject.org, make sure
 they get tested in both Ubuntu and Debian for any obvious installation
 issues or runtime issues.
 
@@ -242,7 +252,7 @@ Announcing on multiple medias is the best way to get some help testing! An easy
 way to get some useful links for sharing:
 
 ```bash
-echo "Ubuntu/Debian: https://test.docker.com/ubuntu or curl -sSL https://test.docker.com/ | sh"
+echo "Ubuntu/Debian: curl -sSL https://test.docker.com/ | sh"
 echo "Linux 64bit binary: https://test.docker.com/builds/Linux/x86_64/docker-${VERSION#v}"
 echo "Darwin/OSX 64bit client binary: https://test.docker.com/builds/Darwin/x86_64/docker-${VERSION#v}"
 echo "Darwin/OSX 32bit client binary: https://test.docker.com/builds/Darwin/i386/docker-${VERSION#v}"
@@ -257,7 +267,7 @@ We recommend announcing the release candidate on:
 - The [docker-maintainers](https://groups.google.com/a/dockerproject.org/forum/#!forum/maintainers) group
 - Any social media that can bring some attention to the release candidate
 
-### 8. Iterate on successive release candidates
+### 9. Iterate on successive release candidates
 
 Spend several days along with the community explicitly investing time and
 resources to try and break Docker in every possible way, documenting any
@@ -307,7 +317,7 @@ git push -f $GITHUBUSER bump_$VERSION
 Repeat step 6 to tag the code, publish new binaries, announce availability, and
 get help testing.
 
-### 9. Finalize the bump branch
+### 10. Finalize the bump branch
 
 When you're happy with the quality of a release candidate, you can move on and
 create the real thing.
@@ -323,25 +333,36 @@ git commit --amend
 
 You will then repeat step 6 to publish the binaries to test
 
-### 10. Get 2 other maintainers to validate the pull request
+### 11. Get 2 other maintainers to validate the pull request
 
-### 11. Publish final binaries
+### 12. Build final rpms and debs
+
+```bash
+docker build -t docker .
+docker run \
+    --rm -t --privileged \
+    -v $(pwd)/bundles:/go/src/github.com/docker/docker/bundles \
+    docker \
+    hack/make.sh binary build-deb build-rpm
+```
+
+### 13. Publish final binaries
 
 Once they're tested and reasonably believed to be working, run against
 get.docker.com:
 
 ```bash
+docker build -t docker .
 docker run \
-       -e AWS_S3_BUCKET=get.docker.com \
+    -e AWS_S3_BUCKET=get.docker.com \ # static binaries are still pushed to s3
-       -e AWS_ACCESS_KEY="..." \
+    -e AWS_ACCESS_KEY="..." \
-       -e AWS_SECRET_KEY="..." \
+    -e AWS_SECRET_KEY="..." \
-       -e GPG_PASSPHRASE="..." \
+    -i -t --privileged \
-       -i -t --privileged \
+    docker \
-       docker \
+    hack/release.sh
-       hack/release.sh
 ```
 
-### 12. Apply tag and create release
+### 14. Apply tag and create release
 
 It's very important that we don't make the tag until after the official
 release is uploaded to get.docker.com!
@@ -360,12 +381,12 @@ You can see examples in this two links:
 https://github.com/docker/docker/releases/tag/v1.8.0
 https://github.com/docker/docker/releases/tag/v1.8.0-rc3
 
-### 13. Go to github to merge the `bump_$VERSION` branch into release
+### 15. Go to github to merge the `bump_$VERSION` branch into release
 
 Don't forget to push that pretty blue button to delete the leftover
 branch afterwards!
 
-### 14. Update the docs branch
+### 16. Update the docs branch
 
 You will need to point the docs branch to the newly created release tag:
 
@@ -384,7 +405,7 @@ distributed CDN system) is flushed. The `make docs-release` command will do this
 _if_ the `DISTRIBUTION_ID` is set correctly - this will take at least 15 minutes to run
 and you can check its progress with the CDN Cloudfront Chrome addon.
 
-### 15. Create a new pull request to merge your bump commit back into master
+### 17. Create a new pull request to merge your bump commit back into master
 
 ```bash
 git checkout master
@@ -398,14 +419,14 @@ echo "https://github.com/$GITHUBUSER/docker/compare/docker:master...$GITHUBUSER:
 Again, get two maintainers to validate, then merge, then push that pretty
 blue button to delete your branch.
 
-### 16. Update the VERSION files
+### 18. Update the VERSION files
 
 Now that version X.Y.Z is out, time to start working on the next! Update the
 content of the `VERSION` file to be the next minor (incrementing Y) and add the
 `-dev` suffix. For example, after 1.5.0 release, the `VERSION` file gets
 updated to `1.6.0-dev` (as in "1.6.0 in the making").
 
-### 17. Rejoice and Evangelize!
+### 19. Rejoice and Evangelize!
 
 Congratulations! You're done.