From 9ca8386c5720d887f6c22b657324e9fc6ff2487d Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 26 Jan 2016 10:59:01 -0800 Subject: [PATCH 1/2] add scripts to clean experimental rpms and debs Signed-off-by: Jessica Frazelle --- hack/make/clean-apt-repo | 43 ++++++++++++++++++++++++ hack/make/clean-yum-repo | 20 +++++++++++ hack/make/update-apt-repo | 70 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100755 hack/make/clean-apt-repo create mode 100755 hack/make/clean-yum-repo create mode 100755 hack/make/update-apt-repo diff --git a/hack/make/clean-apt-repo b/hack/make/clean-apt-repo new file mode 100755 index 0000000000..1c37d98e40 --- /dev/null +++ b/hack/make/clean-apt-repo @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +# This script cleans the experimental pool for the apt repo. +# This is useful when there are a lot of old experimental debs and you only want to keep the most recent. +# + +: ${DOCKER_RELEASE_DIR:=$DEST} +APTDIR=$DOCKER_RELEASE_DIR/apt/repo/pool/experimental +: ${DOCKER_ARCHIVE_DIR:=$DEST/archive} +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +latest_versions=$(dpkg-scanpackages "$APTDIR" /dev/null 2>/dev/null | awk -F ': ' '$1 == "Filename" { print $2 }') + +# get the latest version +latest_docker_engine_file=$(echo "$latest_versions" | grep docker-engine) +latest_docker_engine_version=$(basename ${latest_docker_engine_file%~*}) + +echo "latest docker-engine version: $latest_docker_engine_version" + +# remove all the files that are not that version in experimental +pool_dir=$(dirname "$latest_docker_engine_file") +old_pkgs=( $(ls "$pool_dir" | grep -v "^${latest_docker_engine_version}" | grep "${latest_docker_engine_version%%~git*}") ) + +echo "${old_pkgs[@]}" + +mkdir -p "$DOCKER_ARCHIVE_DIR" +for old_pkg in "${old_pkgs[@]}"; do + echo "moving ${pool_dir}/${old_pkg} to $DOCKER_ARCHIVE_DIR" + mv "${pool_dir}/${old_pkg}" "$DOCKER_ARCHIVE_DIR" +done + +echo +echo "$pool_dir now has contents:" +ls "$pool_dir" + +# now regenerate release files for experimental +export COMPONENT=experimental +source "${DIR}/update-apt-repo" + +echo "You will now want to: " +echo " - re-sign the repo with hack/make/sign-repo" +echo " - re-generate index files with hack/make/generate-index-listing" diff --git a/hack/make/clean-yum-repo b/hack/make/clean-yum-repo new file mode 100755 index 0000000000..1cafbbd97f --- /dev/null +++ b/hack/make/clean-yum-repo @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# This script cleans the experimental pool for the yum repo. +# This is useful when there are a lot of old experimental rpms and you only want to keep the most recent. +# + +: ${DOCKER_RELEASE_DIR:=$DEST} +YUMDIR=$DOCKER_RELEASE_DIR/yum/repo/experimental + +suites=( $(find "$YUMDIR" -mindepth 1 -maxdepth 1 -type d) ) + +for suite in "${suites[@]}"; do + echo "cleanup in: $suite" + ( set -x; repomanage -k2 --old "$suite" | xargs rm -f ) +done + +echo "You will now want to: " +echo " - re-sign the repo with hack/make/sign-repo" +echo " - re-generate index files with hack/make/generate-index-listing" diff --git a/hack/make/update-apt-repo b/hack/make/update-apt-repo new file mode 100755 index 0000000000..ae235c30b4 --- /dev/null +++ b/hack/make/update-apt-repo @@ -0,0 +1,70 @@ +#!/bin/bash +set -e + +# This script updates the apt repo in $DOCKER_RELEASE_DIR/apt/repo. +# This script is a "fix all" for any sort of problems that might have occured with +# the Release or Package files in the repo. +# It should only be used in the rare case of extreme emergencies to regenerate +# Release and Package files for the apt repo. +# +# NOTE: Always be sure to re-sign the repo with hack/make/sign-repos after running +# this script. + +: ${DOCKER_RELEASE_DIR:=$DEST} +APTDIR=$DOCKER_RELEASE_DIR/apt/repo + +# supported arches/sections +arches=( amd64 i386 ) + +# Preserve existing components but don't add any non-existing ones +for component in main testing experimental ; do + if ls "$APTDIR/dists/*/$component" >/dev/null 2>&1 ; then + components+=( $component ) + fi +done + +dists=( $(find "${APTDIR}/dists" -maxdepth 1 -mindepth 1 -type d) ) + +# override component if it is set +if [ "$COMPONENT" ]; then + components=( $COMPONENT ) +fi + +# release the debs +for version in "${dists[@]}"; do + for component in "${components[@]}"; do + codename="${version//debootstrap-}" + + # update the filelist for this codename/component + find "$APTDIR/pool/$component" \ + -name *~${codename#*-}*.deb > "$APTDIR/dists/$codename/$component/filelist" + done +done + +# run the apt-ftparchive commands so we can have pinning +apt-ftparchive generate "$APTDIR/conf/apt-ftparchive.conf" + +for dist in "${dists[@]}"; do + version=$(basename "$dist") + for component in "${components[@]}"; do + codename="${version//debootstrap-}" + + apt-ftparchive \ + -o "APT::FTPArchive::Release::Codename=$codename" \ + -o "APT::FTPArchive::Release::Suite=$codename" \ + -c "$APTDIR/conf/docker-engine-release.conf" \ + release \ + "$APTDIR/dists/$codename" > "$APTDIR/dists/$codename/Release" + + for arch in "${arches[@]}"; do + apt-ftparchive \ + -o "APT::FTPArchive::Release::Codename=$codename" \ + -o "APT::FTPArchive::Release::Suite=$codename" \ + -o "APT::FTPArchive::Release::Component=$component" \ + -o "APT::FTPArchive::Release::Architecture=$arch" \ + -c "$APTDIR/conf/docker-engine-release.conf" \ + release \ + "$APTDIR/dists/$codename/$component/binary-$arch" > "$APTDIR/dists/$codename/$component/binary-$arch/Release" + done + done +done From ca8a4935aa56c0e36b45891d457d78eca194cd14 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Wed, 27 Jan 2016 15:57:24 -0800 Subject: [PATCH 2/2] make docs better for scripts as well Signed-off-by: Jessica Frazelle --- project/PACKAGE-REPO-MAINTENANCE.md | 32 +++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/project/PACKAGE-REPO-MAINTENANCE.md b/project/PACKAGE-REPO-MAINTENANCE.md index 4419721dcf..3763f8798b 100644 --- a/project/PACKAGE-REPO-MAINTENANCE.md +++ b/project/PACKAGE-REPO-MAINTENANCE.md @@ -16,19 +16,14 @@ docker build --rm --force-rm -t docker-dev:master . docker run --rm -it --privileged \ -v /path/to/your/repos/dir:/volumes/repos \ -v $HOME/.gnupg:/root/.gnupg \ + -e GPG_PASSPHRASE \ -e DOCKER_RELEASE_DIR=/volumes/repos \ - docker-dev:master hack/make.sh clean-apt-repo clean-yum-repo + docker-dev:master hack/make.sh clean-apt-repo clean-yum-repo generate-index-listing sign-repos ``` -3. Re-sign the repo with your gpg key +3. Upload the changed repos to `s3` (if you host on s3) -```bash -./hack/make/sign-repos -``` - -4. Upload the changed repos to `s3` (if you host on s3) - -5. Purge the cache, PURGE the cache, PURGE THE CACHE! +4. Purge the cache, PURGE the cache, PURGE THE CACHE! ### How to get out of a sticky situation @@ -48,7 +43,13 @@ Otherwise CELEBRATE! Re-sign the repo with your gpg key: ```bash -./hack/make/sign-repos +docker build --rm --force-rm -t docker-dev:master . +docker run --rm -it --privileged \ + -v /path/to/your/repos/dir:/volumes/repos \ + -v $HOME/.gnupg:/root/.gnupg \ + -e GPG_PASSPHRASE \ + -e DOCKER_RELEASE_DIR=/volumes/repos \ + docker-dev:master hack/make.sh sign-repos ``` Upload the changed repo to `s3` (if that is where you host) @@ -63,16 +64,11 @@ docker build --rm --force-rm -t docker-dev:master . docker run --rm -it --privileged \ -v /path/to/your/repos/dir:/volumes/repos \ -v $HOME/.gnupg:/root/.gnupg \ + -e GPG_PASSPHRASE \ -e DOCKER_RELEASE_DIR=/volumes/repos \ - docker-dev:master hack/make.sh update-apt-repo + docker-dev:master hack/make.sh update-apt-repo generate-index-listing sign-repos ``` -Re-sign the repo with your gpg key: - -```bash -./hack/make/sign-repos -``` - -Upload the changed repo to `s3` (if that is where you host) +4. Upload the changed repo to `s3` (if that is where you host) PURGE THE CACHE.