From b06f62713986681c7be5e9e0a59c3f717fe837cd Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 24 Jun 2013 16:57:22 -0700 Subject: [PATCH 01/13] Library: hipache and ubuntu base --- library/hipache | 1 + library/ubuntu | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 library/hipache create mode 100644 library/ubuntu diff --git a/library/hipache b/library/hipache new file mode 100644 index 0000000000..dcbd143dea --- /dev/null +++ b/library/hipache @@ -0,0 +1 @@ +git://github.com/dotcloud/hipache diff --git a/library/ubuntu b/library/ubuntu new file mode 100644 index 0000000000..4568b67784 --- /dev/null +++ b/library/ubuntu @@ -0,0 +1,2 @@ +git://github.com/dotcloud/ubuntu-quantal +git://github.com/dotcloud/ubuntu-precise From a45490243b29f67f09854fdfe355a144d8c3346e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 10 Jul 2013 13:40:23 -0700 Subject: [PATCH 02/13] Adding Joffrey as library maintainer --- library/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 library/MAINTAINERS diff --git a/library/MAINTAINERS b/library/MAINTAINERS new file mode 100644 index 0000000000..b1c52939ef --- /dev/null +++ b/library/MAINTAINERS @@ -0,0 +1 @@ +Joffrey Fuhrer From 844a9ab85e793517743beecdda651fa42d42a518 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 10 Jul 2013 13:44:07 -0700 Subject: [PATCH 03/13] Define tags in the library files instead of upstream --- library/hipache | 3 ++- library/ubuntu | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/library/hipache b/library/hipache index dcbd143dea..1a153c5e9e 100644 --- a/library/hipache +++ b/library/hipache @@ -1 +1,2 @@ -git://github.com/dotcloud/hipache +latest git://github.com/dotcloud/hipache@7362ff5b812f +0.2.4 git://github.com/dotcloud/hipache@7362ff5b812f diff --git a/library/ubuntu b/library/ubuntu index 4568b67784..046c8db319 100644 --- a/library/ubuntu +++ b/library/ubuntu @@ -1,2 +1,5 @@ -git://github.com/dotcloud/ubuntu-quantal -git://github.com/dotcloud/ubuntu-precise +latest git://github.com/dotcloud/ubuntu-quantal +quantal git://github.com/dotcloud/ubuntu-quantal +12.10 git://github.com/dotcloud/ubuntu-quantal +precise git://github.com/dotcloud/ubuntu-precise +12.04 git://github.com/dotcloud/ubuntu-precise From c19fa83a8aa592d9716a883ac28ce1a20ce2bffb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Jul 2013 18:28:06 +0200 Subject: [PATCH 04/13] Use extended syntax (indicate what type of object needs to be checked out) --- library/hipache | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/hipache b/library/hipache index 1a153c5e9e..1c72b3772f 100644 --- a/library/hipache +++ b/library/hipache @@ -1,2 +1,2 @@ -latest git://github.com/dotcloud/hipache@7362ff5b812f -0.2.4 git://github.com/dotcloud/hipache@7362ff5b812f +latest git://github.com/dotcloud/hipache C:7362ff5b812f +0.2.4 git://github.com/dotcloud/hipache C:7362ff5b812f From 0ac672fea6fc8e670be9dce18bd63d50689230ef Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Jul 2013 17:25:20 +0200 Subject: [PATCH 05/13] Use full hash references in library/ Brew doesn't currently support abridged hashes --- library/hipache | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/hipache b/library/hipache index 1c72b3772f..ded0582c63 100644 --- a/library/hipache +++ b/library/hipache @@ -1,2 +1,2 @@ -latest git://github.com/dotcloud/hipache C:7362ff5b812f -0.2.4 git://github.com/dotcloud/hipache C:7362ff5b812f +latest git://github.com/dotcloud/hipache C:7362ff5b812f93eceafbdbf5e5959f676f731f80 +0.2.4 git://github.com/dotcloud/hipache C:7362ff5b812f93eceafbdbf5e5959f676f731f80 From d47df21a332db1c1d8efedfeb0e36934b09e7a88 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Jul 2013 19:22:07 +0200 Subject: [PATCH 06/13] Add brew script to the contrib folder --- contrib/brew/.gitignore | 1 + contrib/brew/README.md | 78 +++++++++++++++++++++++++++++++ contrib/brew/brew/__init__.py | 1 + contrib/brew/brew/brew.py | 87 +++++++++++++++++++++++++++++++++++ contrib/brew/brew/git.py | 48 +++++++++++++++++++ contrib/brew/docker-brew | 25 ++++++++++ contrib/brew/requirements.txt | 2 + contrib/brew/setup.py | 22 +++++++++ 8 files changed, 264 insertions(+) create mode 100644 contrib/brew/.gitignore create mode 100644 contrib/brew/README.md create mode 100644 contrib/brew/brew/__init__.py create mode 100644 contrib/brew/brew/brew.py create mode 100644 contrib/brew/brew/git.py create mode 100755 contrib/brew/docker-brew create mode 100644 contrib/brew/requirements.txt create mode 100644 contrib/brew/setup.py diff --git a/contrib/brew/.gitignore b/contrib/brew/.gitignore new file mode 100644 index 0000000000..0d20b6487c --- /dev/null +++ b/contrib/brew/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/contrib/brew/README.md b/contrib/brew/README.md new file mode 100644 index 0000000000..d50e66f2a8 --- /dev/null +++ b/contrib/brew/README.md @@ -0,0 +1,78 @@ +# docker-brew + +docker-brew is a command-line tool used to build the docker standard library. + +## Install instructions + +1. Install python if it isn't already available on your OS of choice +1. Install the easy_install tool (`sudo apt-get install python-setuptools` +for Debian) +1. Install the python package manager, `pip` (`easy_install pip`) +1. Run the following command: `pip install -r requirements.txt` +1. You should now be able to use the `docker-brew` script as such. + +## Basics + + ./docker-brew -h + +Display usage and help. + + ./docker-brew + +Default build from the default repo/branch. Images will be created under the +`library/` namespace. Does not perform a remote push. + + ./docker-brew -n mycorp.com -b stable --push git://github.com/mycorp/docker + +Will fetch the library definition files in the `stable` branch of the +`git://github.com/mycorp/docker` repository and create images under the +`mycorp.com` namespace (e.g. `mycorp.com/ubuntu`). Created images will then +be pushed to the official docker repository (pending: support for private +repositories) + +## Library definition files + +The library definition files are plain text files found in the `library/` +subfolder of the docker repository. + +### File names + +The name of a definition file will determine the name of the image(s) it +creates. For example, the `library/ubuntu` file will create images in the +`/ubuntu` repository. If multiple instructions are present in +a single file, all images are expected to be created under a different tag. + +### Instruction format + +Each line represents a build instruction. +There are different formats that `docker-brew` is able to parse. + + + git://github.com/dotcloud/hipache + https://github.com/dotcloud/docker.git + +The simplest format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as `latest`. Use of this format is discouraged because there is no +way to ensure stability. + + + bleeding-edge git://github.com/dotcloud/docker + unstable https://github.com/dotcloud/docker-redis.git + +A more advanced format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as ``. Recommended if we always want to provide a snapshot +of the latest development. Again, no way to ensure stability. + + T: + 2.4.0 git://github.com/dotcloud/docker-redis T:2.4.0 + B: + zfs git://github.com/dotcloud/docker B:zfs-support + C: + 2.2.0 https://github.com/dotcloud/docker-redis.git C:a4bf8923ee4ec566d3ddc212 + +The most complete format. `docker-brew` will fetch data from the provided git +repository from the provided reference (if it's a branch, brew will fetch its +`HEAD`). Generated image will be tagged as ``. Recommended whenever +possible. \ No newline at end of file diff --git a/contrib/brew/brew/__init__.py b/contrib/brew/brew/__init__.py new file mode 100644 index 0000000000..f693ad67a4 --- /dev/null +++ b/contrib/brew/brew/__init__.py @@ -0,0 +1 @@ +from brew import build_library diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py new file mode 100644 index 0000000000..69529201d6 --- /dev/null +++ b/contrib/brew/brew/brew.py @@ -0,0 +1,87 @@ +import os +import logging + +import docker + +import git + +DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' +DEFAULT_BRANCH = 'library' + +logger = logging.getLogger(__name__) +logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', + level='DEBUG') +client = docker.Client() +processed = {} + + +def build_library(repository=None, branch=None, namespace=None, push=False): + if repository is None: + repository = DEFAULT_REPOSITORY + if branch is None: + branch = DEFAULT_BRANCH + + logger.info('Cloning docker repo from {0}, branch: {1}'.format( + repository, branch)) + #FIXME: set destination folder and only pull latest changes instead of + # cloning the whole repo everytime + dst_folder = git.clone_branch(repository, branch) + for buildfile in os.listdir(os.path.join(dst_folder, 'library')): + if buildfile == 'MAINTAINERS': + continue + f = open(os.path.join(dst_folder, 'library', buildfile)) + for line in f: + logger.debug('{0} ---> {1}'.format(buildfile, line)) + args = line.split() + try: + if len(args) > 3: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + + url = None + ref = 'refs/heads/master' + tag = None + if len(args) == 1: # Just a URL, simple mode + url = args[0] + elif len(args) == 2 or len(args) == 3: # docker-tag url + url = args[1] + tag = args[0] + + if len(args) == 3: # docker-tag url B:branch or T:tag + ref = None + if args[2].startswith('B:'): + ref = 'refs/heads/' + args[2][2:] + elif args[2].startswith('T:'): + ref = 'refs/tags/' + args[2][2:] + elif args[2].startswith('C:'): + ref = args[2][2:] + else: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + img = build_repo(url, ref, buildfile, tag, namespace, push) + processed['{0}@{1}'.format(url, ref)] = img + except Exception as e: + logger.exception(e) + f.close() + + +def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): + docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) + img_id = None + if '{0}@{1}'.format(repository, ref) not in processed.keys(): + logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) + dst_folder = git.clone(repository, ref) + if not 'Dockerfile' in os.listdir(dst_folder): + raise RuntimeError('Dockerfile not found in cloned repository') + logger.info('Building using dockerfile...') + img_id, logs = client.build(path=dst_folder) + + if not img_id: + img_id = processed['{0}@{1}'.format(repository, ref)] + logger.info('Committing to {0}:{1}'.format(docker_repo, + docker_tag or 'latest')) + client.tag(img_id, docker_repo, docker_tag) + if push: + logger.info('Pushing result to the main registry') + client.push(docker_repo) + return img_id diff --git a/contrib/brew/brew/git.py b/contrib/brew/brew/git.py new file mode 100644 index 0000000000..40cae8753b --- /dev/null +++ b/contrib/brew/brew/git.py @@ -0,0 +1,48 @@ +import tempfile +import logging + +from dulwich import index +from dulwich.client import get_transport_and_path +from dulwich.repo import Repo + +logger = logging.getLogger(__name__) + + +def clone_branch(repo_url, branch="master", folder=None): + return clone(repo_url, 'refs/heads/' + branch, folder) + + +def clone_tag(repo_url, tag, folder=None): + return clone(repo_url, 'refs/tags/' + tag, folder) + + +def clone(repo_url, ref=None, folder=None): + is_commit = False + if ref is None: + ref = 'refs/heads/master' + elif not ref.startswith('refs/'): + is_commit = True + logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref)) + if folder is None: + folder = tempfile.mkdtemp() + logger.debug("folder = {0}".format(folder)) + rep = Repo.init(folder) + client, relative_path = get_transport_and_path(repo_url) + logger.debug("client={0}".format(client)) + + remote_refs = client.fetch(relative_path, rep) + for k, v in remote_refs.iteritems(): + try: + rep.refs.add_if_new(k, v) + except: + pass + + if is_commit: + rep['HEAD'] = rep.commit(ref) + else: + rep['HEAD'] = remote_refs[ref] + indexfile = rep.index_path() + tree = rep["HEAD"].tree + index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree) + logger.debug("done") + return folder diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew new file mode 100755 index 0000000000..d1843d412b --- /dev/null +++ b/contrib/brew/docker-brew @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse + +import brew + + +DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' +DEFAULT_BRANCH = 'library' + +if __name__ == '__main__': + parser = argparse.ArgumentParser('Build the docker standard library') + parser.add_argument('--push', action='store_true', default=False, + help='push generated repositories to the official registry') + parser.add_argument('-n', metavar='NAMESPACE', default='library', + help='namespace used for generated repositories.' + ' Default is library') + parser.add_argument('-b', metavar='BRANCH', default=DEFAULT_BRANCH, + help='branch in the repository where the library definition' + ' files will be fetched. Default is ' + DEFAULT_BRANCH) + parser.add_argument('repository', default=DEFAULT_REPOSITORY, nargs='?', + help='git repository containing the library definition files.' + ' Default is ' + DEFAULT_REPOSITORY) + args = parser.parse_args() + brew.build_library(args.repository, args.b, args.n, args.push) \ No newline at end of file diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt new file mode 100644 index 0000000000..5278498791 --- /dev/null +++ b/contrib/brew/requirements.txt @@ -0,0 +1,2 @@ +dulwich==0.9.0 +docker==0.1.0 \ No newline at end of file diff --git a/contrib/brew/setup.py b/contrib/brew/setup.py new file mode 100644 index 0000000000..8a099c99ff --- /dev/null +++ b/contrib/brew/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +from setuptools import setup + +ROOT_DIR = os.path.dirname(__file__) +SOURCE_DIR = os.path.join(ROOT_DIR) + +test_requirements = [] +setup( + name="docker-brew", + version='0.0.1', + description="-", + packages=['dockerbrew'], + install_requires=['dulwich', 'docker'] + test_requirements, + zip_safe=False, + classifiers=['Development Status :: 3 - Alpha', + 'Environment :: Other Environment', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Utilities'], + ) From 3781a2cc4ba80373d1820c22555e82c42d92a787 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 19 Jul 2013 19:52:23 +0200 Subject: [PATCH 07/13] Added definition file for busybox and updated ubuntu --- library/busybox | 1 + library/ubuntu | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 library/busybox diff --git a/library/busybox b/library/busybox new file mode 100644 index 0000000000..b091054451 --- /dev/null +++ b/library/busybox @@ -0,0 +1 @@ +latest git://github.com/dotcloud/docker-busybox \ No newline at end of file diff --git a/library/ubuntu b/library/ubuntu index 046c8db319..7a4c77d99f 100644 --- a/library/ubuntu +++ b/library/ubuntu @@ -1,5 +1,7 @@ latest git://github.com/dotcloud/ubuntu-quantal quantal git://github.com/dotcloud/ubuntu-quantal 12.10 git://github.com/dotcloud/ubuntu-quantal -precise git://github.com/dotcloud/ubuntu-precise -12.04 git://github.com/dotcloud/ubuntu-precise +precise git://github.com/dotcloud/ubuntu-quantal B:precise +12.04 git://github.com/dotcloud/ubuntu-quantal B:precise +raring git://github.com/dotcloud/ubuntu-quantal B:raring +13.04 git://github.com/dotcloud/ubuntu-quantal B:raring From 7813f2a25e40018f9ad0a780df1c2e1d897666f5 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 19 Jul 2013 20:05:17 +0200 Subject: [PATCH 08/13] Updated git repos for precise and raring --- library/ubuntu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/ubuntu b/library/ubuntu index 7a4c77d99f..8a5cdfd445 100644 --- a/library/ubuntu +++ b/library/ubuntu @@ -1,7 +1,7 @@ latest git://github.com/dotcloud/ubuntu-quantal quantal git://github.com/dotcloud/ubuntu-quantal 12.10 git://github.com/dotcloud/ubuntu-quantal -precise git://github.com/dotcloud/ubuntu-quantal B:precise -12.04 git://github.com/dotcloud/ubuntu-quantal B:precise -raring git://github.com/dotcloud/ubuntu-quantal B:raring -13.04 git://github.com/dotcloud/ubuntu-quantal B:raring +precise git://github.com/shin-/ubuntu B:precise +12.04 git://github.com/shin-/ubuntu B:precise +raring git://github.com/shin-/ubuntu B:raring +13.04 git://github.com/shin-/ubuntu B:raring From 362f1735e6b60b6fcc6d435b29586252029f15a8 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Jul 2013 16:47:50 +0200 Subject: [PATCH 09/13] Brew: Avoid duplicate commands, added --debug option, added local repo support --- contrib/brew/brew/__init__.py | 2 +- contrib/brew/brew/brew.py | 18 +++++++++++++----- contrib/brew/docker-brew | 17 ++++++++--------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/contrib/brew/brew/__init__.py b/contrib/brew/brew/__init__.py index f693ad67a4..a9476ff5a4 100644 --- a/contrib/brew/brew/__init__.py +++ b/contrib/brew/brew/__init__.py @@ -1 +1 @@ -from brew import build_library +from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py index 69529201d6..ab0b886b2c 100644 --- a/contrib/brew/brew/brew.py +++ b/contrib/brew/brew/brew.py @@ -10,22 +10,30 @@ DEFAULT_BRANCH = 'library' logger = logging.getLogger(__name__) logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', - level='DEBUG') + level='INFO') client = docker.Client() processed = {} -def build_library(repository=None, branch=None, namespace=None, push=False): +def build_library(repository=None, branch=None, namespace=None, push=False, + debug=False): if repository is None: repository = DEFAULT_REPOSITORY if branch is None: branch = DEFAULT_BRANCH + if debug: + logger.setLevel('DEBUG') + + if not (repository.startswith('https://') or repository.startswith('git://')): + logger.info('Repository provided assumed to be a local path') + dst_folder = repository - logger.info('Cloning docker repo from {0}, branch: {1}'.format( - repository, branch)) #FIXME: set destination folder and only pull latest changes instead of # cloning the whole repo everytime - dst_folder = git.clone_branch(repository, branch) + if not dst_folder: + logger.info('Cloning docker repo from {0}, branch: {1}'.format( + repository, branch)) + dst_folder = git.clone_branch(repository, branch) for buildfile in os.listdir(os.path.join(dst_folder, 'library')): if buildfile == 'MAINTAINERS': continue diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew index d1843d412b..3bcd99a3ad 100755 --- a/contrib/brew/docker-brew +++ b/contrib/brew/docker-brew @@ -5,21 +5,20 @@ import argparse import brew -DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' -DEFAULT_BRANCH = 'library' - if __name__ == '__main__': parser = argparse.ArgumentParser('Build the docker standard library') parser.add_argument('--push', action='store_true', default=False, help='push generated repositories to the official registry') + parser.add_argument('--debug', default=False, action='store_true', + help='Enable debugging output') parser.add_argument('-n', metavar='NAMESPACE', default='library', help='namespace used for generated repositories.' ' Default is library') - parser.add_argument('-b', metavar='BRANCH', default=DEFAULT_BRANCH, + parser.add_argument('-b', metavar='BRANCH', default=brew.DEFAULT_BRANCH, help='branch in the repository where the library definition' - ' files will be fetched. Default is ' + DEFAULT_BRANCH) - parser.add_argument('repository', default=DEFAULT_REPOSITORY, nargs='?', - help='git repository containing the library definition files.' - ' Default is ' + DEFAULT_REPOSITORY) + ' files will be fetched. Default is ' + brew.DEFAULT_BRANCH) + parser.add_argument('repository', default=brew.DEFAULT_REPOSITORY, + nargs='?', help='git repository containing the library definition' + ' files. Default is ' + brew.DEFAULT_REPOSITORY) args = parser.parse_args() - brew.build_library(args.repository, args.b, args.n, args.push) \ No newline at end of file + brew.build_library(args.repository, args.b, args.n, args.push, args.debug) From 77ff53769714d4b33555b8e412868aad2cc788f9 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Jul 2013 16:49:19 +0200 Subject: [PATCH 10/13] Brew: Fixed docker-py requirement --- contrib/brew/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt index 5278498791..c66d0798d2 100644 --- a/contrib/brew/requirements.txt +++ b/contrib/brew/requirements.txt @@ -1,2 +1,2 @@ dulwich==0.9.0 -docker==0.1.0 \ No newline at end of file +docker-py==0.1.1 \ No newline at end of file From 94053b42252a1f70c16d2b72399904f79af0b688 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Jul 2013 23:47:44 +0200 Subject: [PATCH 11/13] Brew: Added cache prefilling and build summary --- contrib/brew/brew/brew.py | 54 ++++++++++++++++++++++++++++++++++++++- contrib/brew/docker-brew | 5 +++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py index ab0b886b2c..f361783d54 100644 --- a/contrib/brew/brew/brew.py +++ b/contrib/brew/brew/brew.py @@ -16,7 +16,9 @@ processed = {} def build_library(repository=None, branch=None, namespace=None, push=False, - debug=False): + debug=False, prefill=True): + dst_folder = None + summary = Summary() if repository is None: repository = DEFAULT_REPOSITORY if branch is None: @@ -38,7 +40,9 @@ def build_library(repository=None, branch=None, namespace=None, push=False, if buildfile == 'MAINTAINERS': continue f = open(os.path.join(dst_folder, 'library', buildfile)) + linecnt = 0 for line in f: + linecnt = linecnt + 1 logger.debug('{0} ---> {1}'.format(buildfile, line)) args = line.split() try: @@ -66,11 +70,19 @@ def build_library(repository=None, branch=None, namespace=None, push=False, else: raise RuntimeError('Incorrect line format, ' 'please refer to the docs') + if prefill: + logger.debug('Pulling {0} from official repository (cache ' + 'fill)'.format(buildfile)) + client.pull(buildfile) img = build_repo(url, ref, buildfile, tag, namespace, push) + summary.add_success(buildfile, (linecnt, line), img) processed['{0}@{1}'.format(url, ref)] = img except Exception as e: logger.exception(e) + summary.add_exception(buildfile, (linecnt, line), e) + f.close() + summary.print_summary() def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): @@ -93,3 +105,43 @@ def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): logger.info('Pushing result to the main registry') client.push(docker_repo) return img_id + + +class Summary(object): + def __init__(self): + self._summary = {} + self._has_exc = False + + def _add_data(self, image, linestr, data): + if image not in self._summary: + self._summary[image] = { linestr: data } + else: + self._summary[image][linestr] = data + + def add_exception(self, image, line, exc): + lineno, linestr = line + self._add_data(image, linestr, { 'line': lineno, 'exc': str(exc) }) + self._has_exc = True + + def add_success(self, image, line, img_id): + lineno, linestr = line + self._add_data(image, linestr, { 'line': lineno, 'id': img_id }) + + def print_summary(self, logger=None): + linesep = ''.center(61, '-') + '\n' + s = 'BREW BUILD SUMMARY\n' + linesep + success = 'OVERALL SUCCESS: {}\n'.format(not self._has_exc) + details = linesep + for image, lines in self._summary.iteritems(): + details = details + '{}\n{}'.format(image, linesep) + for linestr, data in lines.iteritems(): + details = details + '{0:2} | {1} | {2:50}\n'.format( + data['line'], + 'KO' if 'exc' in data else 'OK', + data['exc'] if 'exc' in data else data['id'] + ) + details = details + linesep + if logger: + logger.info(s + success + details) + else: + print s, success, details \ No newline at end of file diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew index 3bcd99a3ad..29052771c6 100755 --- a/contrib/brew/docker-brew +++ b/contrib/brew/docker-brew @@ -11,6 +11,8 @@ if __name__ == '__main__': help='push generated repositories to the official registry') parser.add_argument('--debug', default=False, action='store_true', help='Enable debugging output') + parser.add_argument('--noprefill', default=True, action='store_false', + dest='prefill', help='Disable cache prefill') parser.add_argument('-n', metavar='NAMESPACE', default='library', help='namespace used for generated repositories.' ' Default is library') @@ -21,4 +23,5 @@ if __name__ == '__main__': nargs='?', help='git repository containing the library definition' ' files. Default is ' + brew.DEFAULT_REPOSITORY) args = parser.parse_args() - brew.build_library(args.repository, args.b, args.n, args.push, args.debug) + brew.build_library(args.repository, args.b, args.n, args.push, args.debug, + args.prefill) From 12d575a6b1125d74168986ca39250c833ddbb033 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Jul 2013 21:00:36 +0200 Subject: [PATCH 12/13] Script cleans up downloaded repos, uses quiet build --- contrib/brew/brew/brew.py | 11 +++++++---- contrib/brew/requirements.txt | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py index f361783d54..91f7b01b94 100644 --- a/contrib/brew/brew/brew.py +++ b/contrib/brew/brew/brew.py @@ -1,5 +1,6 @@ import os import logging +from shutil import rmtree import docker @@ -82,7 +83,9 @@ def build_library(repository=None, branch=None, namespace=None, push=False, summary.add_exception(buildfile, (linecnt, line), e) f.close() - summary.print_summary() + if dst_folder != repository: + rmtree(dst_folder, True) + summary.print_summary(logger) def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): @@ -94,9 +97,9 @@ def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): if not 'Dockerfile' in os.listdir(dst_folder): raise RuntimeError('Dockerfile not found in cloned repository') logger.info('Building using dockerfile...') - img_id, logs = client.build(path=dst_folder) - - if not img_id: + img_id, logs = client.build(path=dst_folder, quiet=True) + rmtree(dst_folder, True) + else: img_id = processed['{0}@{1}'.format(repository, ref)] logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt index c66d0798d2..8006177ce6 100644 --- a/contrib/brew/requirements.txt +++ b/contrib/brew/requirements.txt @@ -1,2 +1,2 @@ dulwich==0.9.0 -docker-py==0.1.1 \ No newline at end of file +docker-py==0.1.2 \ No newline at end of file From 65c8e9242cc73fd2f1ea949703cf7da34e8e9650 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 6 Aug 2013 21:08:27 +0200 Subject: [PATCH 13/13] brew: added support for push on private registries. --- contrib/brew/brew/brew.py | 14 ++++++++++---- contrib/brew/docker-brew | 12 +++++++----- contrib/brew/requirements.txt | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py index 91f7b01b94..73d59877fb 100644 --- a/contrib/brew/brew/brew.py +++ b/contrib/brew/brew/brew.py @@ -17,7 +17,7 @@ processed = {} def build_library(repository=None, branch=None, namespace=None, push=False, - debug=False, prefill=True): + debug=False, prefill=True, registry=None): dst_folder = None summary = Summary() if repository is None: @@ -75,7 +75,8 @@ def build_library(repository=None, branch=None, namespace=None, push=False, logger.debug('Pulling {0} from official repository (cache ' 'fill)'.format(buildfile)) client.pull(buildfile) - img = build_repo(url, ref, buildfile, tag, namespace, push) + img = build_repo(url, ref, buildfile, tag, namespace, push, + registry) summary.add_success(buildfile, (linecnt, line), img) processed['{0}@{1}'.format(url, ref)] = img except Exception as e: @@ -88,7 +89,7 @@ def build_library(repository=None, branch=None, namespace=None, push=False, summary.print_summary(logger) -def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): +def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry): docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) img_id = None if '{0}@{1}'.format(repository, ref) not in processed.keys(): @@ -105,7 +106,12 @@ def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: - logger.info('Pushing result to the main registry') + logger.info('Pushing result to registry {0}'.format( + registry or "default")) + if registry is not None: + docker_repo = '{0}/{1}'.format(registry, docker_repo) + logger.info('Also tagging {0}'.format(docker_repo)) + client.tag(img_id, docker_repo, docker_tag) client.push(docker_repo) return img_id diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew index 29052771c6..ead6b65075 100755 --- a/contrib/brew/docker-brew +++ b/contrib/brew/docker-brew @@ -8,20 +8,22 @@ import brew if __name__ == '__main__': parser = argparse.ArgumentParser('Build the docker standard library') parser.add_argument('--push', action='store_true', default=False, - help='push generated repositories to the official registry') + help='Push generated repositories') parser.add_argument('--debug', default=False, action='store_true', help='Enable debugging output') parser.add_argument('--noprefill', default=True, action='store_false', dest='prefill', help='Disable cache prefill') parser.add_argument('-n', metavar='NAMESPACE', default='library', - help='namespace used for generated repositories.' + help='Namespace used for generated repositories.' ' Default is library') parser.add_argument('-b', metavar='BRANCH', default=brew.DEFAULT_BRANCH, - help='branch in the repository where the library definition' + help='Branch in the repository where the library definition' ' files will be fetched. Default is ' + brew.DEFAULT_BRANCH) parser.add_argument('repository', default=brew.DEFAULT_REPOSITORY, nargs='?', help='git repository containing the library definition' ' files. Default is ' + brew.DEFAULT_REPOSITORY) + parser.add_argument('--reg', default=None, help='Registry address to' + ' push build results to. Also sets push to true.') args = parser.parse_args() - brew.build_library(args.repository, args.b, args.n, args.push, args.debug, - args.prefill) + brew.build_library(args.repository, args.b, args.n, + args.push or args.reg is not None, args.debug, args.prefill, args.reg) diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt index 8006177ce6..78a574953d 100644 --- a/contrib/brew/requirements.txt +++ b/contrib/brew/requirements.txt @@ -1,2 +1,2 @@ dulwich==0.9.0 -docker-py==0.1.2 \ No newline at end of file +docker-py==0.1.3 \ No newline at end of file