diff --git a/hack/infrastructure/docker-ci.rst b/hack/infrastructure/docker-ci.rst new file mode 100644 index 0000000000..abb8492cf0 --- /dev/null +++ b/hack/infrastructure/docker-ci.rst @@ -0,0 +1,15 @@ +docker-ci github pull request +============================= + +The entire docker pull request test workflow is event driven by github. Its +usage is fully automatic and the results are logged in docker-ci.dotcloud.com + +Each time there is a pull request on docker's github project, github connects +to docker-ci using github's rest API documented in http://developer.github.com/v3/repos/hooks +The issued command to program github's notification PR event was: +curl -u GITHUB_USER:GITHUB_PASSWORD -d '{"name":"web","active":true,"events":["pull_request"],"config":{"url":"http://docker-ci.dotcloud.com:8011/change_hook/github?project=docker"}}' https://api.github.com/repos/dotcloud/docker/hooks + +buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it +can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0) +implemented this capability. Also we added a new scheduler to exclusively filter +PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it. diff --git a/testing/Vagrantfile b/testing/Vagrantfile index 033f14b3de..fd1c5916c8 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -2,11 +2,10 @@ # vi: set ft=ruby : BOX_NAME = "docker-ci" -BOX_URI = "http://files.vagrantup.com/precise64.box" -AWS_AMI = "ami-d0f89fb9" +BOX_URI = "http://cloud-images.ubuntu.com/vagrant/raring/current/raring-server-cloudimg-amd64-vagrant-disk1.box" +AWS_AMI = "ami-10314d79" DOCKER_PATH = "/data/docker" CFG_PATH = "#{DOCKER_PATH}/testing/buildbot" -BUILDBOT_IP = "192.168.33.41" on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \ Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \ (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox) @@ -16,16 +15,22 @@ Vagrant::Config.run do |config| # Setup virtual machine box. This VM configuration code is always executed. config.vm.box = BOX_NAME config.vm.box_url = BOX_URI + config.vm.forward_port 8010, 8010 config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." - config.vm.network :hostonly, BUILDBOT_IP # Deploy buildbot and its dependencies if it was not done if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? # Add memory limitation capabilities pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; ' - # Install new kernel - pkg_cmd << "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; " + # Adjust kernel + pkg_cmd << "apt-get update -qq; " + if on_vbox + pkg_cmd << "apt-get install -q -y linux-image-extra-`uname -r`; " + else + pkg_cmd << "apt-get install -q -y linux-image-generic; " + end + # Deploy buildbot CI pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ "pip install -r #{CFG_PATH}/requirements.txt; " \ @@ -36,10 +41,12 @@ Vagrant::Config.run do |config| "#{CFG_PATH}/setup_credentials.sh #{USER} " \ "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies - pkg_cmd << "apt-get install -q -y python-software-properties; " \ - "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ - "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial golang-stable aufs-tools make; " - # Activate new kernel + pkg_cmd << "curl -s https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz | " \ + "tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go; " \ + "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial aufs-tools make; " \ + "export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \ + "rm -rf ${GOPATH}/src/github.com/dotcloud/docker; " + # Activate new kernel options pkg_cmd << "shutdown -r +1; " config.vm.provision :shell, :inline => pkg_cmd end diff --git a/testing/buildbot/github.py b/testing/buildbot/github.py new file mode 100644 index 0000000000..b0fe98a135 --- /dev/null +++ b/testing/buildbot/github.py @@ -0,0 +1,169 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +#!/usr/bin/env python +""" +github_buildbot.py is based on git_buildbot.py + +github_buildbot.py will determine the repository information from the JSON +HTTP POST it receives from github.com and build the appropriate repository. +If your github repository is private, you must add a ssh key to the github +repository for the user who initiated the build on the buildslave. + +""" + +import re +import datetime +from twisted.python import log +import calendar + +try: + import json + assert json +except ImportError: + import simplejson as json + +# python is silly about how it handles timezones +class fixedOffset(datetime.tzinfo): + """ + fixed offset timezone + """ + def __init__(self, minutes, hours, offsetSign = 1): + self.minutes = int(minutes) * offsetSign + self.hours = int(hours) * offsetSign + self.offset = datetime.timedelta(minutes = self.minutes, + hours = self.hours) + + def utcoffset(self, dt): + return self.offset + + def dst(self, dt): + return datetime.timedelta(0) + +def convertTime(myTestTimestamp): + #"1970-01-01T00:00:00+00:00" + # Normalize myTestTimestamp + if myTestTimestamp[-1] == 'Z': + myTestTimestamp = myTestTimestamp[:-1] + '-00:00' + matcher = re.compile(r'(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)([-+])(\d\d):(\d\d)') + result = matcher.match(myTestTimestamp) + (year, month, day, hour, minute, second, offsetsign, houroffset, minoffset) = \ + result.groups() + if offsetsign == '+': + offsetsign = 1 + else: + offsetsign = -1 + + offsetTimezone = fixedOffset( minoffset, houroffset, offsetsign ) + myDatetime = datetime.datetime( int(year), + int(month), + int(day), + int(hour), + int(minute), + int(second), + 0, + offsetTimezone) + return calendar.timegm( myDatetime.utctimetuple() ) + +def getChanges(request, options = None): + """ + Reponds only to POST events and starts the build process + + :arguments: + request + the http request object + """ + payload = json.loads(request.args['payload'][0]) + if 'pull_request' in payload: + user = payload['repository']['owner']['login'] + repo = payload['repository']['name'] + repo_url = payload['repository']['html_url'] + else: + user = payload['repository']['owner']['name'] + repo = payload['repository']['name'] + repo_url = payload['repository']['url'] + project = request.args.get('project', None) + if project: + project = project[0] + elif project is None: + project = '' + # This field is unused: + #private = payload['repository']['private'] + changes = process_change(payload, user, repo, repo_url, project) + log.msg("Received %s changes from github" % len(changes)) + return (changes, 'git') + +def process_change(payload, user, repo, repo_url, project): + """ + Consumes the JSON as a python object and actually starts the build. + + :arguments: + payload + Python Object that represents the JSON sent by GitHub Service + Hook. + """ + changes = [] + + newrev = payload['after'] if 'after' in payload else payload['pull_request']['head']['sha'] + refname = payload['ref'] if 'ref' in payload else payload['pull_request']['head']['ref'] + + # We only care about regular heads, i.e. branches + match = re.match(r"^(refs\/heads\/|)([^/]+)$", refname) + if not match: + log.msg("Ignoring refname `%s': Not a branch" % refname) + return [] + + branch = match.groups()[1] + if re.match(r"^0*$", newrev): + log.msg("Branch `%s' deleted, ignoring" % branch) + return [] + else: + if 'pull_request' in payload: + changes = [{ + 'category' : 'github_pullrequest', + 'who' : user, + 'files' : [], + 'comments' : payload['pull_request']['title'], + 'revision' : newrev, + 'when' : convertTime(payload['pull_request']['updated_at']), + 'branch' : branch, + 'revlink' : '{0}/commit/{1}'.format(repo_url,newrev), + 'repository' : repo_url, + 'project' : project }] + return changes + for commit in payload['commits']: + files = [] + if 'added' in commit: + files.extend(commit['added']) + if 'modified' in commit: + files.extend(commit['modified']) + if 'removed' in commit: + files.extend(commit['removed']) + when = convertTime( commit['timestamp']) + log.msg("New revision: %s" % commit['id'][:8]) + chdict = dict( + who = commit['author']['name'] + + " <" + commit['author']['email'] + ">", + files = files, + comments = commit['message'], + revision = commit['id'], + when = when, + branch = branch, + revlink = commit['url'], + repository = repo_url, + project = project) + changes.append(chdict) + return changes + diff --git a/testing/buildbot/master.cfg b/testing/buildbot/master.cfg index 05dcacbf58..cc261c7a3e 100644 --- a/testing/buildbot/master.cfg +++ b/testing/buildbot/master.cfg @@ -22,7 +22,7 @@ GITHUB_DOCKER = 'github.com/dotcloud/docker' BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) -DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' +PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build' # Credentials set by setup.sh and Vagrantfile BUILDBOT_PWD = '' @@ -49,6 +49,9 @@ c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] +c['schedulers'] += [SingleBranchScheduler(name='pullrequest', + change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, + builderNames=['pullrequest'])] c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] @@ -57,12 +60,25 @@ c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage', # Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, - command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " - "go get -d {1}; cd {2}; git reset --hard %(src::revision:-unknown)s; " - "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) + command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; " + "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; " + "git reset --hard %(src::revision)s; go test -v".format( + BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))])) + c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] +# Docker pull request test +factory = BuildFactory() +factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True, + command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; " + "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; " + "git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; " + "git checkout %(src::revision)s; git rebase master; go test -v".format( + PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))])) +c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], + factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' diff --git a/testing/buildbot/requirements.txt b/testing/buildbot/requirements.txt index 4e183ba062..c5d7dd0191 100644 --- a/testing/buildbot/requirements.txt +++ b/testing/buildbot/requirements.txt @@ -5,3 +5,4 @@ buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 flask==0.10.1 +simplejson==2.3.2 diff --git a/testing/buildbot/setup.sh b/testing/buildbot/setup.sh index 937533ba1f..99e4f7f104 100755 --- a/testing/buildbot/setup.sh +++ b/testing/buildbot/setup.sh @@ -36,6 +36,9 @@ run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg" run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg" run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" +# Patch github webstatus to capture pull requests +cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks + # Allow buildbot subprocesses (docker tests) to properly run in containers, # in particular with docker -u run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"