Merge pull request #1603 from dotcloud/773-docker-ci-pr
773 docker-ci github pull request
This commit is contained in:
Daniel Mizyrycki
commit
0e6ee9632c
|
|
@ -0,0 +1,15 @@
|
|||
docker-ci github pull request
|
||||
=============================
|
||||
|
||||
The entire docker pull request test workflow is event driven by github. Its
|
||||
usage is fully automatic and the results are logged in docker-ci.dotcloud.com
|
||||
|
||||
Each time there is a pull request on docker's github project, github connects
|
||||
to docker-ci using github's rest API documented in http://developer.github.com/v3/repos/hooks
|
||||
The issued command to program github's notification PR event was:
|
||||
curl -u GITHUB_USER:GITHUB_PASSWORD -d '{"name":"web","active":true,"events":["pull_request"],"config":{"url":"http://docker-ci.dotcloud.com:8011/change_hook/github?project=docker"}}' https://api.github.com/repos/dotcloud/docker/hooks
|
||||
|
||||
buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it
|
||||
can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0)
|
||||
implemented this capability. Also we added a new scheduler to exclusively filter
|
||||
PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it.
|
||||
|
|
@ -2,11 +2,10 @@
|
|||
# vi: set ft=ruby :
|
||||
|
||||
BOX_NAME = "docker-ci"
|
||||
BOX_URI = "http://files.vagrantup.com/precise64.box"
|
||||
AWS_AMI = "ami-d0f89fb9"
|
||||
BOX_URI = "http://cloud-images.ubuntu.com/vagrant/raring/current/raring-server-cloudimg-amd64-vagrant-disk1.box"
|
||||
AWS_AMI = "ami-10314d79"
|
||||
DOCKER_PATH = "/data/docker"
|
||||
CFG_PATH = "#{DOCKER_PATH}/testing/buildbot"
|
||||
BUILDBOT_IP = "192.168.33.41"
|
||||
on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \
|
||||
Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \
|
||||
(on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox)
|
||||
|
|
@ -16,16 +15,22 @@ Vagrant::Config.run do |config|
|
|||
# Setup virtual machine box. This VM configuration code is always executed.
|
||||
config.vm.box = BOX_NAME
|
||||
config.vm.box_url = BOX_URI
|
||||
config.vm.forward_port 8010, 8010
|
||||
config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
|
||||
config.vm.network :hostonly, BUILDBOT_IP
|
||||
|
||||
|
||||
# Deploy buildbot and its dependencies if it was not done
|
||||
if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
|
||||
# Add memory limitation capabilities
|
||||
pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; '
|
||||
# Install new kernel
|
||||
pkg_cmd << "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; "
|
||||
# Adjust kernel
|
||||
pkg_cmd << "apt-get update -qq; "
|
||||
if on_vbox
|
||||
pkg_cmd << "apt-get install -q -y linux-image-extra-`uname -r`; "
|
||||
else
|
||||
pkg_cmd << "apt-get install -q -y linux-image-generic; "
|
||||
end
|
||||
|
||||
# Deploy buildbot CI
|
||||
pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
|
||||
"pip install -r #{CFG_PATH}/requirements.txt; " \
|
||||
|
|
@ -36,10 +41,12 @@ Vagrant::Config.run do |config|
|
|||
"#{CFG_PATH}/setup_credentials.sh #{USER} " \
|
||||
"#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; "
|
||||
# Install docker dependencies
|
||||
pkg_cmd << "apt-get install -q -y python-software-properties; " \
|
||||
"add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial golang-stable aufs-tools make; "
|
||||
# Activate new kernel
|
||||
pkg_cmd << "curl -s https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz | " \
|
||||
"tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go; " \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial aufs-tools make; " \
|
||||
"export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \
|
||||
"rm -rf ${GOPATH}/src/github.com/dotcloud/docker; "
|
||||
# Activate new kernel options
|
||||
pkg_cmd << "shutdown -r +1; "
|
||||
config.vm.provision :shell, :inline => pkg_cmd
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
# This file is part of Buildbot. Buildbot is free software: you can
|
||||
# redistribute it and/or modify it under the terms of the GNU General Public
|
||||
# License as published by the Free Software Foundation, version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# this program; if not, write to the Free Software Foundation, Inc., 51
|
||||
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# Copyright Buildbot Team Members
|
||||
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
github_buildbot.py is based on git_buildbot.py
|
||||
|
||||
github_buildbot.py will determine the repository information from the JSON
|
||||
HTTP POST it receives from github.com and build the appropriate repository.
|
||||
If your github repository is private, you must add a ssh key to the github
|
||||
repository for the user who initiated the build on the buildslave.
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
import datetime
|
||||
from twisted.python import log
|
||||
import calendar
|
||||
|
||||
try:
|
||||
import json
|
||||
assert json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
|
||||
# python is silly about how it handles timezones
|
||||
class fixedOffset(datetime.tzinfo):
|
||||
"""
|
||||
fixed offset timezone
|
||||
"""
|
||||
def __init__(self, minutes, hours, offsetSign = 1):
|
||||
self.minutes = int(minutes) * offsetSign
|
||||
self.hours = int(hours) * offsetSign
|
||||
self.offset = datetime.timedelta(minutes = self.minutes,
|
||||
hours = self.hours)
|
||||
|
||||
def utcoffset(self, dt):
|
||||
return self.offset
|
||||
|
||||
def dst(self, dt):
|
||||
return datetime.timedelta(0)
|
||||
|
||||
def convertTime(myTestTimestamp):
|
||||
#"1970-01-01T00:00:00+00:00"
|
||||
# Normalize myTestTimestamp
|
||||
if myTestTimestamp[-1] == 'Z':
|
||||
myTestTimestamp = myTestTimestamp[:-1] + '-00:00'
|
||||
matcher = re.compile(r'(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)([-+])(\d\d):(\d\d)')
|
||||
result = matcher.match(myTestTimestamp)
|
||||
(year, month, day, hour, minute, second, offsetsign, houroffset, minoffset) = \
|
||||
result.groups()
|
||||
if offsetsign == '+':
|
||||
offsetsign = 1
|
||||
else:
|
||||
offsetsign = -1
|
||||
|
||||
offsetTimezone = fixedOffset( minoffset, houroffset, offsetsign )
|
||||
myDatetime = datetime.datetime( int(year),
|
||||
int(month),
|
||||
int(day),
|
||||
int(hour),
|
||||
int(minute),
|
||||
int(second),
|
||||
0,
|
||||
offsetTimezone)
|
||||
return calendar.timegm( myDatetime.utctimetuple() )
|
||||
|
||||
def getChanges(request, options = None):
|
||||
"""
|
||||
Reponds only to POST events and starts the build process
|
||||
|
||||
:arguments:
|
||||
request
|
||||
the http request object
|
||||
"""
|
||||
payload = json.loads(request.args['payload'][0])
|
||||
if 'pull_request' in payload:
|
||||
user = payload['repository']['owner']['login']
|
||||
repo = payload['repository']['name']
|
||||
repo_url = payload['repository']['html_url']
|
||||
else:
|
||||
user = payload['repository']['owner']['name']
|
||||
repo = payload['repository']['name']
|
||||
repo_url = payload['repository']['url']
|
||||
project = request.args.get('project', None)
|
||||
if project:
|
||||
project = project[0]
|
||||
elif project is None:
|
||||
project = ''
|
||||
# This field is unused:
|
||||
#private = payload['repository']['private']
|
||||
changes = process_change(payload, user, repo, repo_url, project)
|
||||
log.msg("Received %s changes from github" % len(changes))
|
||||
return (changes, 'git')
|
||||
|
||||
def process_change(payload, user, repo, repo_url, project):
|
||||
"""
|
||||
Consumes the JSON as a python object and actually starts the build.
|
||||
|
||||
:arguments:
|
||||
payload
|
||||
Python Object that represents the JSON sent by GitHub Service
|
||||
Hook.
|
||||
"""
|
||||
changes = []
|
||||
|
||||
newrev = payload['after'] if 'after' in payload else payload['pull_request']['head']['sha']
|
||||
refname = payload['ref'] if 'ref' in payload else payload['pull_request']['head']['ref']
|
||||
|
||||
# We only care about regular heads, i.e. branches
|
||||
match = re.match(r"^(refs\/heads\/|)([^/]+)$", refname)
|
||||
if not match:
|
||||
log.msg("Ignoring refname `%s': Not a branch" % refname)
|
||||
return []
|
||||
|
||||
branch = match.groups()[1]
|
||||
if re.match(r"^0*$", newrev):
|
||||
log.msg("Branch `%s' deleted, ignoring" % branch)
|
||||
return []
|
||||
else:
|
||||
if 'pull_request' in payload:
|
||||
changes = [{
|
||||
'category' : 'github_pullrequest',
|
||||
'who' : user,
|
||||
'files' : [],
|
||||
'comments' : payload['pull_request']['title'],
|
||||
'revision' : newrev,
|
||||
'when' : convertTime(payload['pull_request']['updated_at']),
|
||||
'branch' : branch,
|
||||
'revlink' : '{0}/commit/{1}'.format(repo_url,newrev),
|
||||
'repository' : repo_url,
|
||||
'project' : project }]
|
||||
return changes
|
||||
for commit in payload['commits']:
|
||||
files = []
|
||||
if 'added' in commit:
|
||||
files.extend(commit['added'])
|
||||
if 'modified' in commit:
|
||||
files.extend(commit['modified'])
|
||||
if 'removed' in commit:
|
||||
files.extend(commit['removed'])
|
||||
when = convertTime( commit['timestamp'])
|
||||
log.msg("New revision: %s" % commit['id'][:8])
|
||||
chdict = dict(
|
||||
who = commit['author']['name']
|
||||
+ " <" + commit['author']['email'] + ">",
|
||||
files = files,
|
||||
comments = commit['message'],
|
||||
revision = commit['id'],
|
||||
when = when,
|
||||
branch = branch,
|
||||
revlink = commit['url'],
|
||||
repository = repo_url,
|
||||
project = project)
|
||||
changes.append(chdict)
|
||||
return changes
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ GITHUB_DOCKER = 'github.com/dotcloud/docker'
|
|||
BUILDBOT_PATH = '/data/buildbot'
|
||||
DOCKER_PATH = '/data/docker'
|
||||
BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
|
||||
DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker'
|
||||
PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'
|
||||
|
||||
# Credentials set by setup.sh and Vagrantfile
|
||||
BUILDBOT_PWD = ''
|
||||
|
|
@ -49,6 +49,9 @@ c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME,
|
|||
c['schedulers'] += [SingleBranchScheduler(name="all",
|
||||
change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None,
|
||||
builderNames=[BUILDER_NAME])]
|
||||
c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
|
||||
change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
|
||||
builderNames=['pullrequest'])]
|
||||
c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'],
|
||||
hour=0, minute=30)]
|
||||
|
||||
|
|
@ -57,12 +60,25 @@ c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage',
|
|||
# Docker commit test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True,
|
||||
command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; "
|
||||
"go get -d {1}; cd {2}; git reset --hard %(src::revision:-unknown)s; "
|
||||
"go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))]))
|
||||
command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
|
||||
"cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
|
||||
"git reset --hard %(src::revision)s; go test -v".format(
|
||||
BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
|
||||
|
||||
c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker pull request test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True,
|
||||
command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
|
||||
"cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
|
||||
"git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; "
|
||||
"git checkout %(src::revision)s; git rebase master; go test -v".format(
|
||||
PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
|
||||
c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker coverage test
|
||||
coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n'
|
||||
'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n'
|
||||
|
|
|
|||
|
|
@ -5,3 +5,4 @@ buildbot_slave==0.8.7p1
|
|||
nose==1.2.1
|
||||
requests==1.1.0
|
||||
flask==0.10.1
|
||||
simplejson==2.3.2
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
|
|||
run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
|
||||
run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
|
||||
|
||||
# Patch github webstatus to capture pull requests
|
||||
cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks
|
||||
|
||||
# Allow buildbot subprocesses (docker tests) to properly run in containers,
|
||||
# in particular with docker -u
|
||||
run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"
|
||||
|
|
|
|||
Loading…
Reference in New Issue