Merge pull request #1603 from dotcloud/773-docker-ci-pr

773 docker-ci github pull request

hack/infrastructure/docker-ci.rst

@@ -0,0 +1,15 @@
+docker-ci github pull request
+=============================
+
+The entire docker pull request test workflow is event driven by github. Its
+usage is fully automatic and the results are logged in docker-ci.dotcloud.com
+
+Each time there is a pull request on docker's github project, github connects
+to docker-ci using github's rest API documented in http://developer.github.com/v3/repos/hooks
+The issued command to program github's notification PR event was:
+curl -u GITHUB_USER:GITHUB_PASSWORD -d '{"name":"web","active":true,"events":["pull_request"],"config":{"url":"http://docker-ci.dotcloud.com:8011/change_hook/github?project=docker"}}' https://api.github.com/repos/dotcloud/docker/hooks
+
+buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it
+can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0)
+implemented this capability. Also we added a new scheduler to exclusively filter
+PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it.

testing/Vagrantfile

@@ -2,11 +2,10 @@
 # vi: set ft=ruby :
 
 BOX_NAME = "docker-ci"
-BOX_URI = "http://files.vagrantup.com/precise64.box"
+BOX_URI = "http://cloud-images.ubuntu.com/vagrant/raring/current/raring-server-cloudimg-amd64-vagrant-disk1.box"
-AWS_AMI = "ami-d0f89fb9"
+AWS_AMI = "ami-10314d79"
 DOCKER_PATH = "/data/docker"
 CFG_PATH = "#{DOCKER_PATH}/testing/buildbot"
-BUILDBOT_IP = "192.168.33.41"
 on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \
   Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \
   (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox)
@@ -16,16 +15,22 @@ Vagrant::Config.run do |config|
   # Setup virtual machine box. This VM configuration code is always executed.
   config.vm.box = BOX_NAME
   config.vm.box_url = BOX_URI
+  config.vm.forward_port 8010, 8010
   config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
-  config.vm.network :hostonly, BUILDBOT_IP
 
 
   # Deploy buildbot and its dependencies if it was not done
   if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
     # Add memory limitation capabilities
     pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; '
-    # Install new kernel
+    # Adjust kernel
-    pkg_cmd << "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; "
+    pkg_cmd << "apt-get update -qq; "
+    if on_vbox
+      pkg_cmd << "apt-get install -q -y linux-image-extra-`uname -r`; "
+    else
+      pkg_cmd << "apt-get install -q -y linux-image-generic; "
+    end
+
     # Deploy buildbot CI
     pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
       "pip install -r #{CFG_PATH}/requirements.txt; " \
@@ -36,10 +41,12 @@ Vagrant::Config.run do |config|
       "#{CFG_PATH}/setup_credentials.sh #{USER} " \
         "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; "
     # Install docker dependencies
-    pkg_cmd << "apt-get install -q -y python-software-properties; " \
+    pkg_cmd << "curl -s https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz | " \
-      "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \
+      "tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go; " \
-      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial golang-stable aufs-tools make; "
+      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial aufs-tools make; " \
-    # Activate new kernel
+      "export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \
+      "rm -rf ${GOPATH}/src/github.com/dotcloud/docker; "
+    # Activate new kernel options
     pkg_cmd << "shutdown -r +1; "
     config.vm.provision :shell, :inline => pkg_cmd
   end

testing/buildbot/github.py

@@ -0,0 +1,169 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+#!/usr/bin/env python
+"""
+github_buildbot.py is based on git_buildbot.py
+
+github_buildbot.py will determine the repository information from the JSON 
+HTTP POST it receives from github.com and build the appropriate repository.
+If your github repository is private, you must add a ssh key to the github
+repository for the user who initiated the build on the buildslave.
+
+"""
+
+import re
+import datetime
+from twisted.python import log
+import calendar
+
+try:
+    import json
+    assert json
+except ImportError:
+    import simplejson as json
+
+# python is silly about how it handles timezones
+class fixedOffset(datetime.tzinfo):
+    """
+    fixed offset timezone
+    """
+    def __init__(self, minutes, hours, offsetSign = 1):
+        self.minutes = int(minutes) * offsetSign
+        self.hours   = int(hours)   * offsetSign
+        self.offset  = datetime.timedelta(minutes = self.minutes,
+                                         hours   = self.hours)
+
+    def utcoffset(self, dt):
+        return self.offset
+
+    def dst(self, dt):
+        return datetime.timedelta(0)
+    
+def convertTime(myTestTimestamp):
+    #"1970-01-01T00:00:00+00:00"
+    # Normalize myTestTimestamp
+    if myTestTimestamp[-1] == 'Z':
+        myTestTimestamp = myTestTimestamp[:-1] + '-00:00'
+    matcher = re.compile(r'(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)([-+])(\d\d):(\d\d)')
+    result  = matcher.match(myTestTimestamp)
+    (year, month, day, hour, minute, second, offsetsign, houroffset, minoffset) = \
+        result.groups()
+    if offsetsign == '+':
+        offsetsign = 1
+    else:
+        offsetsign = -1
+    
+    offsetTimezone = fixedOffset( minoffset, houroffset, offsetsign )
+    myDatetime = datetime.datetime( int(year),
+                                    int(month),
+                                    int(day),
+                                    int(hour),
+                                    int(minute),
+                                    int(second),
+                                    0,
+                                    offsetTimezone)
+    return calendar.timegm( myDatetime.utctimetuple() )
+
+def getChanges(request, options = None):
+        """
+        Reponds only to POST events and starts the build process
+        
+        :arguments:
+            request
+                the http request object
+        """
+        payload = json.loads(request.args['payload'][0])
+	if 'pull_request' in payload:
+	    user = payload['repository']['owner']['login']
+	    repo = payload['repository']['name']
+            repo_url = payload['repository']['html_url']
+	else:
+	    user = payload['repository']['owner']['name']
+            repo = payload['repository']['name']
+            repo_url = payload['repository']['url']
+        project = request.args.get('project', None)
+        if project:
+            project = project[0]
+        elif project is None:
+            project = ''
+        # This field is unused:
+        #private = payload['repository']['private']
+        changes = process_change(payload, user, repo, repo_url, project)
+        log.msg("Received %s changes from github" % len(changes))
+        return (changes, 'git')
+
+def process_change(payload, user, repo, repo_url, project):
+        """
+        Consumes the JSON as a python object and actually starts the build.
+        
+        :arguments:
+            payload
+                Python Object that represents the JSON sent by GitHub Service
+                Hook.
+        """
+        changes = []
+	
+        newrev = payload['after'] if 'after' in payload else payload['pull_request']['head']['sha']
+        refname = payload['ref'] if 'ref' in payload else payload['pull_request']['head']['ref']
+
+        # We only care about regular heads, i.e. branches
+        match = re.match(r"^(refs\/heads\/|)([^/]+)$", refname)
+        if not match:
+            log.msg("Ignoring refname `%s': Not a branch" % refname)
+            return []
+
+        branch = match.groups()[1]
+        if re.match(r"^0*$", newrev):
+            log.msg("Branch `%s' deleted, ignoring" % branch)
+            return []
+        else: 
+	    if 'pull_request' in payload:
+		changes = [{
+		    'category'   : 'github_pullrequest',
+                    'who'        : user,
+                    'files'      : [],
+                    'comments'   : payload['pull_request']['title'], 
+                    'revision'   : newrev,
+                    'when'       : convertTime(payload['pull_request']['updated_at']),
+                    'branch'     : branch,
+                    'revlink'    : '{0}/commit/{1}'.format(repo_url,newrev),
+                    'repository' : repo_url,
+                    'project'  : project  }] 
+		return changes
+            for commit in payload['commits']:
+                files = []
+                if 'added' in commit:
+                    files.extend(commit['added'])
+                if 'modified' in commit:
+                    files.extend(commit['modified'])
+                if 'removed' in commit:
+                    files.extend(commit['removed'])
+                when =  convertTime( commit['timestamp'])
+                log.msg("New revision: %s" % commit['id'][:8])
+                chdict = dict(
+                    who      = commit['author']['name'] 
+                                + " <" + commit['author']['email'] + ">",
+                    files    = files,
+                    comments = commit['message'], 
+                    revision = commit['id'],
+                    when     = when,
+                    branch   = branch,
+                    revlink  = commit['url'], 
+                    repository = repo_url,
+                    project  = project)
+                changes.append(chdict) 
+            return changes
+        

testing/buildbot/master.cfg

@@ -22,7 +22,7 @@ GITHUB_DOCKER = 'github.com/dotcloud/docker'
 BUILDBOT_PATH = '/data/buildbot'
 DOCKER_PATH = '/data/docker'
 BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
-DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker'
+PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'
 
 # Credentials set by setup.sh and Vagrantfile
 BUILDBOT_PWD = ''
@@ -49,6 +49,9 @@ c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME,
 c['schedulers'] += [SingleBranchScheduler(name="all",
     change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None,
     builderNames=[BUILDER_NAME])]
+c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
+    change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
+    builderNames=['pullrequest'])]
 c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'],
     hour=0, minute=30)]
 
@@ -57,12 +60,25 @@ c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage',
 # Docker commit test
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True,
-    command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; "
+    command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
-    "go get -d {1}; cd {2}; git reset --hard %(src::revision:-unknown)s; "
+    "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
-    "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))]))
+    "git reset --hard %(src::revision)s; go test -v".format(
+    BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
+
 c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
     factory=factory)]
 
+# Docker pull request test
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True,
+    command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
+    "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
+    "git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; "
+    "git checkout %(src::revision)s; git rebase master; go test -v".format(
+    PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
+c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
+    factory=factory)]
+
 # Docker coverage test
 coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n'
     'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n'

testing/buildbot/requirements.txt

@@ -5,3 +5,4 @@ buildbot_slave==0.8.7p1
 nose==1.2.1
 requests==1.1.0
 flask==0.10.1
+simplejson==2.3.2

testing/buildbot/setup.sh

@@ -36,6 +36,9 @@ run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
 run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
 run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
 
+# Patch github webstatus to capture pull requests
+cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks
+
 # Allow buildbot subprocesses (docker tests) to properly run in containers,
 # in particular with docker -u
 run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"