testing; issue #603: Create AWS testing buildbot instance for docker CI

buildbot/README.rst

@@ -1,20 +0,0 @@
-Buildbot
-========
-
-Buildbot is a continuous integration system designed to automate the
-build/test cycle. By automatically rebuilding and testing the tree each time
-something has changed, build problems are pinpointed quickly, before other
-developers are inconvenienced by the failure.
-
-When running 'make hack' at the docker root directory, it spawns a virtual
-machine in the background running a buildbot instance and adds a git
-post-commit hook that automatically run docker tests for you.
-
-You can check your buildbot instance at http://192.168.33.21:8010/waterfall
-
-
-Buildbot dependencies
----------------------
-
-vagrant, virtualbox packages and python package requests
-

buildbot/Vagrantfile

@@ -1,28 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-$BUILDBOT_IP = '192.168.33.21'
-
-def v10(config)
-  config.vm.box = "quantal64_3.5.0-25"
-  config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box"
-  config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/..'
-  config.vm.network :hostonly, $BUILDBOT_IP
-
-  # Ensure puppet is installed on the instance
-  config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y puppet'
-
-  config.vm.provision :puppet do |puppet|
-    puppet.manifests_path = '.'
-    puppet.manifest_file  = 'buildbot.pp'
-    puppet.options = ['--templatedir','.']
-  end
-end
-
-Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config|
-  v10(config)
-end
-
-Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config|
-  v10(config)
-end

buildbot/buildbot-cfg/buildbot-cfg.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-
-# Auto setup of buildbot configuration. Package installation is being done
-# on buildbot.pp
-# Dependencies: buildbot, buildbot-slave, supervisor
-
-SLAVE_NAME='buildworker'
-SLAVE_SOCKET='localhost:9989'
-BUILDBOT_PWD='pass-docker'
-USER='vagrant'
-ROOT_PATH='/data/buildbot'
-DOCKER_PATH='/data/docker'
-BUILDBOT_CFG="$DOCKER_PATH/buildbot/buildbot-cfg"
-IP=$(grep BUILDBOT_IP /data/docker/buildbot/Vagrantfile | awk -F "'" '{ print $2; }')
-
-function run { su $USER -c "$1"; }
-
-export PATH=/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin
-
-# Exit if buildbot has already been installed
-[ -d "$ROOT_PATH" ] && exit 0
-
-# Setup buildbot
-run "mkdir -p ${ROOT_PATH}"
-cd ${ROOT_PATH}
-run "buildbot create-master master"
-run "cp $BUILDBOT_CFG/master.cfg master"
-run "sed -i 's/localhost/$IP/' master/master.cfg"
-run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
-
-# Allow buildbot subprocesses (docker tests) to properly run in containers,
-# in particular with docker -u
-run "sed -i 's/^umask = None/umask = 000/' ${ROOT_PATH}/slave/buildbot.tac"
-
-# Setup supervisor
-cp $BUILDBOT_CFG/buildbot.conf /etc/supervisor/conf.d/buildbot.conf
-sed -i "s/^chmod=0700.*0700./chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf
-kill -HUP `pgrep -f "/usr/bin/python /usr/bin/supervisord"`
-
-# Add git hook
-cp $BUILDBOT_CFG/post-commit $DOCKER_PATH/.git/hooks
-sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit
-

buildbot/buildbot.pp

@@ -1,32 +0,0 @@
-node default {
-    $USER = 'vagrant'
-    $ROOT_PATH = '/data/buildbot'
-    $DOCKER_PATH = '/data/docker'
-
-    exec {'apt_update': command => '/usr/bin/apt-get update' }
-    Package { require => Exec['apt_update'] }
-    group {'puppet': ensure => 'present'}
-
-    # Install dependencies
-    Package { ensure => 'installed' }
-    package { ['python-dev','python-pip','supervisor','lxc','bsdtar','git','golang']: }
-
-    file{[ '/data' ]:
-        owner => $USER, group => $USER, ensure => 'directory' }
-
-    file {'/var/tmp/requirements.txt':
-        content => template('requirements.txt') }
-
-    exec {'requirements':
-        require => [ Package['python-dev'], Package['python-pip'],
-            File['/var/tmp/requirements.txt'] ],
-        cwd     => '/var/tmp',
-        command => "/bin/sh -c '(/usr/bin/pip install -r requirements.txt;
-            rm /var/tmp/requirements.txt)'" }
-
-    exec {'buildbot-cfg-sh':
-        require => [ Package['supervisor'], Exec['requirements']],
-        path    => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin',
-        cwd     => '/data',
-        command => "$DOCKER_PATH/buildbot/buildbot-cfg/buildbot-cfg.sh" }
-}

testing/README.rst

@@ -0,0 +1,44 @@
+=======
+testing
+=======
+
+This directory contains testing related files.
+
+
+Buildbot
+========
+
+Buildbot is a continuous integration system designed to automate the
+build/test cycle. By automatically rebuilding and testing the tree each time
+something has changed, build problems are pinpointed quickly, before other
+developers are inconvenienced by the failure.
+
+We are running buildbot in an AWS instance to verify docker passes all tests
+when commits get pushed to the master branch.
+
+You can check docker's buildbot instance at http://docker-ci.dotcloud.com/waterfall
+
+
+Deployment
+~~~~~~~~~~
+
+::
+
+  # Define AWS credential environment variables
+  export AWS_ACCESS_KEY_ID=xxxxxxxxxxxx
+  export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxx
+  export AWS_KEYPAIR_NAME=xxxxxxxxxxxx
+  export AWS_SSH_PRIVKEY=xxxxxxxxxxxx
+
+  # Checkout docker
+  git clone git://github.com/dotcloud/docker.git
+
+  # Deploy docker on AWS
+  cd docker/testing
+  vagrant up --provider=aws
+
+
+Buildbot AWS dependencies
+-------------------------
+
+vagrant, virtualbox packages and vagrant aws plugin

testing/Vagrantfile

@@ -0,0 +1,56 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+BOX_NAME = "docker-ci"
+BOX_URI = "http://files.vagrantup.com/precise64.box"
+AWS_AMI = "ami-d0f89fb9"
+DOCKER_PATH = "/data/docker"
+CFG_PATH = "#{DOCKER_PATH}/testing/buildbot"
+BUILDBOT_IP = "192.168.33.41"
+on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \
+  Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \
+  (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox)
+USER = on_vbox ? "vagrant": "ubuntu"
+
+Vagrant::Config.run do |config|
+  # Setup virtual machine box. This VM configuration code is always executed.
+  config.vm.box = BOX_NAME
+  config.vm.box_url = BOX_URI
+  config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
+  config.vm.network :hostonly, BUILDBOT_IP
+
+  # Deploy buildbot and its dependencies if it was not done
+  if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
+    pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; "
+    # Deploy buildbot CI
+    pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
+      "pip install -r #{CFG_PATH}/requirements.txt; " \
+      "chown #{USER}.#{USER} /data; cd /data; " \
+      "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; "
+    # Install docker dependencies
+    pkg_cmd << "apt-get install -q -y python-software-properties; " \
+      "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \
+      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable make; "
+    # Activate new kernel
+    pkg_cmd << "shutdown -r +1; "
+    config.vm.provision :shell, :inline => pkg_cmd
+  end
+end
+
+# Providers were added on Vagrant >= 1.1.0
+Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
+  config.vm.provider :aws do |aws, override|
+    aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
+    aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+    aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
+    override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"]
+    override.ssh.username = USER
+    aws.ami = AWS_AMI
+    aws.region = "us-east-1"
+    aws.instance_type = "m1.small"
+    aws.security_groups = "gateway"
+  end
+
+  config.vm.provider :virtualbox do |vb|
+  end
+end

testing/buildbot/README.rst

@@ -0,0 +1 @@
+Buildbot configuration and setup files (except Vagrantfile located on ..)

testing/buildbot/buildbot.conf

@@ -0,0 +1,18 @@
+[program:buildmaster]
+command=twistd --nodaemon --no_save -y buildbot.tac
+directory=/data/buildbot/master
+chown= root:root
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/buildbot-master.log
+stderr_logfile=/var/log/supervisor/buildbot-master.log
+
+[program:buildworker]
+command=twistd --nodaemon --no_save -y buildbot.tac
+directory=/data/buildbot/slave
+chown= root:root
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/buildbot-slave.log
+stderr_logfile=/var/log/supervisor/buildbot-slave.log
+
+[group:buildbot]
+programs=buildmaster,buildworker

testing/buildbot/master.cfg

@@ -0,0 +1,55 @@
+import os
+from buildbot.buildslave import BuildSlave
+from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.basic import SingleBranchScheduler
+from buildbot.changes import filter
+from buildbot.config import BuilderConfig
+from buildbot.process.factory import BuildFactory
+from buildbot.steps.shell import ShellCommand
+from buildbot.status import html
+from buildbot.status.web import authz, auth
+
+PORT_WEB = 80           # Buildbot webserver port
+PORT_GITHUB = 8011      # Buildbot github hook port
+PORT_MASTER = 9989      # Port where buildbot master listen buildworkers
+TEST_USER = 'buildbot'  # Credential to authenticate build triggers
+TEST_PWD = 'docker'     # Credential to authenticate build triggers
+BUILDER_NAME = 'docker'
+BUILDPASSWORD = 'pass-docker'  # Credential to authenticate buildworkers
+GITHUB_DOCKER = "github.com/dotcloud/docker"
+DOCKER_PATH = "/data/docker"
+BUILDER_PATH = "/data/buildbot/slave/{0}/build".format(BUILDER_NAME)
+DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker'
+
+
+c = BuildmasterConfig = {}
+
+c['title'] = "Docker"
+c['titleURL'] = "waterfall"
+c['buildbotURL'] = "http://0.0.0.0:{0}/".format(PORT_WEB)
+c['db'] = {'db_url':"sqlite:///state.sqlite"}
+c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)]
+c['slavePortnum'] = PORT_MASTER
+
+c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])]
+c['schedulers'].append(SingleBranchScheduler(name="all",
+    change_filter=filter.ChangeFilter(branch='master'),treeStableTimer=None,
+    builderNames=[BUILDER_NAME]))
+
+# Docker test command
+test_cmd = ("cd /tmp; rm -rf {0}; export GOPATH={0}; go get -d {1}; cd {2}; "
+    "go test").format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH)
+
+# Builder
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='Docker',logEnviron=False,
+    usePTY=True,command=test_cmd))
+c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
+    factory=factory)]
+
+# Status
+authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]),
+    forceBuild='auth')
+c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]
+c['status'].append(html.WebStatus(http_port=PORT_GITHUB,allowForce=True,
+    change_hook_dialects={ 'github' : True }))

testing/buildbot/setup.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Setup of buildbot configuration. Package installation is being done by
+# Vagrantfile
+# Dependencies: buildbot, buildbot-slave, supervisor
+
+USER=$1
+CFG_PATH=$2
+BUILDBOT_PATH="/data/buildbot"
+DOCKER_PATH="/data/docker"
+SLAVE_NAME="buildworker"
+SLAVE_SOCKET="localhost:9989"
+BUILDBOT_PWD="pass-docker"
+export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
+
+function run { su $USER -c "$1"; }
+
+# Exit if buildbot has already been installed
+[ -d "$BUILDBOT_PATH" ] && exit 0
+
+# Setup buildbot
+run "mkdir -p $BUILDBOT_PATH"
+cd $BUILDBOT_PATH
+run "buildbot create-master master"
+run "cp $CFG_PATH/master.cfg master"
+run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg"
+run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
+
+# Allow buildbot subprocesses (docker tests) to properly run in containers,
+# in particular with docker -u
+run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"
+
+# Setup supervisor
+cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf
+sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf
+kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord")