From 085ee6fcc4cc861b715194e6f144a983350331f3 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 13 May 2013 23:10:00 -0700 Subject: [PATCH] testing; issue #603: Create AWS testing buildbot instance for docker CI --- buildbot/README.rst | 20 ------- buildbot/Vagrantfile | 28 ---------- buildbot/buildbot-cfg/buildbot-cfg.sh | 43 -------------- buildbot/buildbot.pp | 32 ----------- testing/README.rst | 44 +++++++++++++++ testing/Vagrantfile | 56 +++++++++++++++++++ testing/buildbot/README.rst | 1 + testing/buildbot/buildbot.conf | 18 ++++++ testing/buildbot/master.cfg | 55 ++++++++++++++++++ .../buildbot}/requirements.txt | 0 testing/buildbot/setup.sh | 36 ++++++++++++ 11 files changed, 210 insertions(+), 123 deletions(-) delete mode 100644 buildbot/README.rst delete mode 100644 buildbot/Vagrantfile delete mode 100755 buildbot/buildbot-cfg/buildbot-cfg.sh delete mode 100644 buildbot/buildbot.pp create mode 100644 testing/README.rst create mode 100644 testing/Vagrantfile create mode 100644 testing/buildbot/README.rst create mode 100644 testing/buildbot/buildbot.conf create mode 100644 testing/buildbot/master.cfg rename {buildbot => testing/buildbot}/requirements.txt (100%) create mode 100755 testing/buildbot/setup.sh diff --git a/buildbot/README.rst b/buildbot/README.rst deleted file mode 100644 index a52b9769ef..0000000000 --- a/buildbot/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -Buildbot -======== - -Buildbot is a continuous integration system designed to automate the -build/test cycle. By automatically rebuilding and testing the tree each time -something has changed, build problems are pinpointed quickly, before other -developers are inconvenienced by the failure. - -When running 'make hack' at the docker root directory, it spawns a virtual -machine in the background running a buildbot instance and adds a git -post-commit hook that automatically run docker tests for you. - -You can check your buildbot instance at http://192.168.33.21:8010/waterfall - - -Buildbot dependencies ---------------------- - -vagrant, virtualbox packages and python package requests - diff --git a/buildbot/Vagrantfile b/buildbot/Vagrantfile deleted file mode 100644 index ea027f0666..0000000000 --- a/buildbot/Vagrantfile +++ /dev/null @@ -1,28 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -$BUILDBOT_IP = '192.168.33.21' - -def v10(config) - config.vm.box = "quantal64_3.5.0-25" - config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" - config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/..' - config.vm.network :hostonly, $BUILDBOT_IP - - # Ensure puppet is installed on the instance - config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y puppet' - - config.vm.provision :puppet do |puppet| - puppet.manifests_path = '.' - puppet.manifest_file = 'buildbot.pp' - puppet.options = ['--templatedir','.'] - end -end - -Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config| - v10(config) -end - -Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config| - v10(config) -end diff --git a/buildbot/buildbot-cfg/buildbot-cfg.sh b/buildbot/buildbot-cfg/buildbot-cfg.sh deleted file mode 100755 index 5e4e7432fd..0000000000 --- a/buildbot/buildbot-cfg/buildbot-cfg.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Auto setup of buildbot configuration. Package installation is being done -# on buildbot.pp -# Dependencies: buildbot, buildbot-slave, supervisor - -SLAVE_NAME='buildworker' -SLAVE_SOCKET='localhost:9989' -BUILDBOT_PWD='pass-docker' -USER='vagrant' -ROOT_PATH='/data/buildbot' -DOCKER_PATH='/data/docker' -BUILDBOT_CFG="$DOCKER_PATH/buildbot/buildbot-cfg" -IP=$(grep BUILDBOT_IP /data/docker/buildbot/Vagrantfile | awk -F "'" '{ print $2; }') - -function run { su $USER -c "$1"; } - -export PATH=/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin - -# Exit if buildbot has already been installed -[ -d "$ROOT_PATH" ] && exit 0 - -# Setup buildbot -run "mkdir -p ${ROOT_PATH}" -cd ${ROOT_PATH} -run "buildbot create-master master" -run "cp $BUILDBOT_CFG/master.cfg master" -run "sed -i 's/localhost/$IP/' master/master.cfg" -run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" - -# Allow buildbot subprocesses (docker tests) to properly run in containers, -# in particular with docker -u -run "sed -i 's/^umask = None/umask = 000/' ${ROOT_PATH}/slave/buildbot.tac" - -# Setup supervisor -cp $BUILDBOT_CFG/buildbot.conf /etc/supervisor/conf.d/buildbot.conf -sed -i "s/^chmod=0700.*0700./chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf -kill -HUP `pgrep -f "/usr/bin/python /usr/bin/supervisord"` - -# Add git hook -cp $BUILDBOT_CFG/post-commit $DOCKER_PATH/.git/hooks -sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit - diff --git a/buildbot/buildbot.pp b/buildbot/buildbot.pp deleted file mode 100644 index 8109cdc2a0..0000000000 --- a/buildbot/buildbot.pp +++ /dev/null @@ -1,32 +0,0 @@ -node default { - $USER = 'vagrant' - $ROOT_PATH = '/data/buildbot' - $DOCKER_PATH = '/data/docker' - - exec {'apt_update': command => '/usr/bin/apt-get update' } - Package { require => Exec['apt_update'] } - group {'puppet': ensure => 'present'} - - # Install dependencies - Package { ensure => 'installed' } - package { ['python-dev','python-pip','supervisor','lxc','bsdtar','git','golang']: } - - file{[ '/data' ]: - owner => $USER, group => $USER, ensure => 'directory' } - - file {'/var/tmp/requirements.txt': - content => template('requirements.txt') } - - exec {'requirements': - require => [ Package['python-dev'], Package['python-pip'], - File['/var/tmp/requirements.txt'] ], - cwd => '/var/tmp', - command => "/bin/sh -c '(/usr/bin/pip install -r requirements.txt; - rm /var/tmp/requirements.txt)'" } - - exec {'buildbot-cfg-sh': - require => [ Package['supervisor'], Exec['requirements']], - path => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin', - cwd => '/data', - command => "$DOCKER_PATH/buildbot/buildbot-cfg/buildbot-cfg.sh" } -} diff --git a/testing/README.rst b/testing/README.rst new file mode 100644 index 0000000000..405adaed94 --- /dev/null +++ b/testing/README.rst @@ -0,0 +1,44 @@ +======= +testing +======= + +This directory contains testing related files. + + +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +We are running buildbot in an AWS instance to verify docker passes all tests +when commits get pushed to the master branch. + +You can check docker's buildbot instance at http://docker-ci.dotcloud.com/waterfall + + +Deployment +~~~~~~~~~~ + +:: + + # Define AWS credential environment variables + export AWS_ACCESS_KEY_ID=xxxxxxxxxxxx + export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxx + export AWS_KEYPAIR_NAME=xxxxxxxxxxxx + export AWS_SSH_PRIVKEY=xxxxxxxxxxxx + + # Checkout docker + git clone git://github.com/dotcloud/docker.git + + # Deploy docker on AWS + cd docker/testing + vagrant up --provider=aws + + +Buildbot AWS dependencies +------------------------- + +vagrant, virtualbox packages and vagrant aws plugin diff --git a/testing/Vagrantfile b/testing/Vagrantfile new file mode 100644 index 0000000000..7556b191ac --- /dev/null +++ b/testing/Vagrantfile @@ -0,0 +1,56 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +BOX_NAME = "docker-ci" +BOX_URI = "http://files.vagrantup.com/precise64.box" +AWS_AMI = "ami-d0f89fb9" +DOCKER_PATH = "/data/docker" +CFG_PATH = "#{DOCKER_PATH}/testing/buildbot" +BUILDBOT_IP = "192.168.33.41" +on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \ + Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \ + (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox) +USER = on_vbox ? "vagrant": "ubuntu" + +Vagrant::Config.run do |config| + # Setup virtual machine box. This VM configuration code is always executed. + config.vm.box = BOX_NAME + config.vm.box_url = BOX_URI + config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." + config.vm.network :hostonly, BUILDBOT_IP + + # Deploy buildbot and its dependencies if it was not done + if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? + pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " + # Deploy buildbot CI + pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ + "pip install -r #{CFG_PATH}/requirements.txt; " \ + "chown #{USER}.#{USER} /data; cd /data; " \ + "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; " + # Install docker dependencies + pkg_cmd << "apt-get install -q -y python-software-properties; " \ + "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ + "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable make; " + # Activate new kernel + pkg_cmd << "shutdown -r +1; " + config.vm.provision :shell, :inline => pkg_cmd + end +end + +# Providers were added on Vagrant >= 1.1.0 +Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| + config.vm.provider :aws do |aws, override| + aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] + aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] + aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] + override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"] + override.ssh.username = USER + aws.ami = AWS_AMI + aws.region = "us-east-1" + aws.instance_type = "m1.small" + aws.security_groups = "gateway" + end + + config.vm.provider :virtualbox do |vb| + end +end diff --git a/testing/buildbot/README.rst b/testing/buildbot/README.rst new file mode 100644 index 0000000000..7edbf32d0a --- /dev/null +++ b/testing/buildbot/README.rst @@ -0,0 +1 @@ +Buildbot configuration and setup files (except Vagrantfile located on ..) diff --git a/testing/buildbot/buildbot.conf b/testing/buildbot/buildbot.conf new file mode 100644 index 0000000000..e07b2e3c8c --- /dev/null +++ b/testing/buildbot/buildbot.conf @@ -0,0 +1,18 @@ +[program:buildmaster] +command=twistd --nodaemon --no_save -y buildbot.tac +directory=/data/buildbot/master +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-master.log +stderr_logfile=/var/log/supervisor/buildbot-master.log + +[program:buildworker] +command=twistd --nodaemon --no_save -y buildbot.tac +directory=/data/buildbot/slave +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-slave.log +stderr_logfile=/var/log/supervisor/buildbot-slave.log + +[group:buildbot] +programs=buildmaster,buildworker diff --git a/testing/buildbot/master.cfg b/testing/buildbot/master.cfg new file mode 100644 index 0000000000..48df5835f7 --- /dev/null +++ b/testing/buildbot/master.cfg @@ -0,0 +1,55 @@ +import os +from buildbot.buildslave import BuildSlave +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.changes import filter +from buildbot.config import BuilderConfig +from buildbot.process.factory import BuildFactory +from buildbot.steps.shell import ShellCommand +from buildbot.status import html +from buildbot.status.web import authz, auth + +PORT_WEB = 80 # Buildbot webserver port +PORT_GITHUB = 8011 # Buildbot github hook port +PORT_MASTER = 9989 # Port where buildbot master listen buildworkers +TEST_USER = 'buildbot' # Credential to authenticate build triggers +TEST_PWD = 'docker' # Credential to authenticate build triggers +BUILDER_NAME = 'docker' +BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers +GITHUB_DOCKER = "github.com/dotcloud/docker" +DOCKER_PATH = "/data/docker" +BUILDER_PATH = "/data/buildbot/slave/{0}/build".format(BUILDER_NAME) +DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' + + +c = BuildmasterConfig = {} + +c['title'] = "Docker" +c['titleURL'] = "waterfall" +c['buildbotURL'] = "http://0.0.0.0:{0}/".format(PORT_WEB) +c['db'] = {'db_url':"sqlite:///state.sqlite"} +c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)] +c['slavePortnum'] = PORT_MASTER + +c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] +c['schedulers'].append(SingleBranchScheduler(name="all", + change_filter=filter.ChangeFilter(branch='master'),treeStableTimer=None, + builderNames=[BUILDER_NAME])) + +# Docker test command +test_cmd = ("cd /tmp; rm -rf {0}; export GOPATH={0}; go get -d {1}; cd {2}; " + "go test").format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH) + +# Builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='Docker',logEnviron=False, + usePTY=True,command=test_cmd)) +c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], + factory=factory)] + +# Status +authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]), + forceBuild='auth') +c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)] +c['status'].append(html.WebStatus(http_port=PORT_GITHUB,allowForce=True, + change_hook_dialects={ 'github' : True })) diff --git a/buildbot/requirements.txt b/testing/buildbot/requirements.txt similarity index 100% rename from buildbot/requirements.txt rename to testing/buildbot/requirements.txt diff --git a/testing/buildbot/setup.sh b/testing/buildbot/setup.sh new file mode 100755 index 0000000000..828ac3ebe5 --- /dev/null +++ b/testing/buildbot/setup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Setup of buildbot configuration. Package installation is being done by +# Vagrantfile +# Dependencies: buildbot, buildbot-slave, supervisor + +USER=$1 +CFG_PATH=$2 +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" +SLAVE_NAME="buildworker" +SLAVE_SOCKET="localhost:9989" +BUILDBOT_PWD="pass-docker" +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +function run { su $USER -c "$1"; } + +# Exit if buildbot has already been installed +[ -d "$BUILDBOT_PATH" ] && exit 0 + +# Setup buildbot +run "mkdir -p $BUILDBOT_PATH" +cd $BUILDBOT_PATH +run "buildbot create-master master" +run "cp $CFG_PATH/master.cfg master" +run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg" +run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" + +# Allow buildbot subprocesses (docker tests) to properly run in containers, +# in particular with docker -u +run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac" + +# Setup supervisor +cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf +sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf +kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord")