From bb61678b570fabeb5d21c86fa3f7b5456111f7a2 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 1 May 2013 11:20:10 -0700 Subject: [PATCH] development; issue #374: Refactor 'make hack' making Vagrantfile and VM more useful --- Makefile | 8 ++-- hack/{README.md => README.rst} | 0 hack/Vagrantfile | 36 +++++++++++++++ hack/environment/README.rst | 20 +++++++++ hack/environment/bash_profile | 19 ++++++++ .../environment}/buildbot.conf | 0 .../environment}/master.cfg | 9 ++-- .../environment}/post-commit | 0 hack/environment/requirements.txt | 6 +++ hack/environment/setup.sh | 45 +++++++++++++++++++ 10 files changed, 134 insertions(+), 9 deletions(-) rename hack/{README.md => README.rst} (100%) create mode 100644 hack/Vagrantfile create mode 100644 hack/environment/README.rst create mode 100644 hack/environment/bash_profile rename {buildbot/buildbot-cfg => hack/environment}/buildbot.conf (100%) rename {buildbot/buildbot-cfg => hack/environment}/master.cfg (86%) rename {buildbot/buildbot-cfg => hack/environment}/post-commit (100%) create mode 100644 hack/environment/requirements.txt create mode 100755 hack/environment/setup.sh diff --git a/Makefile b/Makefile index d6cede4f52..2d9ba2c605 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,7 @@ $(DOCKER_BIN): $(DOCKER_DIR) $(DOCKER_DIR): @mkdir -p $(dir $@) - @rm -f $@ - @ln -sf $(CURDIR)/ $@ + @if [ -h $@ ]; then rm -f $@; ln -sf $(CURDIR)/ $@; fi @(cd $(DOCKER_MAIN); go get $(GO_OPTIONS)) whichrelease: @@ -75,4 +74,7 @@ fmt: @gofmt -s -l -w . hack: - cd $(CURDIR)/buildbot && vagrant up + cd $(CURDIR)/hack && vagrant up + +ssh-dev: + cd $(CURDIR)/hack && vagrant ssh diff --git a/hack/README.md b/hack/README.rst similarity index 100% rename from hack/README.md rename to hack/README.rst diff --git a/hack/Vagrantfile b/hack/Vagrantfile new file mode 100644 index 0000000000..6e614892ea --- /dev/null +++ b/hack/Vagrantfile @@ -0,0 +1,36 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +BOX_NAME = "ubuntu" +BOX_URI = "http://files.vagrantup.com/precise64.box" +PPA_KEY = "E61D797F63561DC6" +VM_IP = "192.168.33.21" +USER = "vagrant" +GOPATH = "/data/docker" +DOCKER_PATH = "#{GOPATH}/src/github.com/dotcloud/docker" +CFG_PATH = "#{DOCKER_PATH}/hack/environment" +BUILDBOT_PATH = "/data/buildbot" + +Vagrant::Config.run do |config| + # Setup virtual machine box + config.vm.box = BOX_NAME + config.vm.box_url = BOX_URI + config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." + config.vm.network :hostonly, VM_IP + # Stop if deployment has been done + config.vm.provision :shell, :inline => "[ ! -f /usr/bin/git ]" + # Touch for makefile + pkg_cmd = "touch #{DOCKER_PATH}; " + # Install docker dependencies + pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ + "apt-get install -q -y lxc bsdtar git golang make; " \ + "chown -R #{USER}.#{USER} #{GOPATH}; " \ + "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" + config.vm.provision :shell, :inline => pkg_cmd + # Deploy buildbot CI + pkg_cmd = "apt-get install -q -y python-dev python-pip supervisor; " \ + "pip install -r #{CFG_PATH}/requirements.txt; " \ + "chown #{USER}.#{USER} /data; cd /data; " \ + "#{CFG_PATH}/setup.sh #{USER} #{GOPATH} #{DOCKER_PATH} #{CFG_PATH} #{BUILDBOT_PATH}" + config.vm.provision :shell, :inline => pkg_cmd +end diff --git a/hack/environment/README.rst b/hack/environment/README.rst new file mode 100644 index 0000000000..a52b9769ef --- /dev/null +++ b/hack/environment/README.rst @@ -0,0 +1,20 @@ +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +When running 'make hack' at the docker root directory, it spawns a virtual +machine in the background running a buildbot instance and adds a git +post-commit hook that automatically run docker tests for you. + +You can check your buildbot instance at http://192.168.33.21:8010/waterfall + + +Buildbot dependencies +--------------------- + +vagrant, virtualbox packages and python package requests + diff --git a/hack/environment/bash_profile b/hack/environment/bash_profile new file mode 100644 index 0000000000..77eed79116 --- /dev/null +++ b/hack/environment/bash_profile @@ -0,0 +1,19 @@ +# ~/.bash_profile : executed by the command interpreter for login shells. + +# if running bash +if [ -n "$BASH_VERSION" ]; then + # include .bashrc if it exists + if [ -f "$HOME/.bashrc" ]; then + . "$HOME/.bashrc" + fi +fi + +# set PATH so it includes user's private bin if it exists +[ -d "$HOME/bin" ] && PATH="$HOME/bin:$PATH" + +docker=/data/docker/src/github.com/dotcloud/docker +[ -d $docker ] && cd $docker + +export GOPATH=/data/docker +export PATH=$PATH:$GOPATH/bin + diff --git a/buildbot/buildbot-cfg/buildbot.conf b/hack/environment/buildbot.conf similarity index 100% rename from buildbot/buildbot-cfg/buildbot.conf rename to hack/environment/buildbot.conf diff --git a/buildbot/buildbot-cfg/master.cfg b/hack/environment/master.cfg similarity index 86% rename from buildbot/buildbot-cfg/master.cfg rename to hack/environment/master.cfg index c786e418ed..fad023b602 100644 --- a/buildbot/buildbot-cfg/master.cfg +++ b/hack/environment/master.cfg @@ -13,8 +13,8 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers -DOCKER_PATH = '/data/docker' - +GOPATH = '/data/docker' +DOCKER_PATH = '{0}/src/github.com/dotcloud/docker'.format(GOPATH) c = BuildmasterConfig = {} @@ -28,10 +28,7 @@ c['slavePortnum'] = PORT_MASTER c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] # Docker test command -test_cmd = """( - cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp; - cd docker-tmp; make test; exit_status=$?; - cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH) +test_cmd = "GOPATH={0} make -C {1} test".format(GOPATH,DOCKER_PATH) # Builder factory = BuildFactory() diff --git a/buildbot/buildbot-cfg/post-commit b/hack/environment/post-commit similarity index 100% rename from buildbot/buildbot-cfg/post-commit rename to hack/environment/post-commit diff --git a/hack/environment/requirements.txt b/hack/environment/requirements.txt new file mode 100644 index 0000000000..0e451b017d --- /dev/null +++ b/hack/environment/requirements.txt @@ -0,0 +1,6 @@ +sqlalchemy<=0.7.9 +sqlalchemy-migrate>=0.7.2 +buildbot==0.8.7p1 +buildbot_slave==0.8.7p1 +nose==1.2.1 +requests==1.1.0 diff --git a/hack/environment/setup.sh b/hack/environment/setup.sh new file mode 100755 index 0000000000..7aa06a5184 --- /dev/null +++ b/hack/environment/setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Setup of buildbot configuration. Package installation is being done by +# Vagrantfile +# Dependencies: buildbot, buildbot-slave, supervisor + +USER=$1 +GOPATH=$2 +DOCKER_PATH=$3 +CFG_PATH=$4 +BUILDBOT_PATH=$5 +SLAVE_NAME="buildworker" +SLAVE_SOCKET="localhost:9989" +BUILDBOT_PWD="pass-docker" +IP=$(sed -nE 's/VM_IP = "(.+)"/\1/p' ${DOCKER_PATH}/hack/Vagrantfile) +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +function run { su $USER -c "$1"; } + +# Exit if buildbot has already been installed +[ -d "$BUILDBOT_PATH" ] && exit 0 + +# Setup buildbot +run "mkdir -p $BUILDBOT_PATH" +cd $BUILDBOT_PATH +run "buildbot create-master master" +run "cp $CFG_PATH/master.cfg master" +run "sed -i 's/localhost/$IP/' master/master.cfg" +run "sed -i -E 's#(GOPATH = ).+#\1\"$GOPATH\"#' master/master.cfg" +run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg" +run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" + +# Allow buildbot subprocesses (docker tests) to properly run in containers, +# in particular with docker -u +run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac" + +# Setup supervisor +cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf +sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf +kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord") + +# Add git hook +cp $CFG_PATH/post-commit $DOCKER_PATH/.git/hooks +sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit +