From bb61678b570fabeb5d21c86fa3f7b5456111f7a2 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 1 May 2013 11:20:10 -0700 Subject: [PATCH 1/4] development; issue #374: Refactor 'make hack' making Vagrantfile and VM more useful --- Makefile | 8 ++-- hack/{README.md => README.rst} | 0 hack/Vagrantfile | 36 +++++++++++++++ hack/environment/README.rst | 20 +++++++++ hack/environment/bash_profile | 19 ++++++++ .../environment}/buildbot.conf | 0 .../environment}/master.cfg | 9 ++-- .../environment}/post-commit | 0 hack/environment/requirements.txt | 6 +++ hack/environment/setup.sh | 45 +++++++++++++++++++ 10 files changed, 134 insertions(+), 9 deletions(-) rename hack/{README.md => README.rst} (100%) create mode 100644 hack/Vagrantfile create mode 100644 hack/environment/README.rst create mode 100644 hack/environment/bash_profile rename {buildbot/buildbot-cfg => hack/environment}/buildbot.conf (100%) rename {buildbot/buildbot-cfg => hack/environment}/master.cfg (86%) rename {buildbot/buildbot-cfg => hack/environment}/post-commit (100%) create mode 100644 hack/environment/requirements.txt create mode 100755 hack/environment/setup.sh diff --git a/Makefile b/Makefile index d6cede4f52..2d9ba2c605 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,7 @@ $(DOCKER_BIN): $(DOCKER_DIR) $(DOCKER_DIR): @mkdir -p $(dir $@) - @rm -f $@ - @ln -sf $(CURDIR)/ $@ + @if [ -h $@ ]; then rm -f $@; ln -sf $(CURDIR)/ $@; fi @(cd $(DOCKER_MAIN); go get $(GO_OPTIONS)) whichrelease: @@ -75,4 +74,7 @@ fmt: @gofmt -s -l -w . hack: - cd $(CURDIR)/buildbot && vagrant up + cd $(CURDIR)/hack && vagrant up + +ssh-dev: + cd $(CURDIR)/hack && vagrant ssh diff --git a/hack/README.md b/hack/README.rst similarity index 100% rename from hack/README.md rename to hack/README.rst diff --git a/hack/Vagrantfile b/hack/Vagrantfile new file mode 100644 index 0000000000..6e614892ea --- /dev/null +++ b/hack/Vagrantfile @@ -0,0 +1,36 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +BOX_NAME = "ubuntu" +BOX_URI = "http://files.vagrantup.com/precise64.box" +PPA_KEY = "E61D797F63561DC6" +VM_IP = "192.168.33.21" +USER = "vagrant" +GOPATH = "/data/docker" +DOCKER_PATH = "#{GOPATH}/src/github.com/dotcloud/docker" +CFG_PATH = "#{DOCKER_PATH}/hack/environment" +BUILDBOT_PATH = "/data/buildbot" + +Vagrant::Config.run do |config| + # Setup virtual machine box + config.vm.box = BOX_NAME + config.vm.box_url = BOX_URI + config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." + config.vm.network :hostonly, VM_IP + # Stop if deployment has been done + config.vm.provision :shell, :inline => "[ ! -f /usr/bin/git ]" + # Touch for makefile + pkg_cmd = "touch #{DOCKER_PATH}; " + # Install docker dependencies + pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ + "apt-get install -q -y lxc bsdtar git golang make; " \ + "chown -R #{USER}.#{USER} #{GOPATH}; " \ + "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" + config.vm.provision :shell, :inline => pkg_cmd + # Deploy buildbot CI + pkg_cmd = "apt-get install -q -y python-dev python-pip supervisor; " \ + "pip install -r #{CFG_PATH}/requirements.txt; " \ + "chown #{USER}.#{USER} /data; cd /data; " \ + "#{CFG_PATH}/setup.sh #{USER} #{GOPATH} #{DOCKER_PATH} #{CFG_PATH} #{BUILDBOT_PATH}" + config.vm.provision :shell, :inline => pkg_cmd +end diff --git a/hack/environment/README.rst b/hack/environment/README.rst new file mode 100644 index 0000000000..a52b9769ef --- /dev/null +++ b/hack/environment/README.rst @@ -0,0 +1,20 @@ +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +When running 'make hack' at the docker root directory, it spawns a virtual +machine in the background running a buildbot instance and adds a git +post-commit hook that automatically run docker tests for you. + +You can check your buildbot instance at http://192.168.33.21:8010/waterfall + + +Buildbot dependencies +--------------------- + +vagrant, virtualbox packages and python package requests + diff --git a/hack/environment/bash_profile b/hack/environment/bash_profile new file mode 100644 index 0000000000..77eed79116 --- /dev/null +++ b/hack/environment/bash_profile @@ -0,0 +1,19 @@ +# ~/.bash_profile : executed by the command interpreter for login shells. + +# if running bash +if [ -n "$BASH_VERSION" ]; then + # include .bashrc if it exists + if [ -f "$HOME/.bashrc" ]; then + . "$HOME/.bashrc" + fi +fi + +# set PATH so it includes user's private bin if it exists +[ -d "$HOME/bin" ] && PATH="$HOME/bin:$PATH" + +docker=/data/docker/src/github.com/dotcloud/docker +[ -d $docker ] && cd $docker + +export GOPATH=/data/docker +export PATH=$PATH:$GOPATH/bin + diff --git a/buildbot/buildbot-cfg/buildbot.conf b/hack/environment/buildbot.conf similarity index 100% rename from buildbot/buildbot-cfg/buildbot.conf rename to hack/environment/buildbot.conf diff --git a/buildbot/buildbot-cfg/master.cfg b/hack/environment/master.cfg similarity index 86% rename from buildbot/buildbot-cfg/master.cfg rename to hack/environment/master.cfg index c786e418ed..fad023b602 100644 --- a/buildbot/buildbot-cfg/master.cfg +++ b/hack/environment/master.cfg @@ -13,8 +13,8 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers -DOCKER_PATH = '/data/docker' - +GOPATH = '/data/docker' +DOCKER_PATH = '{0}/src/github.com/dotcloud/docker'.format(GOPATH) c = BuildmasterConfig = {} @@ -28,10 +28,7 @@ c['slavePortnum'] = PORT_MASTER c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] # Docker test command -test_cmd = """( - cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp; - cd docker-tmp; make test; exit_status=$?; - cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH) +test_cmd = "GOPATH={0} make -C {1} test".format(GOPATH,DOCKER_PATH) # Builder factory = BuildFactory() diff --git a/buildbot/buildbot-cfg/post-commit b/hack/environment/post-commit similarity index 100% rename from buildbot/buildbot-cfg/post-commit rename to hack/environment/post-commit diff --git a/hack/environment/requirements.txt b/hack/environment/requirements.txt new file mode 100644 index 0000000000..0e451b017d --- /dev/null +++ b/hack/environment/requirements.txt @@ -0,0 +1,6 @@ +sqlalchemy<=0.7.9 +sqlalchemy-migrate>=0.7.2 +buildbot==0.8.7p1 +buildbot_slave==0.8.7p1 +nose==1.2.1 +requests==1.1.0 diff --git a/hack/environment/setup.sh b/hack/environment/setup.sh new file mode 100755 index 0000000000..7aa06a5184 --- /dev/null +++ b/hack/environment/setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Setup of buildbot configuration. Package installation is being done by +# Vagrantfile +# Dependencies: buildbot, buildbot-slave, supervisor + +USER=$1 +GOPATH=$2 +DOCKER_PATH=$3 +CFG_PATH=$4 +BUILDBOT_PATH=$5 +SLAVE_NAME="buildworker" +SLAVE_SOCKET="localhost:9989" +BUILDBOT_PWD="pass-docker" +IP=$(sed -nE 's/VM_IP = "(.+)"/\1/p' ${DOCKER_PATH}/hack/Vagrantfile) +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +function run { su $USER -c "$1"; } + +# Exit if buildbot has already been installed +[ -d "$BUILDBOT_PATH" ] && exit 0 + +# Setup buildbot +run "mkdir -p $BUILDBOT_PATH" +cd $BUILDBOT_PATH +run "buildbot create-master master" +run "cp $CFG_PATH/master.cfg master" +run "sed -i 's/localhost/$IP/' master/master.cfg" +run "sed -i -E 's#(GOPATH = ).+#\1\"$GOPATH\"#' master/master.cfg" +run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg" +run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" + +# Allow buildbot subprocesses (docker tests) to properly run in containers, +# in particular with docker -u +run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac" + +# Setup supervisor +cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf +sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf +kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord") + +# Add git hook +cp $CFG_PATH/post-commit $DOCKER_PATH/.git/hooks +sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit + From eeb03164cf1ea1b6a2abae2536cb1360c9ac8251 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 1 May 2013 15:25:58 -0700 Subject: [PATCH 2/4] development; issue #374: Upgrade development VM box to Ubuntu-13.04 with kernel 3.8 --- hack/Vagrantfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hack/Vagrantfile b/hack/Vagrantfile index 6e614892ea..250731ef47 100644 --- a/hack/Vagrantfile +++ b/hack/Vagrantfile @@ -1,9 +1,8 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -BOX_NAME = "ubuntu" -BOX_URI = "http://files.vagrantup.com/precise64.box" -PPA_KEY = "E61D797F63561DC6" +BOX_NAME = "ubuntu-dev" +BOX_URI = "http://cloud-images.ubuntu.com/raring/current/raring-server-cloudimg-vagrant-amd64-disk1.box" VM_IP = "192.168.33.21" USER = "vagrant" GOPATH = "/data/docker" @@ -23,7 +22,7 @@ Vagrant::Config.run do |config| pkg_cmd = "touch #{DOCKER_PATH}; " # Install docker dependencies pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ - "apt-get install -q -y lxc bsdtar git golang make; " \ + "apt-get install -q -y lxc bsdtar git golang make linux-image-extra-3.8.0-19-generic; " \ "chown -R #{USER}.#{USER} #{GOPATH}; " \ "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" config.vm.provision :shell, :inline => pkg_cmd From c20e46587d3fb09257e6ce7d085467da403210ee Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:43:02 -0700 Subject: [PATCH 3/4] Update commandline Commit doc --- docs/sources/commandline/command/commit.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/sources/commandline/command/commit.rst b/docs/sources/commandline/command/commit.rst index 2af05ff093..c73f8d1898 100644 --- a/docs/sources/commandline/command/commit.rst +++ b/docs/sources/commandline/command/commit.rst @@ -9,3 +9,19 @@ Create a new image from a container's changes -m="": Commit message + -author="": Author (eg. "John Hannibal Smith " + -run="": Config automatically applied when the image is run. "+`(ex: {"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}') + +Full -run example:: + + {"Hostname": "", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "PortSpecs": ["22", "80", "443"], + "Tty": true, + "OpenStdin": true, + "StdinOnce": true, + "Env": ["FOO=BAR", "FOO2=BAR2"], + "Cmd": ["cat", "-e", "/etc/resolv.conf"], + "Dns": ["8.8.8.8", "8.8.4.4"]} From d172da58ceddd93c4a93132eae652e7996e1a2d2 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 1 May 2013 15:59:54 -0700 Subject: [PATCH 4/4] development; issue #374: Update VM documentation --- hack/README.rst | 26 ++++++++++++++++++++++++++ hack/environment/README.rst | 21 +-------------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/hack/README.rst b/hack/README.rst index 06cdd50854..4607b6a4a9 100644 --- a/hack/README.rst +++ b/hack/README.rst @@ -1 +1,27 @@ This directory contains material helpful for hacking on docker. + +make hack +========= + +Set up an Ubuntu 13.04 virtual machine for developers including kernel 3.8 +and buildbot. The environment is setup in a way that can be used through +the usual go workflow and/or the root Makefile. You can either edit on +your host, or inside the VM (using make ssh-dev) and run and test docker +inside the VM. + +dependencies: vagrant, virtualbox packages and python package requests + + +Buildbot +~~~~~~~~ + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +When running 'make hack' at the docker root directory, it spawns a virtual +machine in the background running a buildbot instance and adds a git +post-commit hook that automatically run docker tests for you. + +You can check your buildbot instance at http://192.168.33.21:8010/waterfall diff --git a/hack/environment/README.rst b/hack/environment/README.rst index a52b9769ef..da5c885e66 100644 --- a/hack/environment/README.rst +++ b/hack/environment/README.rst @@ -1,20 +1 @@ -Buildbot -======== - -Buildbot is a continuous integration system designed to automate the -build/test cycle. By automatically rebuilding and testing the tree each time -something has changed, build problems are pinpointed quickly, before other -developers are inconvenienced by the failure. - -When running 'make hack' at the docker root directory, it spawns a virtual -machine in the background running a buildbot instance and adds a git -post-commit hook that automatically run docker tests for you. - -You can check your buildbot instance at http://192.168.33.21:8010/waterfall - - -Buildbot dependencies ---------------------- - -vagrant, virtualbox packages and python package requests - +Files used to setup the developer virtual machine