Merge github.com:dotcloud/docker into registry-api-doc

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Changelog
 
-## 0.2.1 (2012-05-01)
+## 0.2.2 (2013-05-03)
+ + Support for data volumes ('docker run -v=PATH')
+ + Share data volumes between containers ('docker run -volumes-from')
+ + Improved documentation
+ * Upgrade to Go 1.0.3
+ * Various upgrades to the dev environment for contributors
+
+## 0.2.1 (2013-05-01)
  + 'docker commit -run' bundles a layer with default runtime options: command, ports etc. 
  * Improve install process on Vagrant
  + New Dockerfile operation: "maintainer"
@@ -10,7 +17,7 @@
  + 'docker -d -r': restart crashed containers at daemon startup
  * Runtime: improve test coverage
 
-## 0.2.0 (2012-04-23)
+## 0.2.0 (2013-04-23)
  - Runtime: ghost containers can be killed and waited for
  * Documentation: update install intructions
  - Packaging: fix Vagrantfile

Makefile

@@ -38,13 +38,15 @@ $(DOCKER_BIN): $(DOCKER_DIR)
 
 $(DOCKER_DIR):
 	@mkdir -p $(dir $@)
-	@if [ -h $@ ]; then rm -f $@; ln -sf $(CURDIR)/ $@; fi
+	@if [ -h $@ ]; then rm -f $@; fi; ln -sf $(CURDIR)/ $@
 	@(cd $(DOCKER_MAIN); go get $(GO_OPTIONS))
 
 whichrelease:
 	echo $(RELEASE_VERSION)
 
 release: $(BINRELEASE)
+	s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz
+
 srcrelease: $(SRCRELEASE)
 deps: $(DOCKER_DIR)
 

Vagrantfile

@@ -1,8 +1,8 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-BOX_NAME = "ubuntu"
+BOX_NAME = ENV['BOX_NAME'] || "ubuntu"
-BOX_URI = "http://files.vagrantup.com/precise64.box"
+BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box"
 PPA_KEY = "E61D797F63561DC6"
 
 Vagrant::Config.run do |config|
@@ -11,7 +11,7 @@ Vagrant::Config.run do |config|
   config.vm.box_url = BOX_URI
   # Add docker PPA key to the local repository and install docker
   pkg_cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys #{PPA_KEY}; "
-  pkg_cmd << "echo 'deb http://ppa.launchpad.net/dotcloud/lxc-docker/ubuntu precise main' >>/etc/apt/sources.list; "
+  pkg_cmd << "echo 'deb http://ppa.launchpad.net/dotcloud/lxc-docker/ubuntu precise main' >/etc/apt/sources.list.d/lxc-docker.list; "
   pkg_cmd << "apt-get update -qq; apt-get install -q -y lxc-docker"
   if ARGV.include?("--provider=aws".downcase)
     # Add AUFS dependency to amazon's VM

commands.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -18,7 +19,7 @@ import (
 	"unicode"
 )
 
-const VERSION = "0.2.1"
+const VERSION = "0.2.2"
 
 var (
 	GIT_COMMIT string
@@ -400,7 +401,8 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str
 }
 
 func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error {
-	cmd := rcli.Subcmd(stdout, "rm", "CONTAINER [CONTAINER...]", "Remove a container")
+	cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove a container")
+	v := cmd.Bool("v", false, "Remove the volumes associated to the container")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -408,15 +410,40 @@ func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string)
 		cmd.Usage()
 		return nil
 	}
+	volumes := make(map[string]struct{})
 	for _, name := range cmd.Args() {
 		container := srv.runtime.Get(name)
 		if container == nil {
 			return fmt.Errorf("No such container: %s", name)
 		}
+		// Store all the deleted containers volumes
+		for _, volumeId := range container.Volumes {
+			volumes[volumeId] = struct{}{}
+		}
 		if err := srv.runtime.Destroy(container); err != nil {
 			fmt.Fprintln(stdout, "Error destroying container "+name+": "+err.Error())
 		}
 	}
+	if *v {
+		// Retrieve all volumes from all remaining containers
+		usedVolumes := make(map[string]*Container)
+		for _, container := range srv.runtime.List() {
+			for _, containerVolumeId := range container.Volumes {
+				usedVolumes[containerVolumeId] = container
+			}
+		}
+
+		for volumeId := range volumes {
+			// If the requested volu
+			if c, exists := usedVolumes[volumeId]; exists {
+				fmt.Fprintf(stdout, "The volume %s is used by the container %s. Impossible to remove it. Skipping.\n", volumeId, c.Id)
+				continue
+			}
+			if err := srv.runtime.volumes.Delete(volumeId); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 
@@ -913,6 +940,25 @@ func (opts AttachOpts) Get(val string) bool {
 	return false
 }
 
+// PathOpts stores a unique set of absolute paths
+type PathOpts map[string]struct{}
+
+func NewPathOpts() PathOpts {
+	return make(PathOpts)
+}
+
+func (opts PathOpts) String() string {
+	return fmt.Sprintf("%v", map[string]struct{}(opts))
+}
+
+func (opts PathOpts) Set(val string) error {
+	if !filepath.IsAbs(val) {
+		return fmt.Errorf("%s is not an absolute path", val)
+	}
+	opts[filepath.Clean(val)] = struct{}{}
+	return nil
+}
+
 func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) error {
 	cmd := rcli.Subcmd(stdout, "tag", "[OPTIONS] IMAGE REPOSITORY [TAG]", "Tag an image into a repository")
 	force := cmd.Bool("f", false, "Force")

container.go

@@ -48,6 +48,7 @@ type Container struct {
 	runtime *Runtime
 
 	waitLock chan struct{}
+	Volumes  map[string]string
 }
 
 type Config struct {
@@ -66,6 +67,8 @@ type Config struct {
 	Cmd          []string
 	Dns          []string
 	Image        string // Name of the image as it was passed by the operator (eg. could be symbolic)
+	Volumes      map[string]struct{}
+	VolumesFrom  string
 }
 
 func ParseRun(args []string, stdout io.Writer, capabilities *Capabilities) (*Config, error) {
@@ -97,6 +100,11 @@ func ParseRun(args []string, stdout io.Writer, capabilities *Capabilities) (*Con
 	var flDns ListOpts
 	cmd.Var(&flDns, "dns", "Set custom dns servers")
 
+	flVolumes := NewPathOpts()
+	cmd.Var(flVolumes, "v", "Attach a data volume")
+
+	flVolumesFrom := cmd.String("volumes-from", "", "Mount volumes from the specified container")
+
 	if err := cmd.Parse(args); err != nil {
 		return nil, err
 	}
@@ -136,6 +144,8 @@ func ParseRun(args []string, stdout io.Writer, capabilities *Capabilities) (*Con
 		Cmd:          runCmd,
 		Dns:          flDns,
 		Image:        image,
+		Volumes:      flVolumes,
+		VolumesFrom:  *flVolumesFrom,
 	}
 
 	if *flMemory > 0 && !capabilities.SwapLimit {
@@ -394,10 +404,40 @@ func (container *Container) Start() error {
 		log.Printf("WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n")
 		container.Config.MemorySwap = -1
 	}
+	container.Volumes = make(map[string]string)
+
+	// Create the requested volumes volumes
+	for volPath := range container.Config.Volumes {
+		if c, err := container.runtime.volumes.Create(nil, container, "", "", nil); err != nil {
+			return err
+		} else {
+			if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil {
+				return nil
+			}
+			container.Volumes[volPath] = c.Id
+		}
+	}
+
+	if container.Config.VolumesFrom != "" {
+		c := container.runtime.Get(container.Config.VolumesFrom)
+		if c == nil {
+			return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.Id)
+		}
+		for volPath, id := range c.Volumes {
+			if _, exists := container.Volumes[volPath]; exists {
+				return fmt.Errorf("The requested volume %s overlap one of the volume of the container %s", volPath, c.Id)
+			}
+			if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil {
+				return nil
+			}
+			container.Volumes[volPath] = id
+		}
+	}
 
 	if err := container.generateLXCConfig(); err != nil {
 		return err
 	}
+
 	params := []string{
 		"-n", container.Id,
 		"-f", container.lxcConfigPath(),
@@ -456,6 +496,7 @@ func (container *Container) Start() error {
 
 	// Init the lock
 	container.waitLock = make(chan struct{})
+
 	container.ToDisk()
 	go container.monitor()
 	return nil
@@ -787,6 +828,22 @@ func (container *Container) RootfsPath() string {
 	return path.Join(container.root, "rootfs")
 }
 
+func (container *Container) GetVolumes() (map[string]string, error) {
+	ret := make(map[string]string)
+	for volPath, id := range container.Volumes {
+		volume, err := container.runtime.volumes.Get(id)
+		if err != nil {
+			return nil, err
+		}
+		root, err := volume.root()
+		if err != nil {
+			return nil, err
+		}
+		ret[volPath] = path.Join(root, "layer")
+	}
+	return ret, nil
+}
+
 func (container *Container) rwPath() string {
 	return path.Join(container.root, "rw")
 }

docs/sources/commandline/command/run.rst

@@ -17,3 +17,6 @@
       -p=[]: Map a network port to the container
       -t=false: Allocate a pseudo-tty
       -u="": Username or UID
+      -d=[]: Set custom dns servers for the container
+      -v=[]: Creates a new volumes and mount it at the specified path.
+      -volumes-from="": Mount all volumes from the given container.

docs/sources/concepts/buildingblocks.rst

@@ -12,7 +12,7 @@ Images
 ------
 An original container image. These are stored on disk and are comparable with what you normally expect from a stopped virtual machine image. Images are stored (and retrieved from) repository
 
-Images are stored on your local file system under /var/lib/docker/images
+Images are stored on your local file system under /var/lib/docker/graph
 
 
 .. _containers:

docs/sources/examples/couchdb_data_volumes.rst

@@ -0,0 +1,53 @@
+:title: Sharing data between 2 couchdb databases
+:description: Sharing data between 2 couchdb databases
+:keywords: docker, example, package installation, networking, couchdb, data volumes
+
+.. _running_redis_service:
+
+Create a redis service
+======================
+
+.. include:: example_header.inc
+
+Here's an example of using data volumes to share the same data between 2 couchdb containers.
+This could be used for hot upgrades, testing different versions of couchdb on the same data, etc.
+
+Create first database
+---------------------
+
+Note that we're marking /var/lib/couchdb as a data volume.
+
+.. code-block:: bash
+
+    COUCH1=$(docker run -d -v /var/lib/couchdb shykes/couchdb:2013-05-03)
+
+Add data to the first database
+------------------------------
+
+We're assuming your docker host is reachable at `localhost`. If not, replace `localhost` with the public IP of your docker host.
+
+.. code-block:: bash
+
+    HOST=localhost
+    URL="http://$HOST:$(docker port $COUCH1 5984)/_utils/"
+    echo "Navigate to $URL in your browser, and use the couch interface to add data"
+
+Create second database
+----------------------
+
+This time, we're requesting shared access to $COUCH1's volumes.
+
+.. code-block:: bash
+
+    COUCH2=$(docker run -d -volumes-from $COUCH1) shykes/couchdb:2013-05-03)
+
+Browse data on the second database
+----------------------------------
+
+.. code-block:: bash
+
+    HOST=localhost
+    URL="http://$HOST:$(docker port $COUCH2 5984)/_utils/"
+    echo "Navigate to $URL in your browser. You should see the same data as in the first database!"
+
+Congratulations, you are running 2 Couchdb containers, completely isolated from each other *except* for their data.

docs/sources/examples/index.rst

@@ -18,3 +18,4 @@ Contents:
    python_web_app
    running_redis_service
    running_ssh_service
+   couchdb_data_volumes

hack/dockerbuilder/Dockerfile

@@ -4,16 +4,20 @@ maintainer	Solomon Hykes <solomon@dotcloud.com>
 from	ubuntu:12.10
 run	apt-get update
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q s3cmd
+run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl
 # Packages required to checkout and build docker
-run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang
+run	curl -s -o /go.tar.gz https://go.googlecode.com/files/go1.0.3.linux-amd64.tar.gz
+run	tar -C /usr/local -xzf /go.tar.gz
+run	echo "export PATH=$PATH:/usr/local/go/bin" > /.bashrc
+run	echo "export PATH=$PATH:/usr/local/go/bin" > /.bash_profile
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q git
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q build-essential
 # Packages required to build an ubuntu package
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q debhelper
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q autotools-dev
+copy	fake_initctl	/usr/local/bin/initctl
+run	apt-get install -y -q devscripts
 add	.	/src
 run	cp /src/dockerbuilder /usr/local/bin/ && chmod +x /usr/local/bin/dockerbuilder
-run	cp /src/fake_initctl /usr/local/bin/initctl && chmod +x /usr/local/bin/initctl
 run	cp /src/s3cfg /.s3cfg
-run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q devscripts
 cmd	["dockerbuilder"]

hack/dockerbuilder/dockerbuilder

@@ -2,6 +2,8 @@
 set -x
 set -e
 
+export PATH=$PATH:/usr/local/go/bin
+
 PACKAGE=github.com/dotcloud/docker
 
 if [ $# -gt 1 ]; then
@@ -13,12 +15,10 @@ export REVISION=$1
 
 if [ -z "$AWS_ID" ]; then
 	echo "Warning: environment variable AWS_ID is not set. Won't upload to S3."
-	NO_S3=1
 fi
 
 if [ -z "$AWS_KEY" ]; then
 	echo "Warning: environment variable AWS_KEY is not set. Won't upload to S3."
-	NO_S3=1
 fi
 
 if [ -z "$GPG_KEY" ]; then
@@ -26,28 +26,15 @@ if [ -z "$GPG_KEY" ]; then
 	NO_UBUNTU=1
 fi
 
-if [ -z "$REVISION" ]; then
+rm -fr docker-release
-	rm -fr docker-master
+git clone https://github.com/dotcloud/docker docker-release
-	git clone https://github.com/dotcloud/docker docker-master
+cd docker-release
-	cd docker-master
-else 
-	rm -fr docker-$REVISION
-	git init docker-$REVISION
-	cd docker-$REVISION
-	git fetch -t https://github.com/dotcloud/docker $REVISION:$REVISION
-	git reset --hard FETCH_HEAD
-fi
-
 if [ -z "$REVISION" ]; then
 	make release
 else
 	make release RELEASE_VERSION=$REVISION
 fi
 
-if [ -z "$NO_S3" ]; then
-	s3cmd -P put docker-$REVISION.tgz s3://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-$REVISION.tgz
-fi
-
 if [ -z "$NO_UBUNTU" ]; then
 	(cd packaging/ubuntu && make ubuntu)
 fi

lxc_template.go

@@ -79,7 +79,11 @@ lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0
 
 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container
 lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0
-
+{{if .Volumes}}
+{{range $virtualPath, $realPath := .GetVolumes}}
+lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,rw 0 0
+{{end}}
+{{end}}
 
 # drop linux capabilities (apply mainly to the user root in the container)
 lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config

packaging/ubuntu/changelog

@@ -1,3 +1,12 @@
+lxc-docker (0.2.2-1) precise; urgency=low
+ - Support for data volumes ('docker run -v=PATH')
+ - Share data volumes between containers ('docker run -volumes-from')
+ - Improved documentation
+ - Upgrade to Go 1.0.3
+ - Various upgrades to the dev environment for contributors
+
+ -- dotCloud <ops@dotcloud.com>  Fri, 3 May 2013 00:00:00 -0700
+
 
 lxc-docker (0.2.1-1) precise; urgency=low
 

runtime.go

@@ -32,6 +32,7 @@ type Runtime struct {
 	capabilities   *Capabilities
 	kernelVersion  *KernelVersionInfo
 	autoRestart    bool
+	volumes        *Graph
 }
 
 var sysInitPath string
@@ -79,10 +80,10 @@ func (runtime *Runtime) containerRoot(id string) string {
 }
 
 func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) {
-	if userConf.Hostname != "" {
+	if userConf.Hostname == "" {
 		userConf.Hostname = imageConf.Hostname
 	}
-	if userConf.User != "" {
+	if userConf.User == "" {
 		userConf.User = imageConf.User
 	}
 	if userConf.Memory == 0 {
@@ -126,7 +127,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) {
 		runtime.mergeConfig(config, img.Config)
 	}
 
-	if config.Cmd == nil {
+	if config.Cmd == nil || len(config.Cmd) == 0 {
 		return nil, fmt.Errorf("No command specified")
 	}
 
@@ -405,6 +406,10 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
 	if err != nil {
 		return nil, err
 	}
+	volumes, err := NewGraph(path.Join(root, "volumes"))
+	if err != nil {
+		return nil, err
+	}
 	repositories, err := NewTagStore(path.Join(root, "repositories"), g)
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
@@ -432,6 +437,7 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
 		idIndex:        NewTruncIndex(),
 		capabilities:   &Capabilities{},
 		autoRestart:    autoRestart,
+		volumes:        volumes,
 	}
 
 	if err := runtime.restore(); err != nil {

utils.go

@@ -445,7 +445,7 @@ func FindCgroupMountpoint(cgroupType string) (string, error) {
 	// cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
 	for _, line := range strings.Split(string(output), "\n") {
 		parts := strings.Split(line, " ")
-		if parts[2] == "cgroup" {
+		if len(parts) == 6 && parts[2] == "cgroup" {
 			for _, opt := range strings.Split(parts[3], ",") {
 				if opt == cgroupType {
 					return parts[1], nil