From a3a946703ba93e7d937680897ed89eb99c52acef Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 14:28:11 -0700 Subject: [PATCH 01/19] Set the memory soft limit to the same value than the hard limit --- lxc_template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lxc_template.go b/lxc_template.go index 931095c99d..47f2058dc4 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -87,6 +87,7 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se # limits {{if .Config.Ram}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} +lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} {{end}} ` From 06d420d2ead65cb5af760d7946d8844f568f4d65 Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 22:08:22 +0000 Subject: [PATCH 02/19] Fix CmdInspect and CmdImages help/parsing. Fixes #38 --- server/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index bc642cdbb0..9efdaa97da 100644 --- a/server/server.go +++ b/server/server.go @@ -60,6 +60,7 @@ func (srv *Server) Help() string { {"tar", "Stream the contents of a container as a tar archive"}, {"web", "Generate a web UI"}, {"images", "List images"}, + {"inspect", "Return low-level information on a container"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) } @@ -282,7 +283,6 @@ func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -463,7 +463,9 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") - cmd.Parse(args) + if err := cmd.Parse(args); err != nil { + return nil + } if cmd.NArg() > 1 { cmd.Usage() return nil @@ -893,7 +895,7 @@ func New() (*Server, error) { return nil, err } srv := &Server{ - images: images, + images: images, containers: containers, } return srv, nil From a7cb3cb86a39f950a517b61e3a674e0cd080804f Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 22:29:13 +0000 Subject: [PATCH 03/19] Complete fix for #38 -- fix double-usage on help --- server/server.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server/server.go b/server/server.go index 9efdaa97da..3a66d7e032 100644 --- a/server/server.go +++ b/server/server.go @@ -71,7 +71,6 @@ func (srv *Server) Help() string { func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] NAME", "Block until a container stops, then print its exit code.") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -100,7 +99,6 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] NAME", "Stop a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -123,7 +121,6 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] NAME", "Restart a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -146,7 +143,6 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] NAME", "Start a stopped container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -169,7 +165,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "umount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -192,7 +187,6 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -215,7 +209,6 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -237,7 +230,6 @@ func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -259,7 +251,6 @@ func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -315,7 +306,6 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "port", "[OPTIONS] CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() != 2 { @@ -341,7 +331,6 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { From 52b811f50a0906904d688e5fae8f32e1ada90f1a Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 23:11:46 +0000 Subject: [PATCH 04/19] Add all commands to the output of 'dotcloud help'. Fixes #39 --- server/server.go | 54 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/server/server.go b/server/server.go index 3a66d7e032..37404f2c21 100644 --- a/server/server.go +++ b/server/server.go @@ -37,30 +37,40 @@ func (srv *Server) Name() string { return "docker" } +// FIXME: Stop violating DRY by repeating usage here and in Subcmd declarations func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]interface{}{ - {"run", "Run a command in a container"}, - {"ps", "Display a list of containers"}, - {"pull", "Download a tarball and create a container from it"}, - {"put", "Upload a tarball and create a container from it"}, - {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, - {"rm", "Remove containers"}, - {"kill", "Kill a running container"}, - {"wait", "Wait for the state of a container to change"}, - {"stop", "Stop a running container"}, - {"start", "Start a stopped container"}, - {"restart", "Restart a running container"}, - {"logs", "Fetch the logs of a container"}, + {"attach", "Attach to a running container"}, + {"cat", "Write the contents of a container's file to standard output"}, + {"commit", "Create a new image from a container's changes"}, + {"cp", "Create a copy of IMAGE and call it NAME"}, + {"debug", "(debug only) (No documentation available)"}, {"diff", "Inspect changes on a container's filesystem"}, - {"commit", "Save the state of a container"}, - {"attach", "Attach to the standard inputs and outputs of a running container"}, - {"wait", "Block until a container exits, then print its exit code"}, - {"info", "Display system-wide information"}, - {"tar", "Stream the contents of a container as a tar archive"}, - {"web", "Generate a web UI"}, {"images", "List images"}, + {"info", "Display system-wide information"}, {"inspect", "Return low-level information on a container"}, + {"kill", "Kill a running container"}, + {"layers", "(debug only) List filesystem layers"}, + {"logs", "Fetch the logs of a container"}, + {"ls", "List the contents of a container's directory"}, + {"mirror", "(debug only) (No documentation available)"}, + {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, + {"ps", "List containers"}, + {"pull", "Download a new image from a remote location"}, + {"put", "Import a new image from a local archive"}, + {"reset", "Reset changes to a container's filesystem"}, + {"restart", "Restart a running container"}, + {"rm", "Remove a container"}, + {"rmimage", "Remove an image"}, + {"run", "Run a command in a new container"}, + {"start", "Start a stopped container"}, + {"stop", "Stop a running container"}, + {"tar", "Stream the contents of a container as a tar archive"}, + {"umount", "(debug only) Mount a container's filesystem"}, + {"wait", "Block until a container stops, then print its exit code"}, + {"web", "A web UI for docker"}, + {"write", "Write the contents of standard input to a container's file"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) } @@ -89,6 +99,14 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "info", "", "Display system-wide information.") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() > 1 { + cmd.Usage() + return nil + } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, From 75d04a5a7561cabba0aaf2875ef4479381096f91 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 17:40:54 -0700 Subject: [PATCH 05/19] Added support for RamSwap in the generated LXC config (to limit the swap and have the right default settings) --- container.go | 3 ++- lxc_template.go | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/container.go b/container.go index 530c08bc39..c837500549 100644 --- a/container.go +++ b/container.go @@ -53,7 +53,8 @@ type Container struct { type Config struct { Hostname string User string - Ram int64 + Ram int64 // Memory limit (in bytes) + RamSwap int64 // Total memory usage (ram + swap); set `-1' to disable swap Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin diff --git a/lxc_template.go b/lxc_template.go index 47f2058dc4..5a37624a32 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -88,14 +88,29 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se {{if .Config.Ram}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} +{{with $ramSwap := getRamSwap .Config}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} +{{end}} {{end}} ` var LxcTemplateCompiled *template.Template +func getRamSwap(config *Config) int64 { + // By default, RamSwap is set to twice the size of RAM. + // If you want to omit RamSwap, set it to `-1'. + if config.RamSwap < 0 { + return 0 + } + return config.Ram * 2 +} + func init() { var err error - LxcTemplateCompiled, err = template.New("lxc").Parse(LxcTemplate) + funcMap := template.FuncMap{ + "getRamSwap": getRamSwap, + } + LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { panic(err) } From 43875a73a0c2ee51f8bcd36634bbf64d836df6f8 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 11 Mar 2013 17:47:28 -0700 Subject: [PATCH 06/19] added a fix to the upstart command that wasn't setting hte LOCALE correctly, which was causing issues with UTF-8 bundles --- puppet/modules/docker/templates/dockerd.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/docker/templates/dockerd.conf b/puppet/modules/docker/templates/dockerd.conf index a00d322bcb..c05f606391 100644 --- a/puppet/modules/docker/templates/dockerd.conf +++ b/puppet/modules/docker/templates/dockerd.conf @@ -7,5 +7,6 @@ start on runlevel [3] respawn script - /home/vagrant/dockerd/dockerd + test -f /etc/default/locale && . /etc/default/locale || true + LANG=$LANG LC_ALL=$LANG /usr/local/bin/dockerd end script From d4da0a0ddbb3ec1979941bc3d2425aab3a6b3f19 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 11 Mar 2013 17:48:39 -0700 Subject: [PATCH 07/19] changed the docker version in vagrant to pull down latest version, moved some things around and put docker and dockerd into /usr/local/bin now. Also added git to install package list --- puppet/modules/docker/manifests/init.pp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index 2923b302fc..ed3dd62b63 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -1,14 +1,15 @@ class docker { # update this with latest docker binary distro - $docker_url = "https://dl.dropbox.com/u/20637798/docker.tar.gz" + $docker_url = "http://docker.io.s3.amazonaws.com/builds/$kernel/$hardwaremodel/docker-master.tgz" # update this with latest go binary distry $go_url = "http://go.googlecode.com/files/go1.0.3.linux-amd64.tar.gz" - Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget", "bsdtar"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar", "git"]: } + + notify { "docker_url = $docker_url": withpath => true } exec { "debootstrap" : require => Package["debootstrap"], @@ -26,7 +27,7 @@ class docker { exec { "fetch-docker" : require => Package["wget"], command => "/usr/bin/wget -O - $docker_url | /bin/tar xz -C /home/vagrant", - creates => "/home/vagrant/docker/dockerd" + creates => "/home/vagrant/docker-master" } file { "/etc/init/dockerd.conf": @@ -39,10 +40,16 @@ class docker { exec { "copy-docker-bin" : require => Exec["fetch-docker"], - command => "/bin/cp /home/vagrant/docker/docker /usr/local/bin", + command => "/bin/cp /home/vagrant/docker-master/docker /usr/local/bin", creates => "/usr/local/bin/docker" } + exec { "copy-dockerd-bin" : + require => Exec["fetch-docker"], + command => "/bin/cp /home/vagrant/docker-master/dockerd /usr/local/bin", + creates => "/usr/local/bin/dockerd" + } + service { "dockerd" : ensure => "running", start => "/sbin/initctl start dockerd", From 4e5ae883722ad814ef19e3f9d333f6218db752a5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:15:29 -0700 Subject: [PATCH 08/19] Implemented unit tests for the generated LXC config --- container_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++ rcli/http.go | 9 +++----- rcli/tcp.go | 12 +++++------ rcli/types.go | 10 ++++----- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/container_test.go b/container_test.go index 8c187ecb97..f79abe336e 100644 --- a/container_test.go +++ b/container_test.go @@ -1,9 +1,12 @@ package docker import ( + "bufio" "fmt" "io" "io/ioutil" + "math/rand" + "os" "sort" "strings" "testing" @@ -561,6 +564,58 @@ func TestEnv(t *testing.T) { } } +func grepFile(t *testing.T, path string, pattern string) { + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + r := bufio.NewReader(f) + var ( + line string + ) + err = nil + for err == nil { + line, err = r.ReadString('\n') + if strings.Contains(line, pattern) == true { + return + } + } + t.Fatalf("grepFile: pattern \"%s\" not found in \"%s\"", pattern, path) +} + +func TestLXCConfig(t *testing.T) { + docker, err := newTestDocker() + if err != nil { + t.Fatal(err) + } + // Ram is allocated randomly for testing + rand.Seed(time.Now().UTC().UnixNano()) + ramMin := 33554432 + ramMax := 536870912 + ram := ramMin + rand.Intn(ramMax-ramMin) + container, err := docker.Create( + "config_test", + "/bin/true", + []string{}, + []string{testLayerPath}, + &Config{ + Hostname: "foobar", + Ram: int64(ram), + }, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container) + container.generateLXCConfig() + grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", ram)) + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", ram*2)) +} + func BenchmarkRunSequencial(b *testing.B) { docker, err := newTestDocker() if err != nil { diff --git a/rcli/http.go b/rcli/http.go index e6cb5657d9..cc8d3b149e 100644 --- a/rcli/http.go +++ b/rcli/http.go @@ -1,13 +1,12 @@ package rcli import ( + "fmt" "net/http" "net/url" "path" - "fmt" ) - // Use this key to encode an RPC call into an URL, // eg. domain.tld/path/to/method?q=get_user&q=gordon const ARG_URL_KEY = "q" @@ -16,18 +15,16 @@ func URLToCall(u *url.URL) (method string, args []string) { return path.Base(u.Path), u.Query()[ARG_URL_KEY] } - func ListenAndServeHTTP(addr string, service Service) error { return http.ListenAndServe(addr, http.HandlerFunc( - func (w http.ResponseWriter, r *http.Request) { + func(w http.ResponseWriter, r *http.Request) { cmd, args := URLToCall(r.URL) if err := call(service, r.Body, &AutoFlush{w}, append([]string{cmd}, args...)...); err != nil { - fmt.Fprintf(w, "Error: " + err.Error() + "\n") + fmt.Fprintf(w, "Error: "+err.Error()+"\n") } })) } - type AutoFlush struct { http.ResponseWriter } diff --git a/rcli/tcp.go b/rcli/tcp.go index 0a06d459ce..869a3bcdb6 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -1,13 +1,13 @@ package rcli import ( + "bufio" + "encoding/json" + "fmt" "io" "io/ioutil" - "net" "log" - "fmt" - "encoding/json" - "bufio" + "net" ) // Connect to a remote endpoint using protocol `proto` and address `addr`, @@ -44,7 +44,7 @@ func ListenAndServe(proto, addr string, service Service) error { go func() { if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") - fmt.Fprintf(conn, "Error: " + err.Error() + "\n") + fmt.Fprintf(conn, "Error: "+err.Error()+"\n") } conn.Close() }() @@ -53,7 +53,6 @@ func ListenAndServe(proto, addr string, service Service) error { return nil } - // Parse an rcli call on a new connection, and pass it to `service` if it // is valid. func Serve(conn io.ReadWriter, service Service) error { @@ -68,4 +67,3 @@ func Serve(conn io.ReadWriter, service Service) error { } return nil } - diff --git a/rcli/types.go b/rcli/types.go index b8572cd896..52079291b6 100644 --- a/rcli/types.go +++ b/rcli/types.go @@ -8,13 +8,13 @@ package rcli // are the usual suspects. import ( + "errors" + "flag" "fmt" "io" - "reflect" - "flag" "log" + "reflect" "strings" - "errors" ) type Service interface { @@ -25,7 +25,6 @@ type Service interface { type Cmd func(io.ReadCloser, io.Writer, ...string) error type CmdMethod func(Service, io.ReadCloser, io.Writer, ...string) error - func call(service Service, stdin io.ReadCloser, stdout io.Writer, args ...string) error { if len(args) == 0 { args = []string{"help"} @@ -63,7 +62,7 @@ func getMethod(service Service, name string) Cmd { return nil } } - methodName := "Cmd"+strings.ToUpper(name[:1])+strings.ToLower(name[1:]) + methodName := "Cmd" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:]) method, exists := reflect.TypeOf(service).MethodByName(methodName) if !exists { return nil @@ -91,4 +90,3 @@ func Subcmd(output io.Writer, name, signature, description string) *flag.FlagSet } return flags } - From 948961831ab6cfb0c94e1e80552f4438d1f434a7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:25:02 -0700 Subject: [PATCH 09/19] Renamed Container property Ram to Memory before it is too late --- container.go | 14 +++++++------- container_test.go | 18 +++++++++--------- lxc_template.go | 20 ++++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/container.go b/container.go index c837500549..9aad176ed9 100644 --- a/container.go +++ b/container.go @@ -51,13 +51,13 @@ type Container struct { } type Config struct { - Hostname string - User string - Ram int64 // Memory limit (in bytes) - RamSwap int64 // Total memory usage (ram + swap); set `-1' to disable swap - Ports []int - Tty bool // Attach standard streams to a tty, including stdin if it is not closed. - OpenStdin bool // Open stdin + Hostname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + Ports []int + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin } type NetworkSettings struct { diff --git a/container_test.go b/container_test.go index f79abe336e..f9c835edf9 100644 --- a/container_test.go +++ b/container_test.go @@ -24,7 +24,7 @@ func TestStart(t *testing.T) { []string{"-al"}, []string{testLayerPath}, &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { @@ -60,7 +60,7 @@ func TestRun(t *testing.T) { []string{"-al"}, []string{testLayerPath}, &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { @@ -589,11 +589,11 @@ func TestLXCConfig(t *testing.T) { if err != nil { t.Fatal(err) } - // Ram is allocated randomly for testing + // Memory is allocated randomly for testing rand.Seed(time.Now().UTC().UnixNano()) - ramMin := 33554432 - ramMax := 536870912 - ram := ramMin + rand.Intn(ramMax-ramMin) + memMin := 33554432 + memMax := 536870912 + mem := memMin + rand.Intn(memMax-memMin) container, err := docker.Create( "config_test", "/bin/true", @@ -601,7 +601,7 @@ func TestLXCConfig(t *testing.T) { []string{testLayerPath}, &Config{ Hostname: "foobar", - Ram: int64(ram), + Memory: int64(mem), }, ) if err != nil { @@ -611,9 +611,9 @@ func TestLXCConfig(t *testing.T) { container.generateLXCConfig() grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") grepFile(t, container.lxcConfigPath, - fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", ram)) + fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) grepFile(t, container.lxcConfigPath, - fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", ram*2)) + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) } func BenchmarkRunSequencial(b *testing.B) { diff --git a/lxc_template.go b/lxc_template.go index 5a37624a32..5d92c9e6de 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -85,10 +85,10 @@ lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config # limits -{{if .Config.Ram}} -lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} -lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} -{{with $ramSwap := getRamSwap .Config}} +{{if .Config.Memory}} +lxc.cgroup.memory.limit_in_bytes = {{.Config.Memory}} +lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Memory}} +{{with $ramSwap := getMemorySwap .Config}} lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} {{end}} {{end}} @@ -96,19 +96,19 @@ lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} var LxcTemplateCompiled *template.Template -func getRamSwap(config *Config) int64 { - // By default, RamSwap is set to twice the size of RAM. - // If you want to omit RamSwap, set it to `-1'. - if config.RamSwap < 0 { +func getMemorySwap(config *Config) int64 { + // By default, MemorySwap is set to twice the size of RAM. + // If you want to omit MemorySwap, set it to `-1'. + if config.MemorySwap < 0 { return 0 } - return config.Ram * 2 + return config.Memory * 2 } func init() { var err error funcMap := template.FuncMap{ - "getRamSwap": getRamSwap, + "getMemorySwap": getMemorySwap, } LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { From 3684b67572605de492abba041695bd80ef42bcb5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:51:24 -0700 Subject: [PATCH 10/19] Added -m to the run command in order to set a memory limit to a container --- server/server.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index bc642cdbb0..a490641d28 100644 --- a/server/server.go +++ b/server/server.go @@ -712,10 +712,18 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, + tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] container, err := srv.containers.Create(id, cmd, args, img.Layers, - &docker.Config{Hostname: id, Ports: ports, User: user, Tty: tty, OpenStdin: openStdin}) + &docker.Config{ + Hostname: id, + Ports: ports, + User: user, + Tty: tty, + OpenStdin: openStdin, + Memory: memory, + }) if err != nil { return nil, err } @@ -799,6 +807,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") fl_comment := cmd.String("c", "", "Comment") + fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") var fl_ports ports cmd.Var(&fl_ports, "p", "Map a network port to the container") if err := cmd.Parse(args); err != nil { @@ -826,7 +835,8 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) return errors.New("No such image: " + name) } // Create new container - container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, *fl_stdin, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, + *fl_stdin, *fl_memory, *fl_comment, cmdline[0], cmdline[1:]...) if err != nil { return errors.New("Error creating container: " + err.Error()) } From f8fee421819e3c272e1c20890da7c90b4da971ad Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:55:14 -0700 Subject: [PATCH 11/19] Missed a rename --- lxc_template.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lxc_template.go b/lxc_template.go index 5d92c9e6de..2a60e8b96e 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -88,8 +88,8 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se {{if .Config.Memory}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Memory}} lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Memory}} -{{with $ramSwap := getMemorySwap .Config}} -lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} +{{with $memSwap := getMemorySwap .Config}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{$memSwap}} {{end}} {{end}} ` From 53e2e8209fd882a3b4f073ce30dec444cc9c7cc0 Mon Sep 17 00:00:00 2001 From: jpetazzo Date: Mon, 11 Mar 2013 19:03:22 -0700 Subject: [PATCH 12/19] Add missing shebang cloud-init expects a shebang in the beginning of the script --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index af5c214d78..52cd3c0a08 100644 --- a/install.sh +++ b/install.sh @@ -1,4 +1,5 @@ -# This script is meant for quick & easy install via 'curl URL-OF-SCRIPPT | bash' +#!/bin/sh +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay cd /tmp From af2aac8736d71fe79492f9abf3a16fba97857eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:07:16 -0700 Subject: [PATCH 13/19] Stream and unpack the tarball on the fly We don't need temporary files. Also, get the tarball from get.docker.io instead of Dropbox. --- install.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index 52cd3c0a08..2ecb8b47ae 100644 --- a/install.sh +++ b/install.sh @@ -2,18 +2,13 @@ # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay -cd /tmp - echo "Ensuring dependencies are installed..." apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null -echo "Downloading docker binary..." -wget -q https://dl.dropbox.com/u/20637798/docker.tar.gz 2>&1 > /dev/null -tar -xf docker.tar.gz 2>&1 > /dev/null - -echo "Installing into /usr/local/bin..." -mv docker/docker /usr/local/bin -mv dockerd/dockerd /usr/local/bin +echo "Downloading docker binary and uncompressing into /usr/local/bin..." +curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | +tar -C /usr/local/bin --strip-components=1 -zxf- \ +docker-master/docker docker-master/dockerd if [[ -f /etc/init/dockerd.conf ]] then @@ -26,10 +21,5 @@ fi echo "Restarting dockerd..." restart dockerd > /dev/null -echo "Cleaning up..." -rmdir docker -rmdir dockerd -rm docker.tar.gz - echo "Finished!" echo From fda456b82e51224fe4e7ae6c0bc2b0f0812729ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:08:21 -0700 Subject: [PATCH 14/19] Use "start" instead of "restart" Because when the job isn't running, "restart" won't work. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 2ecb8b47ae..43e8ecce1e 100644 --- a/install.sh +++ b/install.sh @@ -18,8 +18,8 @@ else echo "exec /usr/local/bin/dockerd" > /etc/init/dockerd.conf fi -echo "Restarting dockerd..." -restart dockerd > /dev/null +echo "Starting dockerd..." +start dockerd > /dev/null echo "Finished!" echo From c1fa4efad66a1ae8cbccd7c55bc195be03d94b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:09:19 -0700 Subject: [PATCH 15/19] Try to load, or install, AUFS modules When running precise or quantal, installing the matching -extra kernel will install the required file. --- install.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 43e8ecce1e..4e592e5dcf 100644 --- a/install.sh +++ b/install.sh @@ -2,8 +2,34 @@ # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay -echo "Ensuring dependencies are installed..." -apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null +echo "Ensuring basic dependencies are installed..." +apt-get -qq update +apt-get -qq install lxc wget bsdtar + +echo "Looking in /proc/filesystems to see if we have AUFS support..." +if grep -q aufs /proc/filesystems +then + echo "Found." +else + echo "Ahem, it looks like the current kernel does not support AUFS." + echo "Let's see if we can load the AUFS module with modprobe..." + if modprobe aufs + then + echo "Module loaded." + else + echo "Ahem, things didn't turn out as expected." + KPKG=linux-image-extra-$(uname -r) + echo "Trying to install $KPKG..." + if apt-get -qq install $KPKG + then + echo "Installed." + else + echo "Oops, we couldn't install the -extra kernel." + echo "Are you sure you are running a supported version of Ubuntu?" + echo "Proceeding anyway, but Docker will probably NOT WORK!" + fi + fi +fi echo "Downloading docker binary and uncompressing into /usr/local/bin..." curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | From f90f6a0b9248489c43188ca79b9e0dfc306fd8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:13:08 -0700 Subject: [PATCH 16/19] Get rid of bashism --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 4e592e5dcf..f8dca17476 100644 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ #!/bin/sh -# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | sh' # Courtesy of Jeff Lindsay echo "Ensuring basic dependencies are installed..." @@ -36,7 +36,7 @@ curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | tar -C /usr/local/bin --strip-components=1 -zxf- \ docker-master/docker docker-master/dockerd -if [[ -f /etc/init/dockerd.conf ]] +if [ -f /etc/init/dockerd.conf ] then echo "Upstart script already exists." else From b728c1a81fbb441442d635788e814d47436c613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:37:38 -0700 Subject: [PATCH 17/19] Add script upload instructions --- install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index f8dca17476..0cfba4ddc3 100644 --- a/install.sh +++ b/install.sh @@ -1,6 +1,10 @@ #!/bin/sh # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | sh' -# Courtesy of Jeff Lindsay +# Original version by Jeff Lindsay +# Revamped by Jerome Petazzoni +# +# This script canonical location is http://get.docker.io/; to update it, run: +# s3cmd put -m text/x-shellscript -P install.sh s3://get.docker.io/index echo "Ensuring basic dependencies are installed..." apt-get -qq update @@ -47,5 +51,5 @@ fi echo "Starting dockerd..." start dockerd > /dev/null -echo "Finished!" +echo "Done." echo From 94baf2c5b9b2eb49b554d933f9997da072905d88 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Tue, 12 Mar 2013 12:05:04 -0700 Subject: [PATCH 18/19] upgraded kernel to 3.5.0-25 to fix a kernel bug --- Vagrantfile | 4 ++-- puppet/modules/docker/manifests/init.pp | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index dcc3e50bdb..c823a39581 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,11 +7,11 @@ Vagrant::Config.run do |config| # please see the online documentation at vagrantup.com. # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = "quantal64" + config.vm.box = "quantal64_3.5.0-25" # The url from where the 'config.vm.box' box will be fetched if it # doesn't already exist on the user's system. - config.vm.box_url = "http://unworkable.org/~niallo/quantal64.box" + config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" # Boot with a GUI so you can see the screen. (Default is headless) # config.vm.boot_mode = :gui diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index ed3dd62b63..38b40a8e39 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -7,7 +7,11 @@ class docker { Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget", "bsdtar", "git"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar", "git", + "linux-image-3.5.0-25-generic", + "linux-image-extra-3.5.0-25-generic", + "virtualbox-guest-utils", + "linux-headers-3.5.0-25-generic"]: } notify { "docker_url = $docker_url": withpath => true } @@ -50,6 +54,11 @@ class docker { creates => "/usr/local/bin/dockerd" } + exec { "vbox-add" : + require => Package["linux-headers-3.5.0-25-generic"], + command => "/etc/init.d/vboxadd setup", + } + service { "dockerd" : ensure => "running", start => "/sbin/initctl start dockerd", From b2de17bf2cfdf90f347109452bcb8763138cff46 Mon Sep 17 00:00:00 2001 From: Louis Opter Date: Tue, 12 Mar 2013 12:12:40 -0700 Subject: [PATCH 19/19] Automatically remove the rcfile generated by docker -i from /tmp --- client/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/client.go b/client/client.go index 4c4ea1c5e3..814aed2f18 100644 --- a/client/client.go +++ b/client/client.go @@ -111,6 +111,7 @@ func InteractiveMode(scripts ...string) error { if err != nil { return err } + defer os.Remove(rcfile.Name()) io.WriteString(rcfile, "enable -n help\n") os.Setenv("PATH", tmp+":"+os.Getenv("PATH")) os.Setenv("PS1", "\\h docker> ")