diff --git a/README.md b/README.md index f778ff521d..44ff977298 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Note that some methods are community contributions and not yet officially suppor * [Ubuntu 12.04 and 12.10 (officially supported)](http://docs.docker.io/en/latest/installation/ubuntulinux/) * [Arch Linux](http://docs.docker.io/en/latest/installation/archlinux/) -* [MacOS X (with Vagrant)](http://docs.docker.io/en/latest/installation/macos/) +* [Mac OS X (with Vagrant)](http://docs.docker.io/en/latest/installation/vagrant/) * [Windows (with Vagrant)](http://docs.docker.io/en/latest/installation/windows/) * [Amazon EC2 (with Vagrant)](http://docs.docker.io/en/latest/installation/amazon/) @@ -262,7 +262,7 @@ Setting up a dev environment Instructions that have been verified to work on Ubuntu 12.10, ```bash -sudo apt-get -y install lxc wget bsdtar curl golang git +sudo apt-get -y install lxc curl xz-utils golang git export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH diff --git a/Vagrantfile b/Vagrantfile index 5b3a1f476d..aadabb8711 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,6 +3,7 @@ BOX_NAME = ENV['BOX_NAME'] || "ubuntu" BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box" +VF_BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64_vmware_fusion.box" AWS_REGION = ENV['AWS_REGION'] || "us-east-1" AWS_AMI = ENV['AWS_AMI'] || "ami-d0f89fb9" FORWARD_DOCKER_PORTS = ENV['FORWARD_DOCKER_PORTS'] @@ -67,6 +68,13 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| rs.image = /Ubuntu/ end + config.vm.provider :vmware_fusion do |f, override| + override.vm.box = BOX_NAME + override.vm.box_url = VF_BOX_URI + override.vm.synced_folder ".", "/vagrant", disabled: true + f.vmx["displayName"] = "docker" + end + config.vm.provider :virtualbox do |vb| config.vm.box = BOX_NAME config.vm.box_url = BOX_URI diff --git a/archive.go b/archive.go index e10fbfdae3..b129e28797 100644 --- a/archive.go +++ b/archive.go @@ -1,6 +1,7 @@ package docker import ( + "bufio" "errors" "fmt" "github.com/dotcloud/docker/utils" @@ -22,6 +23,37 @@ const ( Xz ) +func DetectCompression(source []byte) Compression { + for _, c := range source[:10] { + utils.Debugf("%x", c) + } + + sourceLen := len(source) + for compression, m := range map[Compression][]byte{ + Bzip2: {0x42, 0x5A, 0x68}, + Gzip: {0x1F, 0x8B, 0x08}, + Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, + } { + fail := false + if len(m) > sourceLen { + utils.Debugf("Len too short") + continue + } + i := 0 + for _, b := range m { + if b != source[i] { + fail = true + break + } + i++ + } + if !fail { + return compression + } + } + return Uncompressed +} + func (compression *Compression) Flag() string { switch *compression { case Bzip2: @@ -57,15 +89,14 @@ func Tar(path string, compression Compression) (io.Reader, error) { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) { - args := []string{"bsdtar", "-f", "-", "-C", path} + args := []string{"tar", "-f", "-", "-C", path} if filter == nil { filter = []string{"."} } for _, f := range filter { args = append(args, "-c"+compression.Flag(), f) } - cmd := exec.Command(args[0], args[1:]...) - return CmdStream(cmd) + return CmdStream(exec.Command(args[0], args[1:]...)) } // Untar reads a stream of bytes from `archive`, parses it as a tar archive, @@ -74,8 +105,18 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader // identity (uncompressed), gzip, bzip2, xz. // FIXME: specify behavior when target path exists vs. doesn't exist. func Untar(archive io.Reader, path string) error { - cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x") - cmd.Stdin = archive + + bufferedArchive := bufio.NewReaderSize(archive, 10) + buf, err := bufferedArchive.Peek(10) + if err != nil { + return err + } + compression := DetectCompression(buf) + + utils.Debugf("Archive compression detected: %s", compression.Extension()) + + cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag()) + cmd.Stdin = bufferedArchive // Hardcode locale environment for predictable outcome regardless of host configuration. // (see https://github.com/dotcloud/docker/issues/355) cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"} diff --git a/archive_test.go b/archive_test.go index f583604497..bb4235ad5b 100644 --- a/archive_test.go +++ b/archive_test.go @@ -1,10 +1,13 @@ package docker import ( + "bytes" + "fmt" "io" "io/ioutil" "os" "os/exec" + "path" "testing" "time" ) @@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) { } } -func TestTarUntar(t *testing.T) { - archive, err := Tar(".", Uncompressed) +func tarUntar(t *testing.T, origin string, compression Compression) error { + archive, err := Tar(origin, compression) if err != nil { t.Fatal(err) } + + buf := make([]byte, 10) + if _, err := archive.Read(buf); err != nil { + return err + } + archive = io.MultiReader(bytes.NewReader(buf), archive) + + detectedCompression := DetectCompression(buf) + if detectedCompression.Extension() != compression.Extension() { + return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension()) + } + tmp, err := ioutil.TempDir("", "docker-test-untar") if err != nil { - t.Fatal(err) + return err } defer os.RemoveAll(tmp) if err := Untar(archive, tmp); err != nil { - t.Fatal(err) + return err } if _, err := os.Stat(tmp); err != nil { - t.Fatalf("Error stating %s: %s", tmp, err.Error()) + return err + } + return nil +} + +func TestTarUntar(t *testing.T) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(origin) + if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { + t.Fatal(err) + } + + for _, c := range []Compression{ + Uncompressed, + Gzip, + Bzip2, + Xz, + } { + if err := tarUntar(t, origin, c); err != nil { + t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) + } } } diff --git a/commands.go b/commands.go index ae114f84d7..05a484ddb1 100644 --- a/commands.go +++ b/commands.go @@ -1061,6 +1061,10 @@ func (cli *DockerCli) CmdAttach(args ...string) error { return err } + if !container.State.Running { + return fmt.Errorf("Impossible to attach to a stopped container, start it first") + } + splitStderr := container.Config.Tty connections := 1 @@ -1260,16 +1264,6 @@ func (cli *DockerCli) CmdRun(args ...string) error { fmt.Fprintln(os.Stderr, "WARNING: ", warning) } - splitStderr := !config.Tty - - connections := 0 - if config.AttachStdin || config.AttachStdout || (!splitStderr && config.AttachStderr) { - connections += 1 - } - if splitStderr && config.AttachStderr { - connections += 1 - } - //start the container _, _, err = cli.call("POST", "/containers/"+out.ID+"/start", nil) if err != nil { @@ -1278,19 +1272,11 @@ func (cli *DockerCli) CmdRun(args ...string) error { if !config.AttachStdout && !config.AttachStderr { fmt.Println(out.ID) - } - if connections > 0 { - chErrors := make(chan error, connections) + } else { if config.Tty { cli.monitorTtySize(out.ID) } - if splitStderr && config.AttachStderr { - go func() { - chErrors <- cli.hijack("POST", "/containers/"+out.ID+"/attach?logs=1&stream=1&stderr=1", config.Tty, nil, os.Stderr) - }() - } - v := url.Values{} v.Set("logs", "1") v.Set("stream", "1") @@ -1301,19 +1287,12 @@ func (cli *DockerCli) CmdRun(args ...string) error { if config.AttachStdout { v.Set("stdout", "1") } - if !splitStderr && config.AttachStderr { + if config.AttachStderr { v.Set("stderr", "1") } - go func() { - chErrors <- cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout) - }() - for connections > 0 { - err := <-chErrors - if err != nil { - utils.Debugf("Error hijack: %s", err) - return err - } - connections -= 1 + if err := cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout); err != nil { + utils.Debugf("Error hijack: %s", err) + return err } } return nil diff --git a/contrib/install.sh b/contrib/install.sh index 7db577a9da..cf097da670 100755 --- a/contrib/install.sh +++ b/contrib/install.sh @@ -8,7 +8,7 @@ echo "Ensuring basic dependencies are installed..." apt-get -qq update -apt-get -qq install lxc wget bsdtar +apt-get -qq install lxc wget echo "Looking in /proc/filesystems to see if we have AUFS support..." if grep -q aufs /proc/filesystems diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index ce2ecd41e8..1f39364cb1 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -33,7 +33,7 @@ Installation sudo apt-get install python-software-properties sudo add-apt-repository ppa:gophers/go sudo apt-get update - sudo apt-get -y install lxc wget bsdtar curl golang-stable git aufs-tools + sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH diff --git a/docs/sources/installation/binaries.rst b/docs/sources/installation/binaries.rst index e7a07b6db1..6d87787752 100644 --- a/docs/sources/installation/binaries.rst +++ b/docs/sources/installation/binaries.rst @@ -30,8 +30,7 @@ Dependencies: * 3.8 Kernel (read more about :ref:`kernel`) * AUFS filesystem support * lxc -* bsdtar - +* xz-utils Get the docker binary: ---------------------- diff --git a/graph_test.go b/graph_test.go index 8dedb96669..18682338d9 100644 --- a/graph_test.go +++ b/graph_test.go @@ -192,11 +192,19 @@ func TestDelete(t *testing.T) { } assertNImages(graph, t, 0) + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } // Test 2 create (same name) / 1 delete img1, err := graph.Create(archive, nil, "Testing", "", nil) if err != nil { t.Fatal(err) } + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil { t.Fatal(err) } @@ -212,6 +220,10 @@ func TestDelete(t *testing.T) { } assertNImages(graph, t, 1) + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } // Test delete twice (pull -> rm -> pull -> rm) if err := graph.Register(archive, false, img1); err != nil { t.Fatal(err) diff --git a/hack/Vagrantfile b/hack/Vagrantfile index 318f835f4f..e02dfe06db 100644 --- a/hack/Vagrantfile +++ b/hack/Vagrantfile @@ -22,7 +22,7 @@ Vagrant::Config.run do |config| pkg_cmd = "touch #{DOCKER_PATH}; " # Install docker dependencies pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ - "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ + "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ "chown -R #{USER}.#{USER} #{GOPATH}; " \ "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" config.vm.provision :shell, :inline => pkg_cmd diff --git a/registry/registry.go b/registry/registry.go index 131b02708e..18bdad26f2 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" ) @@ -106,40 +107,45 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut } // Retrieve an image from the Registry. -// Returns the Image object as well as the layer as an Archive (io.Reader) -func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, error) { +func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { // Get the JSON req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { - return nil, fmt.Errorf("Failed to download json: %s", err) + return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) if err != nil { - return nil, fmt.Errorf("Failed to download json: %s", err) + return nil, -1, fmt.Errorf("Failed to download json: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) } + + imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) + if err != nil { + return nil, -1, err + } + jsonString, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) } - return jsonString, nil + return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) { +func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { - return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err) + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) if err != nil { - return nil, -1, err + return nil, err } - return res.Body, int(res.ContentLength), nil + return res.Body, nil } func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { @@ -481,7 +487,7 @@ type Registry struct { func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { httpTransport := &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, } r := &Registry{ diff --git a/server.go b/server.go index 8e41091c32..7375dddd90 100644 --- a/server.go +++ b/server.go @@ -321,7 +321,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin for _, id := range history { if !srv.runtime.graph.Exists(id) { out.Write(sf.FormatStatus("Pulling %s metadata", id)) - imgJSON, err := r.GetRemoteImageJSON(id, endpoint, token) + imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -333,12 +333,12 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin // Get the layer out.Write(sf.FormatStatus("Pulling %s fs layer", id)) - layer, contentLength, err := r.GetRemoteImageLayer(img.ID, endpoint, token) + layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil { return err } } @@ -942,9 +942,6 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std if container.State.Ghost { return fmt.Errorf("Impossible to attach to a ghost container") } - if !container.State.Running { - return fmt.Errorf("Impossible to attach to a stopped container, start it first") - } var ( cStdin io.ReadCloser diff --git a/testing/Vagrantfile b/testing/Vagrantfile index e304a8d087..f2f6ca8248 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -30,7 +30,7 @@ Vagrant::Config.run do |config| # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ - "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; " + "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " # Activate new kernel pkg_cmd << "shutdown -r +1; " config.vm.provision :shell, :inline => pkg_cmd diff --git a/utils/utils.go b/utils/utils.go index b77c1ea053..af56f80af8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -86,7 +86,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } if r.readProgress-r.lastUpdate > updateEvery || err != nil { if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template, r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) + fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) } else { fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a") }