Merge branch 'master' into fix-add-behavior

README.md

@@ -108,7 +108,7 @@ Note that some methods are community contributions and not yet officially suppor
 
 * [Ubuntu 12.04 and 12.10 (officially supported)](http://docs.docker.io/en/latest/installation/ubuntulinux/)
 * [Arch Linux](http://docs.docker.io/en/latest/installation/archlinux/)
-* [MacOS X (with Vagrant)](http://docs.docker.io/en/latest/installation/macos/)
+* [Mac OS X (with Vagrant)](http://docs.docker.io/en/latest/installation/vagrant/)
 * [Windows (with Vagrant)](http://docs.docker.io/en/latest/installation/windows/)
 * [Amazon EC2 (with Vagrant)](http://docs.docker.io/en/latest/installation/amazon/)
 
@@ -262,7 +262,7 @@ Setting up a dev environment
 Instructions that have been verified to work on Ubuntu 12.10,
 
 ```bash
-sudo apt-get -y install lxc wget bsdtar curl golang git
+sudo apt-get -y install lxc curl xz-utils golang git
 
 export GOPATH=~/go/
 export PATH=$GOPATH/bin:$PATH

Vagrantfile

@@ -3,6 +3,7 @@
 
 BOX_NAME = ENV['BOX_NAME'] || "ubuntu"
 BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box"
+VF_BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64_vmware_fusion.box"
 AWS_REGION = ENV['AWS_REGION'] || "us-east-1"
 AWS_AMI    = ENV['AWS_AMI']    || "ami-d0f89fb9"
 FORWARD_DOCKER_PORTS = ENV['FORWARD_DOCKER_PORTS']
@@ -67,6 +68,13 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
     rs.image    = /Ubuntu/
   end
 
+  config.vm.provider :vmware_fusion do |f, override|
+    override.vm.box = BOX_NAME
+    override.vm.box_url = VF_BOX_URI
+    override.vm.synced_folder ".", "/vagrant", disabled: true
+    f.vmx["displayName"] = "docker"
+  end
+
   config.vm.provider :virtualbox do |vb|
     config.vm.box = BOX_NAME
     config.vm.box_url = BOX_URI

archive.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"github.com/dotcloud/docker/utils"
@@ -22,6 +23,37 @@ const (
 	Xz
 )
 
+func DetectCompression(source []byte) Compression {
+	for _, c := range source[:10] {
+		utils.Debugf("%x", c)
+	}
+
+	sourceLen := len(source)
+	for compression, m := range map[Compression][]byte{
+		Bzip2: {0x42, 0x5A, 0x68},
+		Gzip:  {0x1F, 0x8B, 0x08},
+		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
+	} {
+		fail := false
+		if len(m) > sourceLen {
+			utils.Debugf("Len too short")
+			continue
+		}
+		i := 0
+		for _, b := range m {
+			if b != source[i] {
+				fail = true
+				break
+			}
+			i++
+		}
+		if !fail {
+			return compression
+		}
+	}
+	return Uncompressed
+}
+
 func (compression *Compression) Flag() string {
 	switch *compression {
 	case Bzip2:
@@ -57,15 +89,14 @@ func Tar(path string, compression Compression) (io.Reader, error) {
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
 func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
-	args := []string{"bsdtar", "-f", "-", "-C", path}
+	args := []string{"tar", "-f", "-", "-C", path}
 	if filter == nil {
 		filter = []string{"."}
 	}
 	for _, f := range filter {
 		args = append(args, "-c"+compression.Flag(), f)
 	}
-	cmd := exec.Command(args[0], args[1:]...)
+	return CmdStream(exec.Command(args[0], args[1:]...))
-	return CmdStream(cmd)
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
@@ -74,8 +105,18 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
 func Untar(archive io.Reader, path string) error {
-	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x")
+
-	cmd.Stdin = archive
+	bufferedArchive := bufio.NewReaderSize(archive, 10)
+	buf, err := bufferedArchive.Peek(10)
+	if err != nil {
+		return err
+	}
+	compression := DetectCompression(buf)
+
+	utils.Debugf("Archive compression detected: %s", compression.Extension())
+
+	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
+	cmd.Stdin = bufferedArchive
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
 	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}

archive_test.go

@@ -1,10 +1,13 @@
 package docker
 
 import (
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"testing"
 	"time"
 )
@@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) {
 	}
 }
 
-func TestTarUntar(t *testing.T) {
+func tarUntar(t *testing.T, origin string, compression Compression) error {
-	archive, err := Tar(".", Uncompressed)
+	archive, err := Tar(origin, compression)
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	buf := make([]byte, 10)
+	if _, err := archive.Read(buf); err != nil {
+		return err
+	}
+	archive = io.MultiReader(bytes.NewReader(buf), archive)
+
+	detectedCompression := DetectCompression(buf)
+	if detectedCompression.Extension() != compression.Extension() {
+		return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
+	}
+
 	tmp, err := ioutil.TempDir("", "docker-test-untar")
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 	defer os.RemoveAll(tmp)
 	if err := Untar(archive, tmp); err != nil {
-		t.Fatal(err)
+		return err
 	}
 	if _, err := os.Stat(tmp); err != nil {
-		t.Fatalf("Error stating %s: %s", tmp, err.Error())
+		return err
+	}
+	return nil
+}
+
+func TestTarUntar(t *testing.T) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(origin)
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range []Compression{
+		Uncompressed,
+		Gzip,
+		Bzip2,
+		Xz,
+	} {
+		if err := tarUntar(t, origin, c); err != nil {
+			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
+		}
 	}
 }

commands.go

@@ -1061,6 +1061,10 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 		return err
 	}
 
+	if !container.State.Running {
+		return fmt.Errorf("Impossible to attach to a stopped container, start it first")
+	}
+
 	splitStderr := container.Config.Tty
 
 	connections := 1
@@ -1260,16 +1264,6 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		fmt.Fprintln(os.Stderr, "WARNING: ", warning)
 	}
 
-	splitStderr := !config.Tty
-
-	connections := 0
-	if config.AttachStdin || config.AttachStdout || (!splitStderr && config.AttachStderr) {
-		connections += 1
-	}
-	if splitStderr && config.AttachStderr {
-		connections += 1
-	}
-
 	//start the container
 	_, _, err = cli.call("POST", "/containers/"+out.ID+"/start", nil)
 	if err != nil {
@@ -1278,19 +1272,11 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 
 	if !config.AttachStdout && !config.AttachStderr {
 		fmt.Println(out.ID)
-	}
+	} else {
-	if connections > 0 {
-		chErrors := make(chan error, connections)
 		if config.Tty {
 			cli.monitorTtySize(out.ID)
 		}
 
-		if splitStderr && config.AttachStderr {
-			go func() {
-				chErrors <- cli.hijack("POST", "/containers/"+out.ID+"/attach?logs=1&stream=1&stderr=1", config.Tty, nil, os.Stderr)
-			}()
-		}
-
 		v := url.Values{}
 		v.Set("logs", "1")
 		v.Set("stream", "1")
@@ -1301,19 +1287,12 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		if config.AttachStdout {
 			v.Set("stdout", "1")
 		}
-		if !splitStderr && config.AttachStderr {
+		if config.AttachStderr {
 			v.Set("stderr", "1")
 		}
-		go func() {
+		if err := cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout); err != nil {
-			chErrors <- cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout)
+			utils.Debugf("Error hijack: %s", err)
-		}()
+			return err
-		for connections > 0 {
-			err := <-chErrors
-			if err != nil {
-				utils.Debugf("Error hijack: %s", err)
-				return err
-			}
-			connections -= 1
 		}
 	}
 	return nil

contrib/install.sh

@@ -8,7 +8,7 @@
 
 echo "Ensuring basic dependencies are installed..."
 apt-get -qq update
-apt-get -qq install lxc wget bsdtar
+apt-get -qq install lxc wget
 
 echo "Looking in /proc/filesystems to see if we have AUFS support..."
 if grep -q aufs /proc/filesystems

docs/sources/contributing/devenvironment.rst

@@ -33,7 +33,7 @@ Installation
     sudo apt-get install python-software-properties
     sudo add-apt-repository ppa:gophers/go
     sudo apt-get update
-    sudo apt-get -y install lxc wget bsdtar curl golang-stable git aufs-tools
+    sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools
 
     export GOPATH=~/go/
     export PATH=$GOPATH/bin:$PATH

docs/sources/installation/binaries.rst

@@ -30,8 +30,7 @@ Dependencies:
 * 3.8 Kernel (read more about :ref:`kernel`)
 * AUFS filesystem support
 * lxc
-* bsdtar
+* xz-utils
-
 
 Get the docker binary:
 ----------------------

graph_test.go

@@ -192,11 +192,19 @@ func TestDelete(t *testing.T) {
 	}
 	assertNImages(graph, t, 0)
 
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Test 2 create (same name) / 1 delete
 	img1, err := graph.Create(archive, nil, "Testing", "", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil {
 		t.Fatal(err)
 	}
@@ -212,6 +220,10 @@ func TestDelete(t *testing.T) {
 	}
 	assertNImages(graph, t, 1)
 
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Test delete twice (pull -> rm -> pull -> rm)
 	if err := graph.Register(archive, false, img1); err != nil {
 		t.Fatal(err)

hack/Vagrantfile

@@ -22,7 +22,7 @@ Vagrant::Config.run do |config|
   pkg_cmd = "touch #{DOCKER_PATH}; "
   # Install docker dependencies
   pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \
-    "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
+    "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
     "chown -R #{USER}.#{USER} #{GOPATH}; " \
     "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile"
   config.vm.provision :shell, :inline => pkg_cmd

registry/registry.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 )
 
@@ -106,40 +107,45 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut
 }
 
 // Retrieve an image from the Registry.
-// Returns the Image object as well as the layer as an Archive (io.Reader)
+func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) {
-func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, error) {
 	// Get the JSON
 	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to download json: %s", err)
+		return nil, -1, fmt.Errorf("Failed to download json: %s", err)
 	}
 	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
 	res, err := r.client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to download json: %s", err)
+		return nil, -1, fmt.Errorf("Failed to download json: %s", err)
 	}
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP code %d", res.StatusCode)
+		return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode)
 	}
+
+	imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size"))
+	if err != nil {
+		return nil, -1, err
+	}
+
 	jsonString, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString)
+		return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString)
 	}
-	return jsonString, nil
+	return jsonString, imageSize, nil
 }
 
-func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) {
+func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) {
 	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil)
 	if err != nil {
-		return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err)
+		return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
 	}
 	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
 	res, err := r.client.Do(req)
 	if err != nil {
-		return nil, -1, err
+		return nil, err
 	}
-	return res.Body, int(res.ContentLength), nil
+	return res.Body, nil
 }
 
 func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) {
@@ -481,7 +487,7 @@ type Registry struct {
 func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry {
 	httpTransport := &http.Transport{
 		DisableKeepAlives: true,
-		Proxy: http.ProxyFromEnvironment,
+		Proxy:             http.ProxyFromEnvironment,
 	}
 
 	r := &Registry{

server.go

@@ -321,7 +321,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin
 	for _, id := range history {
 		if !srv.runtime.graph.Exists(id) {
 			out.Write(sf.FormatStatus("Pulling %s metadata", id))
-			imgJSON, err := r.GetRemoteImageJSON(id, endpoint, token)
+			imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
 			if err != nil {
 				// FIXME: Keep goging in case of error?
 				return err
@@ -333,12 +333,12 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin
 
 			// Get the layer
 			out.Write(sf.FormatStatus("Pulling %s fs layer", id))
-			layer, contentLength, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
+			layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
 			if err != nil {
 				return err
 			}
 			defer layer.Close()
-			if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil {
+			if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%v/%v (%v)"), sf), false, img); err != nil {
 				return err
 			}
 		}
@@ -942,9 +942,6 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std
 		if container.State.Ghost {
 			return fmt.Errorf("Impossible to attach to a ghost container")
 		}
-		if !container.State.Running {
-			return fmt.Errorf("Impossible to attach to a stopped container, start it first")
-		}
 
 		var (
 			cStdin           io.ReadCloser

testing/Vagrantfile

@@ -30,7 +30,7 @@ Vagrant::Config.run do |config|
     # Install docker dependencies
     pkg_cmd << "apt-get install -q -y python-software-properties; " \
       "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \
-      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; "
+      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; "
     # Activate new kernel
     pkg_cmd << "shutdown -r +1; "
     config.vm.provision :shell, :inline => pkg_cmd

utils/utils.go

@@ -86,7 +86,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) {
 	}
 	if r.readProgress-r.lastUpdate > updateEvery || err != nil {
 		if r.readTotal > 0 {
-			fmt.Fprintf(r.output, r.template, r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
+			fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
 		} else {
 			fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
 		}