From a96a26c62f6333e61134759256db43a7f5946631 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 17 Dec 2013 14:19:48 +0100 Subject: [PATCH 1/2] Handle compressed tars in ApplyLayer When pulling from a registry we get a compressed tar archive, so we need to wrap the stream in the right kind of compress reader. Unfortunately go doesn't have an Xz decompression method, but I don't think any docker layers use that atm anyway. --- archive/archive.go | 31 +++++++++++++++++++++++++++++++ archive/diff.go | 5 +++++ 2 files changed, 36 insertions(+) diff --git a/archive/archive.go b/archive/archive.go index 838f8f6e7f..78d5aae18c 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -3,6 +3,8 @@ package archive import ( "archive/tar" "bytes" + "compress/gzip" + "compress/bzip2" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -59,6 +61,35 @@ func DetectCompression(source []byte) Compression { return Uncompressed } +func DecompressStream(archive io.Reader) (io.Reader, error) { + buf := make([]byte, 10) + totalN := 0 + for totalN < 10 { + n, err := archive.Read(buf[totalN:]) + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("Tarball too short") + } + return nil, err + } + totalN += n + utils.Debugf("[tar autodetect] n: %d", n) + } + compression := DetectCompression(buf) + wrap := io.MultiReader(bytes.NewReader(buf), archive) + + switch compression { + case Uncompressed: + return wrap, nil + case Gzip: + return gzip.NewReader(wrap) + case Bzip2: + return bzip2.NewReader(wrap), nil + default: + return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) + } +} + func (compression *Compression) Flag() string { switch *compression { case Bzip2: diff --git a/archive/diff.go b/archive/diff.go index 87457f38c1..72a53de02e 100644 --- a/archive/diff.go +++ b/archive/diff.go @@ -34,6 +34,11 @@ func ApplyLayer(dest string, layer Archive) error { oldmask := syscall.Umask(0) defer syscall.Umask(oldmask) + layer, err := DecompressStream(layer) + if err != nil { + return err + } + tr := tar.NewReader(layer) var dirs []*tar.Header From b8a4f570fb31091f43caeba5b824ae38a5bc69e8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 18 Dec 2013 10:50:22 +0100 Subject: [PATCH 2/2] archive: Re-add XZ compression support This shells out to the xz binary to support .tar.xz layers, as there is no compression/xz support in go. --- archive/archive.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/archive/archive.go b/archive/archive.go index 78d5aae18c..ae2c030cb7 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -61,6 +61,12 @@ func DetectCompression(source []byte) Compression { return Uncompressed } +func xzDecompress(archive io.Reader) (io.Reader, error) { + args := []string{"xz", "-d", "-c", "-q"} + + return CmdStream(exec.Command(args[0], args[1:]...), archive, nil) +} + func DecompressStream(archive io.Reader) (io.Reader, error) { buf := make([]byte, 10) totalN := 0 @@ -85,6 +91,8 @@ func DecompressStream(archive io.Reader) (io.Reader, error) { return gzip.NewReader(wrap) case Bzip2: return bzip2.NewReader(wrap), nil + case Xz: + return xzDecompress(wrap) default: return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) } @@ -186,7 +194,7 @@ func TarFilter(path string, options *TarOptions) (io.Reader, error) { } } - return CmdStream(exec.Command(args[0], args[1:]...), &files, func() { + return CmdStream(exec.Command(args[0], args[1:]...), bytes.NewBufferString(files), func() { if tmpDir != "" { _ = os.RemoveAll(tmpDir) } @@ -332,7 +340,7 @@ func CopyFileWithTar(src, dst string) error { // CmdStream executes a command, and returns its stdout as a stream. // If the command fails to run or doesn't complete successfully, an error // will be returned, including anything written on stderr. -func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) { +func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) { if input != nil { stdin, err := cmd.StdinPipe() if err != nil { @@ -343,7 +351,7 @@ func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) { } // Write stdin if any go func() { - _, _ = stdin.Write([]byte(*input)) + io.Copy(stdin, input) stdin.Close() }() }