From 00e3277107a63218b656a36457caf380f9cabf37 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 21 Sep 2015 14:36:35 -0700 Subject: [PATCH 1/2] Add basic support for .wh..wh..opq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the case where directory is removed in aufs and then the same layer is imported to a different graphdriver. Currently when you do `rm -rf /foo && mkdir /foo` in a layer in aufs the files under `foo` would only be be hidden on aufs. The problems with this fix: 1) When a new diff is recreated from non-aufs driver the `opq` files would not be there. This should not mean layer differences for the user but still different content in the tar (one would have one `opq` file, the others would have `.wh.*` for every file inside that folder). This difference also only happens if the tar-split file isn’t stored for the layer. 2) New files that have the filenames before `.wh..wh..opq` when they are sorted do not get picked up by non-aufs graphdrivers. Fixing this would require a bigger refactoring that is planned in the future. Signed-off-by: Tonis Tiigi --- daemon/graphdriver/aufs/aufs.go | 2 +- pkg/archive/diff.go | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index f356164e7b..5ac905b867 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -322,7 +322,7 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, - ExcludePatterns: []string{".wh..wh.*"}, + ExcludePatterns: []string{".wh..wh.*", "!.wh..wh..opq"}, }) } diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index 50656cb5fd..c343d8af91 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -100,7 +100,10 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { return 0, err } } - continue + + if hdr.Name != ".wh..wh..opq" { + continue + } } path := filepath.Join(dest, hdr.Name) rel, err := filepath.Rel(dest, path) @@ -116,9 +119,23 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { if strings.HasPrefix(base, ".wh.") { originalBase := base[len(".wh."):] - originalPath := filepath.Join(filepath.Dir(path), originalBase) - if err := os.RemoveAll(originalPath); err != nil { - return 0, err + dir := filepath.Dir(path) + if originalBase == ".wh..opq" { + fi, err := os.Lstat(dir) + if err != nil && !os.IsNotExist(err) { + return 0, err + } + if err := os.RemoveAll(dir); err != nil { + return 0, err + } + if err := os.Mkdir(dir, fi.Mode()&os.ModePerm); err != nil { + return 0, err + } + } else { + originalPath := filepath.Join(dir, originalBase) + if err := os.RemoveAll(originalPath); err != nil { + return 0, err + } } } else { // If path exits we almost always just want to remove and replace it. From 2fb5d0c32376951ef41a6f64bb7dbd8f6fd14fba Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 29 Sep 2015 10:18:28 -0700 Subject: [PATCH 2/2] Add constants for AUFS whiteout files Signed-off-by: Tonis Tiigi --- daemon/graphdriver/aufs/aufs.go | 2 +- pkg/archive/changes.go | 8 ++++---- pkg/archive/diff.go | 14 +++++++------- pkg/archive/whiteouts.go | 23 +++++++++++++++++++++++ 4 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 pkg/archive/whiteouts.go diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index 5ac905b867..3b89794007 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -322,7 +322,7 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, - ExcludePatterns: []string{".wh..wh.*", "!.wh..wh..opq"}, + ExcludePatterns: []string{archive.WhiteoutMetaPrefix + "*", "!" + archive.WhiteoutOpaqueDir}, }) } diff --git a/pkg/archive/changes.go b/pkg/archive/changes.go index 76c36b5215..c745362c55 100644 --- a/pkg/archive/changes.go +++ b/pkg/archive/changes.go @@ -102,7 +102,7 @@ func Changes(layers []string, rw string) ([]Change, error) { } // Skip AUFS metadata - if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched { + if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched { return err } @@ -113,8 +113,8 @@ func Changes(layers []string, rw string) ([]Change, error) { // Find out what kind of modification happened file := filepath.Base(path) // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, ".wh.") { - originalFile := file[len(".wh."):] + if strings.HasPrefix(file, WhiteoutPrefix) { + originalFile := file[len(WhiteoutPrefix):] change.Path = filepath.Join(filepath.Dir(path), originalFile) change.Kind = ChangeDelete } else { @@ -362,7 +362,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { if change.Kind == ChangeDelete { whiteOutDir := filepath.Dir(change.Path) whiteOutBase := filepath.Base(change.Path) - whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase) + whiteOut := filepath.Join(whiteOutDir, WhiteoutPrefix+whiteOutBase) timestamp := time.Now() hdr := &tar.Header{ Name: whiteOut[1:], diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index c343d8af91..c5aa4d79f3 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -83,11 +83,11 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { } // Skip AUFS metadata dirs - if strings.HasPrefix(hdr.Name, ".wh..wh.") { + if strings.HasPrefix(hdr.Name, WhiteoutMetaPrefix) { // Regular files inside /.wh..wh.plnk can be used as hardlink targets // We don't want this directory, but we need the files in them so that // such hardlinks can be resolved. - if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg { + if strings.HasPrefix(hdr.Name, WhiteoutLinkDir) && hdr.Typeflag == tar.TypeReg { basename := filepath.Base(hdr.Name) aufsHardlinks[basename] = hdr if aufsTempdir == "" { @@ -101,7 +101,7 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { } } - if hdr.Name != ".wh..wh..opq" { + if hdr.Name != WhiteoutOpaqueDir { continue } } @@ -117,10 +117,9 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { } base := filepath.Base(path) - if strings.HasPrefix(base, ".wh.") { - originalBase := base[len(".wh."):] + if strings.HasPrefix(base, WhiteoutPrefix) { dir := filepath.Dir(path) - if originalBase == ".wh..opq" { + if base == WhiteoutOpaqueDir { fi, err := os.Lstat(dir) if err != nil && !os.IsNotExist(err) { return 0, err @@ -132,6 +131,7 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { return 0, err } } else { + originalBase := base[len(WhiteoutPrefix):] originalPath := filepath.Join(dir, originalBase) if err := os.RemoveAll(originalPath); err != nil { return 0, err @@ -156,7 +156,7 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) { // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so // we manually retarget these into the temporary files we extracted them into - if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") { + if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), WhiteoutLinkDir) { linkBasename := filepath.Base(hdr.Linkname) srcHdr = aufsHardlinks[linkBasename] if srcHdr == nil { diff --git a/pkg/archive/whiteouts.go b/pkg/archive/whiteouts.go new file mode 100644 index 0000000000..3d9c313213 --- /dev/null +++ b/pkg/archive/whiteouts.go @@ -0,0 +1,23 @@ +package archive + +// Whiteouts are files with a special meaning for the layered filesystem. +// Docker uses AUFS whiteout files inside exported archives. In other +// filesystems these files are generated/handled on tar creation/extraction. + +// WhiteoutPrefix prefix means file is a whiteout. If this is followed by a +// filename this means that file has been removed from the base layer. +const WhiteoutPrefix = ".wh." + +// WhiteoutMetaPrefix prefix means whiteout has a special meaning and is not +// for remoing an actaul file. Normally these files are excluded from exported +// archives. +const WhiteoutMetaPrefix = WhiteoutPrefix + WhiteoutPrefix + +// WhiteoutLinkDir is a directory AUFS uses for storing hardlink links to other +// layers. Normally these should not go into exported archives and all changed +// hardlinks should be copied to the top layer. +const WhiteoutLinkDir = WhiteoutMetaPrefix + "plnk" + +// WhiteoutOpaqueDir file means directory has been made opaque - meaning +// readdir calls to this directory do not follow to lower layers. +const WhiteoutOpaqueDir = WhiteoutMetaPrefix + ".opq"