Merge pull request #16470 from tonistiigi/fix-aufs-opq

Add basic support for .wh..wh..opq

daemon/graphdriver/aufs/aufs.go

@@ -322,7 +322,7 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
 	// AUFS doesn't need the parent layer to produce a diff.
 	return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
 		Compression:     archive.Uncompressed,
-		ExcludePatterns: []string{".wh..wh.*"},
+		ExcludePatterns: []string{archive.WhiteoutMetaPrefix + "*", "!" + archive.WhiteoutOpaqueDir},
 	})
 }
 

pkg/archive/changes.go

@@ -102,7 +102,7 @@ func Changes(layers []string, rw string) ([]Change, error) {
 		}
 
 		// Skip AUFS metadata
-		if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched {
+		if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
 			return err
 		}
 
@@ -113,8 +113,8 @@ func Changes(layers []string, rw string) ([]Change, error) {
 		// Find out what kind of modification happened
 		file := filepath.Base(path)
 		// If there is a whiteout, then the file was removed
-		if strings.HasPrefix(file, ".wh.") {
-			originalFile := file[len(".wh."):]
+		if strings.HasPrefix(file, WhiteoutPrefix) {
+			originalFile := file[len(WhiteoutPrefix):]
 			change.Path = filepath.Join(filepath.Dir(path), originalFile)
 			change.Kind = ChangeDelete
 		} else {
@@ -362,7 +362,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
 			if change.Kind == ChangeDelete {
 				whiteOutDir := filepath.Dir(change.Path)
 				whiteOutBase := filepath.Base(change.Path)
-				whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase)
+				whiteOut := filepath.Join(whiteOutDir, WhiteoutPrefix+whiteOutBase)
 				timestamp := time.Now()
 				hdr := &tar.Header{
 					Name:       whiteOut[1:],

pkg/archive/diff.go

@@ -83,11 +83,11 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) {
 		}
 
 		// Skip AUFS metadata dirs
-		if strings.HasPrefix(hdr.Name, ".wh..wh.") {
+		if strings.HasPrefix(hdr.Name, WhiteoutMetaPrefix) {
 			// Regular files inside /.wh..wh.plnk can be used as hardlink targets
 			// We don't want this directory, but we need the files in them so that
 			// such hardlinks can be resolved.
-			if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg {
+			if strings.HasPrefix(hdr.Name, WhiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
 				basename := filepath.Base(hdr.Name)
 				aufsHardlinks[basename] = hdr
 				if aufsTempdir == "" {
@@ -100,8 +100,11 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) {
 					return 0, err
 				}
 			}
+
+			if hdr.Name != WhiteoutOpaqueDir {
 				continue
 			}
+		}
 		path := filepath.Join(dest, hdr.Name)
 		rel, err := filepath.Rel(dest, path)
 		if err != nil {
@@ -114,12 +117,26 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) {
 		}
 		base := filepath.Base(path)
 
-		if strings.HasPrefix(base, ".wh.") {
-			originalBase := base[len(".wh."):]
-			originalPath := filepath.Join(filepath.Dir(path), originalBase)
+		if strings.HasPrefix(base, WhiteoutPrefix) {
+			dir := filepath.Dir(path)
+			if base == WhiteoutOpaqueDir {
+				fi, err := os.Lstat(dir)
+				if err != nil && !os.IsNotExist(err) {
+					return 0, err
+				}
+				if err := os.RemoveAll(dir); err != nil {
+					return 0, err
+				}
+				if err := os.Mkdir(dir, fi.Mode()&os.ModePerm); err != nil {
+					return 0, err
+				}
+			} else {
+				originalBase := base[len(WhiteoutPrefix):]
+				originalPath := filepath.Join(dir, originalBase)
 				if err := os.RemoveAll(originalPath); err != nil {
 					return 0, err
 				}
+			}
 		} else {
 			// If path exits we almost always just want to remove and replace it.
 			// The only exception is when it is a directory *and* the file from
@@ -139,7 +156,7 @@ func UnpackLayer(dest string, layer Reader) (size int64, err error) {
 
 			// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
 			// we manually retarget these into the temporary files we extracted them into
-			if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") {
+			if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), WhiteoutLinkDir) {
 				linkBasename := filepath.Base(hdr.Linkname)
 				srcHdr = aufsHardlinks[linkBasename]
 				if srcHdr == nil {

pkg/archive/whiteouts.go

@@ -0,0 +1,23 @@
+package archive
+
+// Whiteouts are files with a special meaning for the layered filesystem.
+// Docker uses AUFS whiteout files inside exported archives. In other
+// filesystems these files are generated/handled on tar creation/extraction.
+
+// WhiteoutPrefix prefix means file is a whiteout. If this is followed by a
+// filename this means that file has been removed from the base layer.
+const WhiteoutPrefix = ".wh."
+
+// WhiteoutMetaPrefix prefix means whiteout has a special meaning and is not
+// for remoing an actaul file. Normally these files are excluded from exported
+// archives.
+const WhiteoutMetaPrefix = WhiteoutPrefix + WhiteoutPrefix
+
+// WhiteoutLinkDir is a directory AUFS uses for storing hardlink links to other
+// layers. Normally these should not go into exported archives and all changed
+// hardlinks should be copied to the top layer.
+const WhiteoutLinkDir = WhiteoutMetaPrefix + "plnk"
+
+// WhiteoutOpaqueDir file means directory has been made opaque - meaning
+// readdir calls to this directory do not follow to lower layers.
+const WhiteoutOpaqueDir = WhiteoutMetaPrefix + ".opq"