Merge pull request #24003 from tonistiigi/fix-duplicate-layers

Fix duplicate layers in exported tar

image/tarexport/save.go

@@ -29,6 +29,7 @@ type saveSession struct {
 	outDir      string
 	images      map[image.ID]*imageDescriptor
 	savedLayers map[string]struct{}
+	diffIDPaths map[layer.DiffID]string // cache every diffID blob to avoid duplicates
 }
 
 func (l *tarexporter) Save(names []string, outStream io.Writer) error {
@@ -117,6 +118,7 @@ func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor,
 
 func (s *saveSession) save(outStream io.Writer) error {
 	s.savedLayers = make(map[string]struct{})
+	s.diffIDPaths = make(map[layer.DiffID]string)
 
 	// get image json
 	tempDir, err := ioutil.TempDir("", "docker-export-")
@@ -297,35 +299,46 @@ func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, creat
 	}
 
 	// serialize filesystem
-	tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
-	if err != nil {
-		return distribution.Descriptor{}, err
-	}
-	defer tarFile.Close()
-
+	layerPath := filepath.Join(outDir, legacyLayerFileName)
 	l, err := s.ls.Get(id)
 	if err != nil {
 		return distribution.Descriptor{}, err
 	}
 	defer layer.ReleaseAndLog(s.ls, l)
 
-	arch, err := l.TarStream()
-	if err != nil {
-		return distribution.Descriptor{}, err
-	}
-	defer arch.Close()
-
-	if _, err := io.Copy(tarFile, arch); err != nil {
-		return distribution.Descriptor{}, err
-	}
-
-	for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
-		// todo: maybe save layer created timestamp?
-		if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
+	if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists {
+		relPath, err := filepath.Rel(layerPath, oldPath)
+		if err != nil {
 			return distribution.Descriptor{}, err
 		}
-	}
+		os.Symlink(relPath, layerPath)
+	} else {
 
+		tarFile, err := os.Create(layerPath)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		defer tarFile.Close()
+
+		arch, err := l.TarStream()
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		defer arch.Close()
+
+		if _, err := io.Copy(tarFile, arch); err != nil {
+			return distribution.Descriptor{}, err
+		}
+
+		for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
+			// todo: maybe save layer created timestamp?
+			if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
+				return distribution.Descriptor{}, err
+			}
+		}
+
+		s.diffIDPaths[l.DiffID()] = layerPath
+	}
 	s.savedLayers[legacyImg.ID] = struct{}{}
 
 	var src distribution.Descriptor