Merge pull request #24003 from tonistiigi/fix-duplicate-layers

Fix duplicate layers in exported tar

image/tarexport/save.go

@@ -29,6 +29,7 @@ type saveSession struct {
 	outDir      string
 	images      map[image.ID]*imageDescriptor
 	savedLayers map[string]struct{}
+	diffIDPaths map[layer.DiffID]string // cache every diffID blob to avoid duplicates
 }
 
 func (l *tarexporter) Save(names []string, outStream io.Writer) error {
@@ -117,6 +118,7 @@ func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor,
 
 func (s *saveSession) save(outStream io.Writer) error {
 	s.savedLayers = make(map[string]struct{})
+	s.diffIDPaths = make(map[layer.DiffID]string)
 
 	// get image json
 	tempDir, err := ioutil.TempDir("", "docker-export-")
@@ -297,18 +299,27 @@ func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, creat
 	}
 
 	// serialize filesystem
-	tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
+	layerPath := filepath.Join(outDir, legacyLayerFileName)
-	if err != nil {
-		return distribution.Descriptor{}, err
-	}
-	defer tarFile.Close()
-
 	l, err := s.ls.Get(id)
 	if err != nil {
 		return distribution.Descriptor{}, err
 	}
 	defer layer.ReleaseAndLog(s.ls, l)
 
+	if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists {
+		relPath, err := filepath.Rel(layerPath, oldPath)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		os.Symlink(relPath, layerPath)
+	} else {
+
+		tarFile, err := os.Create(layerPath)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		defer tarFile.Close()
+
 		arch, err := l.TarStream()
 		if err != nil {
 			return distribution.Descriptor{}, err
@@ -326,6 +337,8 @@ func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, creat
 			}
 		}
 
+		s.diffIDPaths[l.DiffID()] = layerPath
+	}
 	s.savedLayers[legacyImg.ID] = struct{}{}
 
 	var src distribution.Descriptor