Merge pull request #20475 from Microsoft/jstarks/filegetter

graphdriver: Replace DiffPath with DiffGetter

daemon/graphdriver/aufs/aufs.go

@@ -34,6 +34,7 @@ import (
 	"syscall"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/vbatts/tar-split/tar/storage"
 
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
@@ -374,10 +375,19 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
 	})
 }
 
-// DiffPath returns path to the directory that contains files for the layer
+type fileGetNilCloser struct {
-// differences. Used for direct access for tar-split.
+	storage.FileGetter
-func (a *Driver) DiffPath(id string) (string, func() error, error) {
+}
-	return path.Join(a.rootPath(), "diff", id), func() error { return nil }, nil
+
+func (f fileGetNilCloser) Close() error {
+	return nil
+}
+
+// DiffGetter returns a FileGetCloser that can read files from the directory that
+// contains files for the layer differences. Used for direct access for tar-split.
+func (a *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
+	p := path.Join(a.rootPath(), "diff", id)
+	return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil
 }
 
 func (a *Driver) applyDiff(id string, diff archive.Reader) error {

daemon/graphdriver/driver.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/vbatts/tar-split/tar/storage"
 
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/idtools"
@@ -92,6 +93,23 @@ type Driver interface {
 	DiffSize(id, parent string) (size int64, err error)
 }
 
+// DiffGetterDriver is the interface for layered file system drivers that
+// provide a specialized function for getting file contents for tar-split.
+type DiffGetterDriver interface {
+	Driver
+	// DiffGetter returns an interface to efficiently retrieve the contents
+	// of files in a layer.
+	DiffGetter(id string) (FileGetCloser, error)
+}
+
+// FileGetCloser extends the storage.FileGetter interface with a Close method
+// for cleaning up.
+type FileGetCloser interface {
+	storage.FileGetter
+	// Close cleans up any resources associated with the FileGetCloser.
+	Close() error
+}
+
 func init() {
 	drivers = make(map[string]InitFunc)
 }

daemon/graphdriver/windows/windows.go

@@ -22,6 +22,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/random"
+	"github.com/vbatts/tar-split/tar/storage"
 )
 
 // init registers the windows graph drivers to the register.
@@ -47,6 +48,8 @@ type Driver struct {
 	active map[string]int
 }
 
+var _ graphdriver.DiffGetterDriver = &Driver{}
+
 // InitFilter returns a new Windows storage filter driver.
 func InitFilter(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
 	logrus.Debugf("WindowsGraphDriver InitFilter at %s", home)
@@ -564,8 +567,20 @@ func (d *Driver) setLayerChain(id string, chain []string) error {
 	return nil
 }
 
-// DiffPath returns a directory that contains files needed to construct layer diff.
+type fileGetDestroyCloser struct {
-func (d *Driver) DiffPath(id string) (path string, release func() error, err error) {
+	storage.FileGetter
+	d          *Driver
+	folderName string
+}
+
+func (f *fileGetDestroyCloser) Close() error {
+	// TODO: activate layers and release here?
+	return hcsshim.DestroyLayer(f.d.info, f.folderName)
+}
+
+// DiffGetter returns a FileGetCloser that can read files from the directory that
+// contains files for the layer differences. Used for direct access for tar-split.
+func (d *Driver) DiffGetter(id string) (fg graphdriver.FileGetCloser, err error) {
 	id, err = d.resolveID(id)
 	if err != nil {
 		return
@@ -597,9 +612,6 @@ func (d *Driver) DiffPath(id string) (path string, release func() error, err err
 		return
 	}
 
-	return tempFolder, func() error {
+	_, folderName := filepath.Split(tempFolder)
-		// TODO: activate layers and release here?
+	return &fileGetDestroyCloser{storage.NewPathFileGetter(tempFolder), d, folderName}, nil
-		_, folderName := filepath.Split(tempFolder)
-		return hcsshim.DestroyLayer(d.info, folderName)
-	}, nil
 }

layer/layer_store.go

@@ -577,11 +577,7 @@ func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc Mou
 }
 
 func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error {
-	type diffPathDriver interface {
+	diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver)
-		DiffPath(string) (string, func() error, error)
-	}
-
-	diffDriver, ok := ls.driver.(diffPathDriver)
 	if !ok {
 		diffDriver = &naiveDiffPathDriver{ls.driver}
 	}
@@ -589,17 +585,16 @@ func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size
 	defer metadata.Close()
 
 	// get our relative path to the container
-	fsPath, releasePath, err := diffDriver.DiffPath(graphID)
+	fileGetCloser, err := diffDriver.DiffGetter(graphID)
 	if err != nil {
 		return err
 	}
-	defer releasePath()
+	defer fileGetCloser.Close()
 
 	metaUnpacker := storage.NewJSONUnpacker(metadata)
 	upackerCounter := &unpackSizeCounter{metaUnpacker, size}
-	fileGetter := storage.NewPathFileGetter(fsPath)
+	logrus.Debugf("Assembling tar data for %s", graphID)
-	logrus.Debugf("Assembling tar data for %s from %s", graphID, fsPath)
+	return asm.WriteOutputTarStream(fileGetCloser, upackerCounter, w)
-	return asm.WriteOutputTarStream(fileGetter, upackerCounter, w)
 }
 
 func (ls *layerStore) Cleanup() error {
@@ -618,12 +613,20 @@ type naiveDiffPathDriver struct {
 	graphdriver.Driver
 }
 
-func (n *naiveDiffPathDriver) DiffPath(id string) (string, func() error, error) {
+type fileGetPutter struct {
+	storage.FileGetter
+	driver graphdriver.Driver
+	id     string
+}
+
+func (w *fileGetPutter) Close() error {
+	return w.driver.Put(w.id)
+}
+
+func (n *naiveDiffPathDriver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
 	p, err := n.Driver.Get(id, "")
 	if err != nil {
-		return "", nil, err
+		return nil, err
 	}
-	return p, func() error {
+	return &fileGetPutter{storage.NewPathFileGetter(p), n.Driver, id}, nil
-		return n.Driver.Put(id)
-	}, nil
 }