From 58bec40d16265362fd4e41dbd652e6fba903794d Mon Sep 17 00:00:00 2001 From: John Starks Date: Thu, 18 Feb 2016 17:58:23 -0800 Subject: [PATCH] graphdriver: Replace DiffPath with DiffGetter This allows a graph driver to provide a custom FileGetter for tar-split to use. Windows will use this to provide a more efficient implementation in a follow-up change. Signed-off-by: John Starks --- daemon/graphdriver/aufs/aufs.go | 18 +++++++++++---- daemon/graphdriver/driver.go | 18 +++++++++++++++ daemon/graphdriver/windows/windows.go | 26 +++++++++++++++------ layer/layer_store.go | 33 +++++++++++++++------------ 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index 51054fa6ef..07dba9e0fe 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -34,6 +34,7 @@ import ( "syscall" "github.com/Sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -367,10 +368,19 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { }) } -// DiffPath returns path to the directory that contains files for the layer -// differences. Used for direct access for tar-split. -func (a *Driver) DiffPath(id string) (string, func() error, error) { - return path.Join(a.rootPath(), "diff", id), func() error { return nil }, nil +type fileGetNilCloser struct { + storage.FileGetter +} + +func (f fileGetNilCloser) Close() error { + return nil +} + +// DiffGetter returns a FileGetCloser that can read files from the directory that +// contains files for the layer differences. Used for direct access for tar-split. +func (a *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { + p := path.Join(a.rootPath(), "diff", id) + return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } func (a *Driver) applyDiff(id string, diff archive.Reader) error { diff --git a/daemon/graphdriver/driver.go b/daemon/graphdriver/driver.go index d9ab839c2e..abc400083d 100644 --- a/daemon/graphdriver/driver.go +++ b/daemon/graphdriver/driver.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" @@ -92,6 +93,23 @@ type Driver interface { DiffSize(id, parent string) (size int64, err error) } +// DiffGetterDriver is the interface for layered file system drivers that +// provide a specialized function for getting file contents for tar-split. +type DiffGetterDriver interface { + Driver + // DiffGetter returns an interface to efficiently retrieve the contents + // of files in a layer. + DiffGetter(id string) (FileGetCloser, error) +} + +// FileGetCloser extends the storage.FileGetter interface with a Close method +// for cleaning up. +type FileGetCloser interface { + storage.FileGetter + // Close cleans up any resources associated with the FileGetCloser. + Close() error +} + func init() { drivers = make(map[string]InitFunc) } diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go index 77f4f1b774..58af3e5e04 100644 --- a/daemon/graphdriver/windows/windows.go +++ b/daemon/graphdriver/windows/windows.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/random" + "github.com/vbatts/tar-split/tar/storage" ) // init registers the windows graph drivers to the register. @@ -47,6 +48,8 @@ type Driver struct { active map[string]int } +var _ graphdriver.DiffGetterDriver = &Driver{} + // InitFilter returns a new Windows storage filter driver. func InitFilter(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { logrus.Debugf("WindowsGraphDriver InitFilter at %s", home) @@ -564,8 +567,20 @@ func (d *Driver) setLayerChain(id string, chain []string) error { return nil } -// DiffPath returns a directory that contains files needed to construct layer diff. -func (d *Driver) DiffPath(id string) (path string, release func() error, err error) { +type fileGetDestroyCloser struct { + storage.FileGetter + d *Driver + folderName string +} + +func (f *fileGetDestroyCloser) Close() error { + // TODO: activate layers and release here? + return hcsshim.DestroyLayer(f.d.info, f.folderName) +} + +// DiffGetter returns a FileGetCloser that can read files from the directory that +// contains files for the layer differences. Used for direct access for tar-split. +func (d *Driver) DiffGetter(id string) (fg graphdriver.FileGetCloser, err error) { id, err = d.resolveID(id) if err != nil { return @@ -597,9 +612,6 @@ func (d *Driver) DiffPath(id string) (path string, release func() error, err err return } - return tempFolder, func() error { - // TODO: activate layers and release here? - _, folderName := filepath.Split(tempFolder) - return hcsshim.DestroyLayer(d.info, folderName) - }, nil + _, folderName := filepath.Split(tempFolder) + return &fileGetDestroyCloser{storage.NewPathFileGetter(tempFolder), d, folderName}, nil } diff --git a/layer/layer_store.go b/layer/layer_store.go index 229ba6a3a2..4b01ea0fc0 100644 --- a/layer/layer_store.go +++ b/layer/layer_store.go @@ -577,11 +577,7 @@ func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc Mou } func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error { - type diffPathDriver interface { - DiffPath(string) (string, func() error, error) - } - - diffDriver, ok := ls.driver.(diffPathDriver) + diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver) if !ok { diffDriver = &naiveDiffPathDriver{ls.driver} } @@ -589,17 +585,16 @@ func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size defer metadata.Close() // get our relative path to the container - fsPath, releasePath, err := diffDriver.DiffPath(graphID) + fileGetCloser, err := diffDriver.DiffGetter(graphID) if err != nil { return err } - defer releasePath() + defer fileGetCloser.Close() metaUnpacker := storage.NewJSONUnpacker(metadata) upackerCounter := &unpackSizeCounter{metaUnpacker, size} - fileGetter := storage.NewPathFileGetter(fsPath) - logrus.Debugf("Assembling tar data for %s from %s", graphID, fsPath) - return asm.WriteOutputTarStream(fileGetter, upackerCounter, w) + logrus.Debugf("Assembling tar data for %s", graphID) + return asm.WriteOutputTarStream(fileGetCloser, upackerCounter, w) } func (ls *layerStore) Cleanup() error { @@ -618,12 +613,20 @@ type naiveDiffPathDriver struct { graphdriver.Driver } -func (n *naiveDiffPathDriver) DiffPath(id string) (string, func() error, error) { +type fileGetPutter struct { + storage.FileGetter + driver graphdriver.Driver + id string +} + +func (w *fileGetPutter) Close() error { + return w.driver.Put(w.id) +} + +func (n *naiveDiffPathDriver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { p, err := n.Driver.Get(id, "") if err != nil { - return "", nil, err + return nil, err } - return p, func() error { - return n.Driver.Put(id) - }, nil + return &fileGetPutter{storage.NewPathFileGetter(p), n.Driver, id}, nil }