From f5916b10ae02c7db83052a97205ac345a3d96300 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 16 Dec 2015 15:32:16 -0500 Subject: [PATCH] Remove the graph driver from the daemon, move it into the layer store. Support restoreCustomImage for windows with a new interface to extract the graph driver from the LayerStore. Signed-off-by: Daniel Nephin --- daemon/container_operations_unix.go | 3 +- daemon/daemon.go | 69 +++++++--------- daemon/daemon_unix.go | 3 +- daemon/daemon_windows.go | 121 +++++++++++++++------------- daemon/delete.go | 2 +- daemon/graphdriver/driver.go | 12 +-- daemon/info.go | 4 +- distribution/xfer/download_test.go | 12 +++ docker/daemon.go | 2 +- layer/layer.go | 4 + layer/layer_store.go | 51 +++++++++++- layer/layer_test.go | 4 +- layer/layer_windows.go | 5 ++ layer/migration_test.go | 4 +- 14 files changed, 180 insertions(+), 116 deletions(-) diff --git a/daemon/container_operations_unix.go b/daemon/container_operations_unix.go index bdc80357ef..59de95ff03 100644 --- a/daemon/container_operations_unix.go +++ b/daemon/container_operations_unix.go @@ -289,7 +289,8 @@ func (daemon *Daemon) getSize(container *container.Container) (int64, int64) { sizeRw, err = container.RWLayer.Size() if err != nil { - logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", daemon.driver, container.ID, err) + logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", + daemon.GraphDriverName(), container.ID, err) // FIXME: GetSize should return an error. Not changing it now in case // there is a side-effect. sizeRw = -1 diff --git a/daemon/daemon.go b/daemon/daemon.go index f858e73dd2..2fba5ba2d6 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -30,7 +30,6 @@ import ( "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/daemon/execdriver/execdrivers" - "github.com/docker/docker/daemon/graphdriver" _ "github.com/docker/docker/daemon/graphdriver/vfs" // register vfs "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" @@ -146,7 +145,6 @@ type Daemon struct { idIndex *truncindex.TruncIndex configStore *Config containerGraphDB *graphdb.Database - driver graphdriver.Driver execDriver execdriver.Driver statsCollector *statsCollector defaultLogConfig containertypes.LogConfig @@ -297,7 +295,7 @@ func (daemon *Daemon) restore() error { var ( debug = os.Getenv("DEBUG") != "" - currentDriver = daemon.driver.String() + currentDriver = daemon.GraphDriverName() containers = make(map[string]*cr) ) @@ -520,7 +518,7 @@ func (daemon *Daemon) newContainer(name string, config *containertypes.Config, i base.ImageID = imgID base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName} base.Name = name - base.Driver = daemon.driver.String() + base.Driver = daemon.GraphDriverName() return base, err } @@ -703,20 +701,9 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo } os.Setenv("TMPDIR", realTmp) - // Set the default driver - graphdriver.DefaultDriver = config.GraphDriver - - // Load storage driver - driver, err := graphdriver.New(config.Root, config.GraphOptions, uidMaps, gidMaps) - if err != nil { - return nil, fmt.Errorf("error initializing graphdriver: %v", err) - } - logrus.Debugf("Using graph driver %s", driver) - d := &Daemon{} - d.driver = driver - - // Ensure the graph driver is shutdown at a later point + // Ensure the daemon is properly shutdown if there is a failure during + // initialization defer func() { if err != nil { if err := d.Shutdown(); err != nil { @@ -733,25 +720,32 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo } logrus.Debugf("Using default logging driver %s", config.LogConfig.Type) - // Configure and validate the kernels security support - if err := configureKernelSecuritySupport(config, d.driver.String()); err != nil { - return nil, err - } - daemonRepo := filepath.Join(config.Root, "containers") - if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { return nil, err } - imageRoot := filepath.Join(config.Root, "image", d.driver.String()) - fms, err := layer.NewFSMetadataStore(filepath.Join(imageRoot, "layerdb")) + driverName := os.Getenv("DOCKER_DRIVER") + if driverName == "" { + driverName = config.GraphDriver + } + d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{ + StorePath: config.Root, + MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s"), + GraphDriver: driverName, + GraphDriverOptions: config.GraphOptions, + UIDMaps: uidMaps, + GIDMaps: gidMaps, + }) if err != nil { return nil, err } - d.layerStore, err = layer.NewStore(fms, d.driver) - if err != nil { + graphDriver := d.layerStore.DriverName() + imageRoot := filepath.Join(config.Root, "image", graphDriver) + + // Configure and validate the kernels security support + if err := configureKernelSecuritySupport(config, graphDriver); err != nil { return nil, err } @@ -797,11 +791,11 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err) } - if err := restoreCustomImage(d.driver, d.imageStore, d.layerStore, referenceStore); err != nil { + if err := restoreCustomImage(d.imageStore, d.layerStore, referenceStore); err != nil { return nil, fmt.Errorf("Couldn't restore custom images: %s", err) } - if err := v1.Migrate(config.Root, d.driver.String(), d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil { + if err := v1.Migrate(config.Root, graphDriver, d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil { return nil, err } @@ -953,9 +947,9 @@ func (daemon *Daemon) Shutdown() error { } } - if daemon.driver != nil { - if err := daemon.driver.Cleanup(); err != nil { - logrus.Errorf("Error during graph storage driver.Cleanup(): %v", err) + if daemon.layerStore != nil { + if err := daemon.layerStore.Cleanup(); err != nil { + logrus.Errorf("Error during layer Store.Cleanup(): %v", err) } } @@ -982,7 +976,7 @@ func (daemon *Daemon) Mount(container *container.Container) error { if container.BaseFS != "" && runtime.GOOS != "windows" { daemon.Unmount(container) return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')", - daemon.driver, container.ID, container.BaseFS, dir) + daemon.GraphDriverName(), container.ID, container.BaseFS, dir) } } container.BaseFS = dir // TODO: combine these fields @@ -1193,7 +1187,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { VirtualSize: size, // TODO: field unused, deprecate } - imageInspect.GraphDriver.Name = daemon.driver.String() + imageInspect.GraphDriver.Name = daemon.GraphDriverName() imageInspect.GraphDriver.Data = layerMetadata @@ -1322,10 +1316,9 @@ func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) { return daemon.imageStore.Get(imgID) } -// GraphDriver returns the currently used driver for processing -// container layers. -func (daemon *Daemon) GraphDriver() graphdriver.Driver { - return daemon.driver +// GraphDriverName returns the name of the graph driver used by the layer.Store +func (daemon *Daemon) GraphDriverName() string { + return daemon.layerStore.DriverName() } // ExecutionDriver returns the currently used driver for creating and diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 9552e9769c..a319f9688c 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -15,7 +15,6 @@ import ( pblkiodev "github.com/docker/docker/api/types/blkiodev" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/graphdriver" derr "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -728,7 +727,7 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container daemon.Unmount(container) } -func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, rs reference.Store) error { +func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error { // Unix has no custom images to register return nil } diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 0bf05841e0..a85f14c7e4 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -156,67 +156,78 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container } } -func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, rs reference.Store) error { - if wd, ok := driver.(*windows.Driver); ok { - imageInfos, err := wd.GetCustomImageInfos() +func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error { + type graphDriverStore interface { + GraphDriver() graphdriver.Driver + } + + gds, ok := ls.(graphDriverStore) + if !ok { + return nil + } + + driver := gds.GraphDriver() + wd, ok := driver.(*windows.Driver) + if !ok { + return nil + } + + imageInfos, err := wd.GetCustomImageInfos() + if err != nil { + return err + } + + // Convert imageData to valid image configuration + for i := range imageInfos { + name := strings.ToLower(imageInfos[i].Name) + + type registrar interface { + RegisterDiffID(graphID string, size int64) (layer.Layer, error) + } + r, ok := ls.(registrar) + if !ok { + return errors.New("Layerstore doesn't support RegisterDiffID") + } + if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil { + return err + } + // layer is intentionally not released + + rootFS := image.NewRootFS() + rootFS.BaseLayer = filepath.Base(imageInfos[i].Path) + + // Create history for base layer + config, err := json.Marshal(&image.Image{ + V1Image: image.V1Image{ + DockerVersion: dockerversion.Version, + Architecture: runtime.GOARCH, + OS: runtime.GOOS, + Created: imageInfos[i].CreatedTime, + }, + RootFS: rootFS, + History: []image.History{}, + }) + + named, err := reference.ParseNamed(name) if err != nil { return err } - // Convert imageData to valid image configuration - for i := range imageInfos { - name := strings.ToLower(imageInfos[i].Name) - - type registrar interface { - RegisterDiffID(graphID string, size int64) (layer.Layer, error) - } - r, ok := ls.(registrar) - if !ok { - return errors.New("Layerstore doesn't support RegisterDiffID") - } - if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil { - return err - } - // layer is intentionally not released - - rootFS := image.NewRootFS() - rootFS.BaseLayer = filepath.Base(imageInfos[i].Path) - - // Create history for base layer - config, err := json.Marshal(&image.Image{ - V1Image: image.V1Image{ - DockerVersion: dockerversion.Version, - Architecture: runtime.GOARCH, - OS: runtime.GOOS, - Created: imageInfos[i].CreatedTime, - }, - RootFS: rootFS, - History: []image.History{}, - }) - - named, err := reference.ParseNamed(name) - if err != nil { - return err - } - - ref, err := reference.WithTag(named, imageInfos[i].Version) - if err != nil { - return err - } - - id, err := is.Create(config) - if err != nil { - return err - } - - if err := rs.AddTag(ref, id, true); err != nil { - return err - } - - logrus.Debugf("Registered base layer %s as %s", ref, id) + ref, err := reference.WithTag(named, imageInfos[i].Version) + if err != nil { + return err } - } + id, err := is.Create(config) + if err != nil { + return err + } + if err := rs.AddTag(ref, id, true); err != nil { + return err + } + + logrus.Debugf("Registered base layer %s as %s", ref, id) + } return nil } diff --git a/daemon/delete.go b/daemon/delete.go index e2ce1a8568..dc01433291 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -133,7 +133,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo metadata, err := daemon.layerStore.ReleaseRWLayer(container.RWLayer) layer.LogReleaseMetadata(metadata) if err != nil && err != layer.ErrMountDoesNotExist { - return derr.ErrorCodeRmDriverFS.WithArgs(daemon.driver, container.ID, err) + return derr.ErrorCodeRmDriverFS.WithArgs(daemon.GraphDriverName(), container.ID, err) } if err = daemon.execDriver.Clean(container.ID); err != nil { diff --git a/daemon/graphdriver/driver.go b/daemon/graphdriver/driver.go index 649d0a4b70..d9ab839c2e 100644 --- a/daemon/graphdriver/driver.go +++ b/daemon/graphdriver/driver.go @@ -22,8 +22,6 @@ const ( ) var ( - // DefaultDriver if a storage driver is not specified. - DefaultDriver string // All registered drivers drivers map[string]InitFunc @@ -130,12 +128,10 @@ func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []id } // New creates the driver and initializes it at the specified root. -func New(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (driver Driver, err error) { - for _, name := range []string{os.Getenv("DOCKER_DRIVER"), DefaultDriver} { - if name != "" { - logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver - return GetDriver(name, root, options, uidMaps, gidMaps) - } +func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (driver Driver, err error) { + if name != "" { + logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver + return GetDriver(name, root, options, uidMaps, gidMaps) } // Guess for prior driver diff --git a/daemon/info.go b/daemon/info.go index 76427cc351..7c42ad92b2 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -58,8 +58,8 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { ID: daemon.ID, Containers: len(daemon.List()), Images: len(daemon.imageStore.Map()), - Driver: daemon.GraphDriver().String(), - DriverStatus: daemon.GraphDriver().Status(), + Driver: daemon.GraphDriverName(), + DriverStatus: daemon.layerStore.DriverStatus(), Plugins: daemon.showPluginsInfo(), IPv4Forwarding: !sysInfo.IPv4ForwardingDisabled, BridgeNfIptables: !sysInfo.BridgeNfCallIptablesDisabled, diff --git a/distribution/xfer/download_test.go b/distribution/xfer/download_test.go index 474e523694..5d42703e63 100644 --- a/distribution/xfer/download_test.go +++ b/distribution/xfer/download_test.go @@ -128,6 +128,18 @@ func (ls *mockLayerStore) ReleaseRWLayer(layer.RWLayer) ([]layer.Metadata, error } +func (ls *mockLayerStore) Cleanup() error { + return nil +} + +func (ls *mockLayerStore) DriverStatus() [][2]string { + return [][2]string{} +} + +func (ls *mockLayerStore) DriverName() string { + return "mock" +} + type mockDownloadDescriptor struct { currentDownloads *int32 id string diff --git a/docker/daemon.go b/docker/daemon.go index 6aac30bf19..dd7a938403 100644 --- a/docker/daemon.go +++ b/docker/daemon.go @@ -240,7 +240,7 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { "version": dockerversion.Version, "commit": dockerversion.GitCommit, "execdriver": d.ExecutionDriver().Name(), - "graphdriver": d.GraphDriver().String(), + "graphdriver": d.GraphDriverName(), }).Info("Docker daemon") api.InitRouters(d) diff --git a/layer/layer.go b/layer/layer.go index f2fa5bc6c9..6c4785385d 100644 --- a/layer/layer.go +++ b/layer/layer.go @@ -170,6 +170,10 @@ type Store interface { CreateRWLayer(id string, parent ChainID, mountLabel string, initFunc MountInit) (RWLayer, error) GetRWLayer(id string) (RWLayer, error) ReleaseRWLayer(RWLayer) ([]Metadata, error) + + Cleanup() error + DriverStatus() [][2]string + DriverName() string } // MetadataTransaction represents functions for setting layer metadata diff --git a/layer/layer_store.go b/layer/layer_store.go index 80dedfbfbc..6afdde4de8 100644 --- a/layer/layer_store.go +++ b/layer/layer_store.go @@ -11,6 +11,7 @@ import ( "github.com/docker/distribution/digest" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" @@ -34,11 +35,41 @@ type layerStore struct { mountL sync.Mutex } -// NewStore creates a new Store instance using -// the provided metadata store and graph driver. -// The metadata store will be used to restore +// StoreOptions are the options used to create a new Store instance +type StoreOptions struct { + StorePath string + MetadataStorePathTemplate string + GraphDriver string + GraphDriverOptions []string + UIDMaps []idtools.IDMap + GIDMaps []idtools.IDMap +} + +// NewStoreFromOptions creates a new Store instance +func NewStoreFromOptions(options StoreOptions) (Store, error) { + driver, err := graphdriver.New( + options.StorePath, + options.GraphDriver, + options.GraphDriverOptions, + options.UIDMaps, + options.GIDMaps) + if err != nil { + return nil, fmt.Errorf("error initializing graphdriver: %v", err) + } + logrus.Debugf("Using graph driver %s", driver) + + fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver)) + if err != nil { + return nil, err + } + + return NewStoreFromGraphDriver(fms, driver) +} + +// NewStoreFromGraphDriver creates a new Store instance using the provided +// metadata store and graph driver. The metadata store will be used to restore // the Store. -func NewStore(store MetadataStore, driver graphdriver.Driver) (Store, error) { +func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver) (Store, error) { ls := &layerStore{ store: store, driver: driver, @@ -581,6 +612,18 @@ func (ls *layerStore) assembleTar(graphID string, metadata io.ReadCloser, size * return pR, nil } +func (ls *layerStore) Cleanup() error { + return ls.driver.Cleanup() +} + +func (ls *layerStore) DriverStatus() [][2]string { + return ls.driver.Status() +} + +func (ls *layerStore) DriverName() string { + return ls.driver.String() +} + type naiveDiffPathDriver struct { graphdriver.Driver } diff --git a/layer/layer_test.go b/layer/layer_test.go index 65cc15b2ae..5a96b7c209 100644 --- a/layer/layer_test.go +++ b/layer/layer_test.go @@ -67,7 +67,7 @@ func newTestStore(t *testing.T) (Store, func()) { if err != nil { t.Fatal(err) } - ls, err := NewStore(fms, graph) + ls, err := NewStoreFromGraphDriver(fms, graph) if err != nil { t.Fatal(err) } @@ -398,7 +398,7 @@ func TestStoreRestore(t *testing.T) { assertActivityCount(t, m, 0) - ls2, err := NewStore(ls.(*layerStore).store, ls.(*layerStore).driver) + ls2, err := NewStoreFromGraphDriver(ls.(*layerStore).store, ls.(*layerStore).driver) if err != nil { t.Fatal(err) } diff --git a/layer/layer_windows.go b/layer/layer_windows.go index c40144d860..e6396fa997 100644 --- a/layer/layer_windows.go +++ b/layer/layer_windows.go @@ -6,6 +6,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" + "github.com/docker/docker/daemon/graphdriver" ) // GetLayerPath returns the path to a layer @@ -94,3 +95,7 @@ func (ls *layerStore) mountID(name string) string { // windows has issues if container ID doesn't match mount ID return name } + +func (ls *layerStore) GraphDriver() graphdriver.Driver { + return ls.driver +} diff --git a/layer/migration_test.go b/layer/migration_test.go index 509e0c4eb5..2079bb6e79 100644 --- a/layer/migration_test.go +++ b/layer/migration_test.go @@ -89,7 +89,7 @@ func TestLayerMigration(t *testing.T) { if err != nil { t.Fatal(err) } - ls, err := NewStore(fms, graph) + ls, err := NewStoreFromGraphDriver(fms, graph) if err != nil { t.Fatal(err) } @@ -205,7 +205,7 @@ func TestLayerMigrationNoTarsplit(t *testing.T) { if err != nil { t.Fatal(err) } - ls, err := NewStore(fms, graph) + ls, err := NewStoreFromGraphDriver(fms, graph) if err != nil { t.Fatal(err) }