kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

remove layerstore indexing by OS (used for LCOW)

Browse source 
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn 2021-03-19 15:34:08 +01:00
parent b7b7571331
commit dc7cbb9b33
No known key found for this signature in database
GPG key ID: 76698F39D527CE8C
27 changed files with 124 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
daemon/container.go
View file

@ -158,7 +158,7 @@ func (daemon *Daemon) newContainer(name string, operatingSystem string, config *
base.ImageID = imgID
base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName}
base.Name = name
base.Driver = daemon.imageService.GraphDriverForOS(operatingSystem)
base.Driver = daemon.imageService.GraphDriverName()
base.OS = operatingSystem
return base, err
}

13
daemon/daemon.go
View file

@ -250,7 +250,7 @@ func (daemon *Daemon) restore() error {
}
// Ignore the container if it does not support the current driver being used by the graph
if (c.Driver == "" && daemon.graphDriver == "aufs") || c.Driver == daemon.graphDriver {
rwlayer, err := daemon.imageService.GetLayerByID(c.ID, c.OS)
rwlayer, err := daemon.imageService.GetLayerByID(c.ID)
if err != nil {
log.WithError(err).Error("failed to load container mount")
return
@ -1003,10 +1003,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
return nil, err
}
// TODO remove multiple imagestores map now that LCOW is no more
imageStore, err := image.NewImageStore(ifs, map[string]image.LayerGetReleaser{
runtime.GOOS: layerStore,
})
imageStore, err := image.NewImageStore(ifs, layerStore)
if err != nil {
return nil, err
}
@ -1084,7 +1081,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
DistributionMetadataStore: distributionMetadataStore,
EventsService: d.EventsService,
ImageStore: imageStore,
LayerStores: map[string]layer.Store{runtime.GOOS: layerStore}, // TODO remove multiple LayerStores map now that LCOW is no more
LayerStore: layerStore,
MaxConcurrentDownloads: *config.MaxConcurrentDownloads,
MaxConcurrentUploads: *config.MaxConcurrentUploads,
MaxDownloadAttempts: *config.MaxDownloadAttempts,
@ -1231,7 +1228,7 @@ func (daemon *Daemon) Shutdown() error {
log.WithError(err).Error("failed to shut down container")
return
}
if mountid, err := daemon.imageService.GetLayerMountID(c.ID, c.OS); err == nil {
if mountid, err := daemon.imageService.GetLayerMountID(c.ID); err == nil {
daemon.cleanupMountsByID(mountid)
}
log.Debugf("shut down container")
@ -1294,7 +1291,7 @@ func (daemon *Daemon) Mount(container *container.Container) error {
if runtime.GOOS != "windows" {
daemon.Unmount(container)
return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
daemon.imageService.GraphDriverForOS(container.OS), container.ID, container.BaseFS, dir)
daemon.imageService.GraphDriverName(), container.ID, container.BaseFS, dir)
}
}
container.BaseFS = dir // TODO: combine these fields

2
daemon/export.go
View file

@ -50,7 +50,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R
if !system.IsOSSupported(container.OS) {
return nil, fmt.Errorf("cannot export %s: %s ", container.ID, system.ErrNotSupportedOperatingSystem)
}
rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS)
rwlayer, err := daemon.imageService.GetLayerByID(container.ID)
if err != nil {
return nil, err
}

6
daemon/images/image_builder.go
View file

@ -206,7 +206,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s
if !system.IsOSSupported(os) {
return nil, nil, system.ErrNotSupportedOperatingSystem
}
layer, err := newROLayerForImage(nil, i.layerStores[os])
layer, err := newROLayerForImage(nil, i.layerStore)
return nil, layer, err
}
@ -220,7 +220,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s
if !system.IsOSSupported(image.OperatingSystem()) {
return nil, nil, system.ErrNotSupportedOperatingSystem
}
layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()])
layer, err := newROLayerForImage(image, i.layerStore)
return image, layer, err
}
}
@ -232,7 +232,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s
if !system.IsOSSupported(image.OperatingSystem()) {
return nil, nil, system.ErrNotSupportedOperatingSystem
}
layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()])
layer, err := newROLayerForImage(image, i.layerStore)
return image, layer, err
}

11
daemon/images/image_commit.go
View file

@ -8,17 +8,12 @@ import (
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
// CommitImage creates a new image from a commit config
func (i *ImageService) CommitImage(c backend.CommitConfig) (image.ID, error) {
layerStore, ok := i.layerStores[c.ContainerOS]
if !ok {
return "", system.ErrNotSupportedOperatingSystem
}
rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
rwTar, err := exportContainerRw(i.layerStore, c.ContainerID, c.ContainerMountLabel)
if err != nil {
return "", err
}
@ -39,11 +34,11 @@ func (i *ImageService) CommitImage(c backend.CommitConfig) (image.ID, error) {
}
}
l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
l, err := i.layerStore.Register(rwTar, parent.RootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(layerStore, l)
defer layer.ReleaseAndLog(i.layerStore, l)
cc := image.ChildConfig{
ContainerID: c.ContainerID,

4
daemon/images/image_exporter.go
View file

@ -12,7 +12,7 @@ import (
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
func (i *ImageService) ExportImage(names []string, outStream io.Writer) error {
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i)
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStore, i.referenceStore, i)
return imageExporter.Save(names, outStream)
}
@ -20,6 +20,6 @@ func (i *ImageService) ExportImage(names []string, outStream io.Writer) error {
// complement of ExportImage. The input stream is an uncompressed tar
// ball containing images and metadata.
func (i *ImageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i)
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStore, i.referenceStore, i)
return imageExporter.Load(inTar, outStream, quiet)
}

4
daemon/images/image_history.go
View file

@ -36,12 +36,12 @@ func (i *ImageService) ImageHistory(name string) ([]*image.HistoryResponseItem,
return nil, system.ErrNotSupportedOperatingSystem
}
rootFS.Append(img.RootFS.DiffIDs[layerCounter])
l, err := i.layerStores[img.OperatingSystem()].Get(rootFS.ChainID())
l, err := i.layerStore.Get(rootFS.ChainID())
if err != nil {
return nil, err
}
layerSize, err = l.DiffSize()
layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l)
layer.ReleaseAndLog(i.layerStore, l)
if err != nil {
return nil, err
}

4
daemon/images/image_import.go
View file

@ -91,11 +91,11 @@ func (i *ImageService) ImportImage(src string, repository, os string, tag string
if err != nil {
return err
}
l, err := i.layerStores[os].Register(inflatedLayerData, "")
l, err := i.layerStore.Register(inflatedLayerData, "")
if err != nil {
return err
}
defer layer.ReleaseAndLog(i.layerStores[os], l)
defer layer.ReleaseAndLog(i.layerStore, l)
created := time.Now().UTC()
imgConfig, err := json.Marshal(&image.Image{

6
daemon/images/image_inspect.go
View file

@ -37,11 +37,11 @@ func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) {
var layerMetadata map[string]string
layerID := img.RootFS.ChainID()
if layerID != "" {
l, err := i.layerStores[img.OperatingSystem()].Get(layerID)
l, err := i.layerStore.Get(layerID)
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l)
defer layer.ReleaseAndLog(i.layerStore, l)
size, err = l.Size()
if err != nil {
return nil, err
@ -87,7 +87,7 @@ func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) {
},
}
imageInspect.GraphDriver.Name = i.layerStores[img.OperatingSystem()].DriverName()
imageInspect.GraphDriver.Name = i.layerStore.DriverName()
imageInspect.GraphDriver.Data = layerMetadata
return imageInspect, nil

7
daemon/images/image_prune.go
View file

@ -68,12 +68,7 @@ func (i *ImageService) ImagesPrune(ctx context.Context, pruneFilters filters.Arg
}
// Filter intermediary images and get their unique size
allLayers := make(map[layer.ChainID]layer.Layer)
for _, ls := range i.layerStores {
for k, v := range ls.Map() {
allLayers[k] = v
}
}
allLayers := i.layerStore.Map()
topImages := map[image.ID]*image.Image{}
for id, img := range allImages {
select {

2
daemon/images/image_push.go
View file

@ -53,7 +53,7 @@ func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHea
ReferenceStore: i.referenceStore,
},
ConfigMediaType: schema2.MediaTypeImageConfig,
LayerStores: distribution.NewLayerProvidersFromStores(i.layerStores),
LayerStores: distribution.NewLayerProvidersFromStore(i.layerStore),
TrustKey: i.trustKey,
UploadManager: i.uploadManager,
}

8
daemon/images/image_unix.go
View file

@ -3,8 +3,6 @@
package images // import "github.com/docker/docker/daemon/images"
import (
"runtime"
"github.com/sirupsen/logrus"
)
@ -17,17 +15,17 @@ func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64)
// Safe to index by runtime.GOOS as Unix hosts don't support multiple
// container operating systems.
rwlayer, err := i.layerStores[runtime.GOOS].GetRWLayer(containerID)
rwlayer, err := i.layerStore.GetRWLayer(containerID)
if err != nil {
logrus.Errorf("Failed to compute size of container rootfs %v: %v", containerID, err)
return sizeRw, sizeRootfs
}
defer i.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer)
defer i.layerStore.ReleaseRWLayer(rwlayer)
sizeRw, err = rwlayer.Size()
if err != nil {
logrus.Errorf("Driver %s couldn't return diff size of container %s: %s",
i.layerStores[runtime.GOOS].DriverName(), containerID, err)
i.layerStore.DriverName(), containerID, err)
// FIXME: GetSize should return an error. Not changing it now in case
// there is a side-effect.
sizeRw = -1

4
daemon/images/image_windows.go
View file

@ -23,9 +23,9 @@ func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer)
if !system.IsOSSupported(img.OperatingSystem()) {
return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID())
}
layerPath, err := layer.GetLayerPath(i.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
layerPath, err := layer.GetLayerPath(i.layerStore, img.RootFS.ChainID())
if err != nil {
return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", i.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", i.layerStore, img.RootFS.ChainID())
}
// Reverse order, expecting parent first
folders = append([]string{layerPath}, folders...)

22
daemon/images/images.go
View file

@ -124,7 +124,7 @@ func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttr
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := i.layerStores[img.OperatingSystem()].Get(layerID)
l, err := i.layerStore.Get(layerID)
if err != nil {
// The layer may have been deleted between the call to `Map()` or
// `Heads()` and the call to `Get()`, so we just ignore this error
@ -135,7 +135,7 @@ func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttr
}
size, err = l.Size()
layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l)
layer.ReleaseAndLog(i.layerStore, l)
if err != nil {
return nil, err
}
@ -190,15 +190,7 @@ func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttr
// lazily init variables
if imagesMap == nil {
allContainers = i.containers.List()
// allLayers is built from all layerstores combined
allLayers = make(map[layer.ChainID]layer.Layer)
for _, ls := range i.layerStores {
layers := ls.Map()
for k, v := range layers {
allLayers[k] = v
}
}
allLayers = i.layerStore.Map()
imagesMap = make(map[*image.Image]*types.ImageSummary)
layerRefs = make(map[layer.ChainID]int)
}
@ -285,11 +277,11 @@ func (i *ImageService) SquashImage(id, parent string) (string, error) {
if !system.IsOSSupported(img.OperatingSystem()) {
return "", errors.Wrap(err, system.ErrNotSupportedOperatingSystem.Error())
}
l, err := i.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID())
l, err := i.layerStore.Get(img.RootFS.ChainID())
if err != nil {
return "", errors.Wrap(err, "error getting image layer")
}
defer i.layerStores[img.OperatingSystem()].Release(l)
defer i.layerStore.Release(l)
ts, err := l.TarStreamFrom(parentChainID)
if err != nil {
@ -297,11 +289,11 @@ func (i *ImageService) SquashImage(id, parent string) (string, error) {
}
defer ts.Close()
newL, err := i.layerStores[img.OperatingSystem()].Register(ts, parentChainID)
newL, err := i.layerStore.Register(ts, parentChainID)
if err != nil {
return "", errors.Wrap(err, "error registering layer")
}
defer i.layerStores[img.OperatingSystem()].Release(newL)
defer i.layerStore.Release(newL)
newImage := *img
newImage.RootFS = nil

80
daemon/images/service.go
View file

@ -3,7 +3,6 @@ package images // import "github.com/docker/docker/daemon/images"
import (
"context"
"os"
"runtime"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/leases"
@ -37,7 +36,7 @@ type ImageServiceConfig struct {
DistributionMetadataStore metadata.Store
EventsService *daemonevents.Events
ImageStore image.Store
LayerStores map[string]layer.Store
LayerStore layer.Store
MaxConcurrentDownloads int
MaxConcurrentUploads int
MaxDownloadAttempts int
@ -57,10 +56,10 @@ func NewImageService(config ImageServiceConfig) *ImageService {
return &ImageService{
containers: config.ContainerStore,
distributionMetadataStore: config.DistributionMetadataStore,
downloadManager: xfer.NewLayerDownloadManager(config.LayerStores, config.MaxConcurrentDownloads, xfer.WithMaxDownloadAttempts(config.MaxDownloadAttempts)),
downloadManager: xfer.NewLayerDownloadManager(config.LayerStore, config.MaxConcurrentDownloads, xfer.WithMaxDownloadAttempts(config.MaxDownloadAttempts)),
eventsService: config.EventsService,
imageStore: &imageStoreWithLease{Store: config.ImageStore, leases: config.Leases, ns: config.ContentNamespace},
layerStores: config.LayerStores,
layerStore: config.LayerStore,
referenceStore: config.ReferenceStore,
registryService: config.RegistryService,
trustKey: config.TrustKey,
@ -78,7 +77,7 @@ type ImageService struct {
downloadManager *xfer.LayerDownloadManager
eventsService *daemonevents.Events
imageStore image.Store
layerStores map[string]layer.Store // By operating system
layerStore layer.Store
pruneRunning int32
referenceStore dockerreference.Store
registryService registry.Service
@ -93,7 +92,7 @@ type ImageService struct {
type DistributionServices struct {
DownloadManager distribution.RootFSDownloadManager
V2MetadataService metadata.V2MetadataService
LayerStore layer.Store // TODO: lcow
LayerStore layer.Store
ImageStore image.Store
ReferenceStore dockerreference.Store
}
@ -103,7 +102,7 @@ func (i *ImageService) DistributionServices() DistributionServices {
return DistributionServices{
DownloadManager: i.downloadManager,
V2MetadataService: metadata.NewV2MetadataService(i.distributionMetadataStore),
LayerStore: i.layerStores[runtime.GOOS],
LayerStore: i.layerStore,
ImageStore: i.imageStore,
ReferenceStore: i.referenceStore,
}
@ -143,61 +142,52 @@ func (i *ImageService) CreateLayer(container *container.Container, initFunc laye
// Indexing by OS is safe here as validation of OS has already been performed in create() (the only
// caller), and guaranteed non-nil
return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts)
return i.layerStore.CreateRWLayer(container.ID, layerID, rwLayerOpts)
}
// GetLayerByID returns a layer by ID and operating system
// GetLayerByID returns a layer by ID
// called from daemon.go Daemon.restore(), and Daemon.containerExport()
func (i *ImageService) GetLayerByID(cid string, os string) (layer.RWLayer, error) {
return i.layerStores[os].GetRWLayer(cid)
func (i *ImageService) GetLayerByID(cid string) (layer.RWLayer, error) {
return i.layerStore.GetRWLayer(cid)
}
// LayerStoreStatus returns the status for each layer store
// called from info.go
func (i *ImageService) LayerStoreStatus() map[string][][2]string {
result := make(map[string][][2]string)
for os, store := range i.layerStores {
result[os] = store.DriverStatus()
}
return result
func (i *ImageService) LayerStoreStatus() [][2]string {
return i.layerStore.DriverStatus()
}
// GetLayerMountID returns the mount ID for a layer
// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup)
// TODO: needs to be refactored to Unmount (see callers), or removed and replaced
// with GetLayerByID
func (i *ImageService) GetLayerMountID(cid string, os string) (string, error) {
return i.layerStores[os].GetMountID(cid)
// TODO: needs to be refactored to Unmount (see callers), or removed and replaced with GetLayerByID
func (i *ImageService) GetLayerMountID(cid string) (string, error) {
return i.layerStore.GetMountID(cid)
}
// Cleanup resources before the process is shutdown.
// called from daemon.go Daemon.Shutdown()
func (i *ImageService) Cleanup() {
for os, ls := range i.layerStores {
if ls != nil {
if err := ls.Cleanup(); err != nil {
logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os)
}
}
if err := i.layerStore.Cleanup(); err != nil {
logrus.Errorf("Error during layer Store.Cleanup(): %v", err)
}
}
// GraphDriverForOS returns the name of the graph drvier
// GraphDriverName returns the name of the graph drvier
// moved from Daemon.GraphDriverName, used by:
// - newContainer
// - to report an error in Daemon.Mount(container)
func (i *ImageService) GraphDriverForOS(os string) string {
return i.layerStores[os].DriverName()
func (i *ImageService) GraphDriverName() string {
return i.layerStore.DriverName()
}
// ReleaseLayer releases a layer allowing it to be removed
// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport()
func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer, containerOS string) error {
metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer)
metadata, err := i.layerStore.ReleaseRWLayer(rwlayer)
layer.LogReleaseMetadata(metadata)
if err != nil && !errors.Is(err, layer.ErrMountDoesNotExist) && !errors.Is(err, os.ErrNotExist) {
return errors.Wrapf(err, "driver %q failed to remove root filesystem",
i.layerStores[containerOS].DriverName())
i.layerStore.DriverName())
}
return nil
}
@ -207,21 +197,19 @@ func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer, containerOS string) e
func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) {
var allLayersSize int64
layerRefs := i.getLayerRefs()
for _, ls := range i.layerStores {
allLayers := ls.Map()
for _, l := range allLayers {
select {
case <-ctx.Done():
return allLayersSize, ctx.Err()
default:
size, err := l.DiffSize()
if err == nil {
if _, ok := layerRefs[l.ChainID()]; ok {
allLayersSize += size
}
} else {
logrus.Warnf("failed to get diff size for layer %v", l.ChainID())
allLayers := i.layerStore.Map()
for _, l := range allLayers {
select {
case <-ctx.Done():
return allLayersSize, ctx.Err()
default:
size, err := l.DiffSize()
if err == nil {
if _, ok := layerRefs[l.ChainID()]; ok {
allLayersSize += size
}
} else {
logrus.Warnf("failed to get diff size for layer %v", l.ChainID())
}
}
}

3
daemon/info.go
View file

@ -148,9 +148,8 @@ func (daemon *Daemon) fillDriverInfo(v *types.Info) {
v.Warnings = append(v.Warnings, fmt.Sprintf("WARNING: the %s storage-driver is deprecated, and will be removed in a future release.", daemon.graphDriver))
}
statuses := daemon.imageService.LayerStoreStatus()
v.Driver = daemon.graphDriver
v.DriverStatus = statuses[runtime.GOOS]
v.DriverStatus = daemon.imageService.LayerStoreStatus()
fillDriverWarnings(v)
}

2
daemon/start.go
View file

@ -235,7 +235,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) {
if err := daemon.conditionalUnmountOnCleanup(container); err != nil {
// FIXME: remove once reference counting for graphdrivers has been refactored
// Ensure that all the mounts are gone
if mountid, err := daemon.imageService.GetLayerMountID(container.ID, container.OS); err == nil {
if mountid, err := daemon.imageService.GetLayerMountID(container.ID); err == nil {
daemon.cleanupMountsByID(mountid)
}
}

14
distribution/config.go
View file

@ -71,8 +71,8 @@ type ImagePushConfig struct {
// ConfigMediaType is the configuration media type for
// schema2 manifests.
ConfigMediaType string
// LayerStores (indexed by operating system) manages layers.
LayerStores map[string]PushLayerProvider
// LayerStores manages layers.
LayerStores PushLayerProvider
// TrustKey is the private key for legacy signatures. This is typically
// an ephemeral key, since these signatures are no longer verified.
TrustKey libtrust.PrivateKey
@ -177,15 +177,11 @@ type storeLayerProvider struct {
ls layer.Store
}
// NewLayerProvidersFromStores returns layer providers backed by
// NewLayerProvidersFromStore returns layer providers backed by
// an instance of LayerStore. Only getting layers as gzipped
// tars is supported.
func NewLayerProvidersFromStores(lss map[string]layer.Store) map[string]PushLayerProvider {
plps := make(map[string]PushLayerProvider)
for os, ls := range lss {
plps[os] = &storeLayerProvider{ls: ls}
}
return plps
func NewLayerProvidersFromStore(ls layer.Store) PushLayerProvider {
return &storeLayerProvider{ls: ls}
}
func (p *storeLayerProvider) Get(lid layer.ChainID) (PushLayer, error) {

7
distribution/push_v2.go
View file

@ -122,12 +122,7 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
return fmt.Errorf("unable to get rootfs for image %s: %s", reference.FamiliarString(ref), err)
}
platform, err := p.config.ImageStore.PlatformFromConfig(imgConfig)
if err != nil {
return fmt.Errorf("unable to get platform for image %s: %s", reference.FamiliarString(ref), err)
}
l, err := p.config.LayerStores[platform.OS].Get(rootfs.ChainID())
l, err := p.config.LayerStores.Get(rootfs.ChainID())
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}

29
distribution/xfer/download.go
View file

@ -24,7 +24,7 @@ const maxDownloadAttempts = 5
// registers and downloads those, taking into account dependencies between
// layers.
type LayerDownloadManager struct {
layerStores map[string]layer.Store
layerStore layer.Store
tm TransferManager
waitDuration time.Duration
maxDownloadAttempts int
@ -36,9 +36,9 @@ func (ldm *LayerDownloadManager) SetConcurrency(concurrency int) {
}
// NewLayerDownloadManager returns a new LayerDownloadManager.
func NewLayerDownloadManager(layerStores map[string]layer.Store, concurrencyLimit int, options ...func(*LayerDownloadManager)) *LayerDownloadManager {
func NewLayerDownloadManager(layerStore layer.Store, concurrencyLimit int, options ...func(*LayerDownloadManager)) *LayerDownloadManager {
manager := LayerDownloadManager{
layerStores: layerStores,
layerStore: layerStore,
tm: NewTransferManager(concurrencyLimit),
waitDuration: time.Second,
maxDownloadAttempts: maxDownloadAttempts,
@ -118,6 +118,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
// Assume that the operating system is the host OS if blank, and validate it
// to ensure we don't cause a panic by an invalid index into the layerstores.
// TODO remove now that LCOW is no longer a thing
if os == "" {
os = runtime.GOOS
}
@ -136,13 +137,13 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
if err == nil {
getRootFS := rootFS
getRootFS.Append(diffID)
l, err := ldm.layerStores[os].Get(getRootFS.ChainID())
l, err := ldm.layerStore.Get(getRootFS.ChainID())
if err == nil {
// Layer already exists.
logrus.Debugf("Layer already exists: %s", descriptor.ID())
progress.Update(progressOutput, descriptor.ID(), "Already exists")
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
topLayer = l
missingLayer = false
@ -161,7 +162,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
// the stack? If so, avoid downloading it more than once.
var topDownloadUncasted Transfer
if existingDownload, ok := downloadsByKey[key]; ok {
xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload, os)
xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload)
defer topDownload.Transfer.Release(watcher)
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
topDownload = topDownloadUncasted.(*downloadTransfer)
@ -173,10 +174,10 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
var xferFunc DoFunc
if topDownload != nil {
xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload, os)
xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload)
defer topDownload.Transfer.Release(watcher)
} else {
xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil, os)
xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil)
}
topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
topDownload = topDownloadUncasted.(*downloadTransfer)
@ -186,7 +187,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
if topDownload == nil {
return rootFS, func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
}, nil
}
@ -197,7 +198,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
defer func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
layer.ReleaseAndLog(ldm.layerStore, topLayer)
}
}()
@ -233,11 +234,11 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
// complete before the registration step, and registers the downloaded data
// on top of parentDownload's resulting layer. Otherwise, it registers the
// layer on top of the ChainID given by parentLayer.
func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer, os string) DoFunc {
func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer) DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStores[os],
layerStore: ldm.layerStore,
}
go func() {
@ -397,11 +398,11 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor,
// parentDownload. This function does not log progress output because it would
// interfere with the progress reporting for sourceDownload, which has the same
// Key.
func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer, os string) DoFunc {
func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer) DoFunc {
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStores[os],
layerStore: ldm.layerStore,
}
go func() {

12
distribution/xfer/download_test.go
View file

@ -269,9 +269,7 @@ func TestSuccessfulDownload(t *testing.T) {
}
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
lsMap := make(map[string]layer.Store)
lsMap[runtime.GOOS] = layerStore
ldm := NewLayerDownloadManager(lsMap, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})
@ -333,9 +331,7 @@ func TestSuccessfulDownload(t *testing.T) {
func TestCancelledDownload(t *testing.T) {
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
lsMap := make(map[string]layer.Store)
lsMap[runtime.GOOS] = layerStore
ldm := NewLayerDownloadManager(lsMap, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})
@ -396,10 +392,8 @@ func TestMaxDownloadAttempts(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
lsMap := make(map[string]layer.Store)
lsMap[runtime.GOOS] = layerStore
ldm := NewLayerDownloadManager(
lsMap,
layerStore,
maxDownloadConcurrency,
func(m *LayerDownloadManager) {
m.waitDuration = time.Millisecond

10
image/store.go
View file

@ -42,14 +42,14 @@ type imageMeta struct {
type store struct {
sync.RWMutex
lss map[string]LayerGetReleaser
lss LayerGetReleaser
images map[ID]*imageMeta
fs StoreBackend
digestSet *digestset.Set
}
// NewImageStore returns new store object for given set of layer stores
func NewImageStore(fs StoreBackend, lss map[string]LayerGetReleaser) (Store, error) {
func NewImageStore(fs StoreBackend, lss LayerGetReleaser) (Store, error) {
is := &store{
lss: lss,
images: make(map[ID]*imageMeta),
@ -78,7 +78,7 @@ func (is *store) restore() error {
logrus.Errorf("not restoring image with unsupported operating system %v, %v, %s", dgst, chainID, img.OperatingSystem())
return nil
}
l, err = is.lss[img.OperatingSystem()].Get(chainID)
l, err = is.lss.Get(chainID)
if err != nil {
if err == layer.ErrLayerDoesNotExist {
logrus.Errorf("layer does not exist, not restoring image %v, %v, %s", dgst, chainID, img.OperatingSystem())
@ -160,7 +160,7 @@ func (is *store) Create(config []byte) (ID, error) {
if !system.IsOSSupported(img.OperatingSystem()) {
return "", system.ErrNotSupportedOperatingSystem
}
l, err = is.lss[img.OperatingSystem()].Get(layerID)
l, err = is.lss.Get(layerID)
if err != nil {
return "", errors.Wrapf(err, "failed to get layer %s", layerID)
}
@ -250,7 +250,7 @@ func (is *store) Delete(id ID) ([]layer.Metadata, error) {
is.fs.Delete(id.Digest())
if imageMeta.layer != nil {
return is.lss[img.OperatingSystem()].Release(imageMeta.layer)
return is.lss.Release(imageMeta.layer)
}
return nil, nil
}

9
image/store_test.go
View file

@ -2,7 +2,6 @@ package image // import "github.com/docker/docker/image"
import (
"fmt"
"runtime"
"testing"
"github.com/docker/docker/layer"
@ -34,9 +33,7 @@ func TestRestore(t *testing.T) {
err = fs.SetMetadata(id2, "parent", []byte(id1))
assert.NilError(t, err)
mlgrMap := make(map[string]LayerGetReleaser)
mlgrMap[runtime.GOOS] = &mockLayerGetReleaser{}
is, err := NewImageStore(fs, mlgrMap)
is, err := NewImageStore(fs, &mockLayerGetReleaser{})
assert.NilError(t, err)
assert.Check(t, cmp.Len(is.Map(), 2))
@ -153,9 +150,7 @@ func TestParentReset(t *testing.T) {
func defaultImageStore(t *testing.T) (Store, func()) {
fsBackend, cleanup := defaultFSStoreBackend(t)
mlgrMap := make(map[string]LayerGetReleaser)
mlgrMap[runtime.GOOS] = &mockLayerGetReleaser{}
store, err := NewImageStore(fsBackend, mlgrMap)
store, err := NewImageStore(fsBackend, &mockLayerGetReleaser{})
assert.NilError(t, err)
return store, cleanup

52
image/tarexport/load.go
View file

@ -10,7 +10,6 @@ import (
"path/filepath"
"runtime"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
@ -83,8 +82,8 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
if err != nil {
return err
}
if err := checkCompatibleOS(img.OS); err != nil {
return err
if !system.IsOSSupported(img.OperatingSystem()) {
return fmt.Errorf("cannot load %s image on %s", img.OperatingSystem(), runtime.GOOS)
}
rootFS := *img.RootFS
rootFS.DiffIDs = nil
@ -93,17 +92,6 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
return fmt.Errorf("invalid manifest, layers length mismatch: expected %d, got %d", expected, actual)
}
// On Windows, validate the platform, defaulting to windows if not present.
os := img.OS
if os == "" {
os = runtime.GOOS
}
if runtime.GOOS == "windows" {
if (os != "windows") && (os != "linux") {
return fmt.Errorf("configuration for this image has an unsupported operating system: %s", os)
}
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
@ -111,14 +99,14 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
}
r := rootFS
r.Append(diffID)
newLayer, err := l.lss[os].Get(r.ChainID())
newLayer, err := l.lss.Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), os, m.LayerSources[diffID], progressOutput)
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), m.LayerSources[diffID], progressOutput)
if err != nil {
return err
}
}
defer layer.ReleaseAndLog(l.lss[os], newLayer)
defer layer.ReleaseAndLog(l.lss, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
@ -180,7 +168,7 @@ func (l *tarexporter) setParentID(id, parentID image.ID) error {
return l.is.SetParent(id, parentID)
}
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, os string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
// We use system.OpenSequential to use sequential file access on Windows, avoiding
// depleting the standby list. On Linux, this equates to a regular os.Open.
rawTar, err := system.OpenSequential(filename)
@ -209,10 +197,10 @@ func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string,
}
defer inflatedLayerData.Close()
if ds, ok := l.lss[os].(layer.DescribableStore); ok {
if ds, ok := l.lss.(layer.DescribableStore); ok {
return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
}
return l.lss[os].Register(inflatedLayerData, rootFS.ChainID())
return l.lss.Register(inflatedLayerData, rootFS.ChainID())
}
func (l *tarexporter) setLoadedTag(ref reference.Named, imgID digest.Digest, outStream io.Writer) error {
@ -303,12 +291,12 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
return err
}
if err := checkCompatibleOS(img.OS); err != nil {
return err
}
if img.OS == "" {
img.OS = runtime.GOOS
}
if !system.IsOSSupported(img.OS) {
return fmt.Errorf("cannot load %s image on %s", img.OS, runtime.GOOS)
}
var parentID image.ID
if img.Parent != "" {
@ -342,7 +330,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, img.OS, distribution.Descriptor{}, progressOutput)
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, distribution.Descriptor{}, progressOutput)
if err != nil {
return err
}
@ -363,7 +351,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
return err
}
metadata, err := l.lss[img.OS].Release(newLayer)
metadata, err := l.lss.Release(newLayer)
layer.LogReleaseMetadata(metadata)
if err != nil {
return err
@ -416,20 +404,6 @@ func checkValidParent(img, parent *image.Image) bool {
return true
}
func checkCompatibleOS(imageOS string) error {
// always compatible if the images OS matches the host OS; also match an empty image OS
if imageOS == runtime.GOOS || imageOS == "" {
return nil
}
// On non-Windows hosts, for compatibility, fail if the image is Windows.
if runtime.GOOS != "windows" && imageOS == "windows" {
return fmt.Errorf("cannot load %s image on %s", imageOS, runtime.GOOS)
}
_, err := platforms.Parse(imageOS)
return err
}
func validateManifest(manifest []manifestItem) error {
// a nil manifest usually indicates a bug, so don't just silently fail.
// if someone really needs to pass an empty manifest, they can pass [].

16
image/tarexport/save.go
View file

@ -162,7 +162,7 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor)
if !system.IsOSSupported(os) {
return fmt.Errorf("os %q is not supported", os)
}
layer, err := l.lss[os].Get(topLayerID)
layer, err := l.lss.Get(topLayerID)
if err != nil {
return err
}
@ -174,11 +174,7 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor)
func (l *tarexporter) releaseLayerReferences(imgDescr map[image.ID]*imageDescriptor) error {
for _, descr := range imgDescr {
if descr.layerRef != nil {
os := descr.image.OS
if os == "" {
os = runtime.GOOS
}
l.lss[os].Release(descr.layerRef)
l.lss.Release(descr.layerRef)
}
}
return nil
@ -374,15 +370,11 @@ func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, creat
// serialize filesystem
layerPath := filepath.Join(outDir, legacyLayerFileName)
operatingSystem := legacyImg.OS
if operatingSystem == "" {
operatingSystem = runtime.GOOS
}
l, err := s.lss[operatingSystem].Get(id)
l, err := s.lss.Get(id)
if err != nil {
return distribution.Descriptor{}, err
}
defer layer.ReleaseAndLog(s.lss[operatingSystem], l)
defer layer.ReleaseAndLog(s.lss, l)
if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists {
relPath, err := filepath.Rel(outDir, oldPath)

4
image/tarexport/tarexport.go
View file

@ -25,7 +25,7 @@ type manifestItem struct {
type tarexporter struct {
is image.Store
lss map[string]layer.Store
lss layer.Store
rs refstore.Store
loggerImgEvent LogImageEvent
}
@ -37,7 +37,7 @@ type LogImageEvent interface {
}
// NewTarExporter returns new Exporter for tar packages
func NewTarExporter(is image.Store, lss map[string]layer.Store, rs refstore.Store, loggerImgEvent LogImageEvent) image.Exporter {
func NewTarExporter(is image.Store, lss layer.Store, rs refstore.Store, loggerImgEvent LogImageEvent) image.Exporter {
return &tarexporter{
is: is,
lss: lss,

5
integration/image/remove_unix_test.go
View file

@ -45,8 +45,7 @@ func TestRemoveImageGarbageCollector(t *testing.T) {
ctx := context.Background()
client := d.NewClientT(t)
layerStores := make(map[string]layer.Store)
layerStores[runtime.GOOS], _ = layer.NewStoreFromOptions(layer.StoreOptions{
layerStore, _ := layer.NewStoreFromOptions(layer.StoreOptions{
Root: d.Root,
MetadataStorePathTemplate: filepath.Join(d.RootDir(), "image", "%s", "layerdb"),
GraphDriver: d.StorageDriver(),
@ -57,7 +56,7 @@ func TestRemoveImageGarbageCollector(t *testing.T) {
OS: runtime.GOOS,
})
i := images.NewImageService(images.ImageServiceConfig{
LayerStores: layerStores,
LayerStore: layerStore,
})
img := "test-garbage-collector"