From 7b9a8f460bfa55dacca74f2ed0164323811e1196 Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 17 Aug 2017 15:43:36 -0700 Subject: [PATCH] Move to a single tag-store Signed-off-by: John Howard --- daemon/daemon.go | 27 +++++++++++++++++++-------- daemon/disk_usage.go | 2 +- daemon/image.go | 16 ++++++++++++---- daemon/image_delete.go | 12 ++++++------ daemon/image_exporter.go | 4 ++-- daemon/image_history.go | 2 +- daemon/image_inspect.go | 2 +- daemon/image_pull.go | 2 +- daemon/image_push.go | 2 +- daemon/image_tag.go | 2 +- daemon/images.go | 2 +- daemon/prune.go | 4 ++-- reference/store.go | 8 ++------ reference/store_test.go | 9 ++++----- 14 files changed, 54 insertions(+), 40 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index ebfdb68d1d..d194bda287 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -75,7 +75,6 @@ type daemonStore struct { imageStore image.Store layerStore layer.Store distributionMetadataStore dmetadata.Store - referenceStore refstore.Store } // Daemon holds information about the Docker daemon. @@ -103,7 +102,8 @@ type Daemon struct { shutdown bool idMappings *idtools.IDMappings stores map[string]daemonStore // By container target platform - PluginStore *plugin.Store // todo: remove + referenceStore refstore.Store + PluginStore *plugin.Store // todo: remove pluginManager *plugin.Manager linkIndex *linkIndex containerd libcontainerd.Client @@ -691,7 +691,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.downloadManager = xfer.NewLayerDownloadManager(lsMap, *config.MaxConcurrentDownloads) logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads) - for platform, ds := range d.stores { imageRoot := filepath.Join(config.Root, "image", ds.graphDriver) ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) @@ -728,18 +727,30 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe eventsService := events.New() + // We have a single tag/reference store for the daemon globally. However, it's + // stored under the graphdriver. On host platforms which only support a single + // container OS, but multiple selectable graphdrivers, this means depending on which + // graphdriver is chosen, the global reference store is under there. For + // platforms which support multiple container operating systems, this is slightly + // more problematic as where does the global ref store get located? Fortunately, + // for Windows, which is currently the only daemon supporting multiple container + // operating systems, the list of graphdrivers available isn't user configurable. + // For backwards compatibility, we just put it under the windowsfilter + // directory regardless. + refStoreLocation := filepath.Join(d.stores[runtime.GOOS].imageRoot, `repositories.json`) + rs, err := refstore.NewReferenceStore(refStoreLocation) + if err != nil { + return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) + } + d.referenceStore = rs + for platform, ds := range d.stores { dms, err := dmetadata.NewFSMetadataStore(filepath.Join(ds.imageRoot, "distribution"), platform) if err != nil { return nil, err } - rs, err := refstore.NewReferenceStore(filepath.Join(ds.imageRoot, "repositories.json"), platform) - if err != nil { - return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err) - } ds.distributionMetadataStore = dms - ds.referenceStore = rs d.stores[platform] = ds // No content-addressability migration on Windows as it never supported pre-CA diff --git a/daemon/disk_usage.go b/daemon/disk_usage.go index 23bce4f70b..a28463eba9 100644 --- a/daemon/disk_usage.go +++ b/daemon/disk_usage.go @@ -20,7 +20,7 @@ func (daemon *Daemon) getLayerRefs(platform string) map[layer.ChainID]int { layerRefs := map[layer.ChainID]int{} for id, img := range tmpImages { dgst := digest.Digest(id) - if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { + if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { continue } diff --git a/daemon/image.go b/daemon/image.go index ac0d6bc654..23670c4e0e 100644 --- a/daemon/image.go +++ b/daemon/image.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "runtime" "github.com/docker/distribution/reference" "github.com/docker/docker/image" @@ -45,10 +46,17 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e return "", "", errImageDoesNotExist{ref} } - for platform := range daemon.stores { - if id, err := daemon.stores[platform].referenceStore.Get(namedRef); err == nil { - return image.IDFromDigest(id), platform, nil + if digest, err := daemon.referenceStore.Get(namedRef); err == nil { + // Search the image stores to get the platform, defaulting to host OS. + imagePlatform := runtime.GOOS + id := image.IDFromDigest(digest) + for platform := range daemon.stores { + if img, err := daemon.stores[platform].imageStore.Get(id); err == nil { + imagePlatform = img.Platform() + break + } } + return id, imagePlatform, nil } // deprecated: repo:shortid https://github.com/docker/docker/pull/799 @@ -56,7 +64,7 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e if tag := tagged.Tag(); stringid.IsShortID(stringid.TruncateID(tag)) { for platform := range daemon.stores { if id, err := daemon.stores[platform].imageStore.Search(tag); err == nil { - for _, storeRef := range daemon.stores[platform].referenceStore.References(id.Digest()) { + for _, storeRef := range daemon.referenceStore.References(id.Digest()) { if storeRef.Name() == namedRef.Name() { return id, platform, nil } diff --git a/daemon/image_delete.go b/daemon/image_delete.go index 84b86580c6..9873924818 100644 --- a/daemon/image_delete.go +++ b/daemon/image_delete.go @@ -70,7 +70,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I return nil, err } - repoRefs := daemon.stores[platform].referenceStore.References(imgID.Digest()) + repoRefs := daemon.referenceStore.References(imgID.Digest()) var removedRepositoryRef bool if !isImageIDPrefix(imgID.String(), imageRef) { @@ -104,7 +104,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") records = append(records, untaggedRecord) - repoRefs = daemon.stores[platform].referenceStore.References(imgID.Digest()) + repoRefs = daemon.referenceStore.References(imgID.Digest()) // If a tag reference was removed and the only remaining // references to the same repository are digest references, @@ -237,7 +237,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe // Ignore the boolean value returned, as far as we're concerned, this // is an idempotent operation and it's okay if the reference didn't // exist in the first place. - _, err := daemon.stores[platform].referenceStore.Delete(ref) + _, err := daemon.referenceStore.Delete(ref) return ref, err } @@ -248,7 +248,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the // given list of records. func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, platform string, records *[]types.ImageDeleteResponseItem) error { - imageRefs := daemon.stores[platform].referenceStore.References(imgID.Digest()) + imageRefs := daemon.referenceStore.References(imgID.Digest()) for _, imageRef := range imageRefs { parsedRef, err := daemon.removeImageRef(platform, imageRef) @@ -383,7 +383,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string, } // Check if any repository tags/digest reference this image. - if mask&conflictActiveReference != 0 && len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 { + if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 { return &imageDeleteConflict{ imgID: imgID, message: "image is referenced in multiple repositories", @@ -411,5 +411,5 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string, // that there are no repository references to the given image and it has no // child images. func (daemon *Daemon) imageIsDangling(imgID image.ID, platform string) bool { - return !(len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0) + return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0) } diff --git a/daemon/image_exporter.go b/daemon/image_exporter.go index a7b0be64c2..ce9aa7a050 100644 --- a/daemon/image_exporter.go +++ b/daemon/image_exporter.go @@ -19,7 +19,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { if system.LCOWSupported() { platform = "linux" } - imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) + imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon) return imageExporter.Save(names, outStream) } @@ -32,6 +32,6 @@ func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet if system.LCOWSupported() { platform = "linux" } - imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) + imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon) return imageExporter.Load(inTar, outStream, quiet) } diff --git a/daemon/image_history.go b/daemon/image_history.go index c9e81554e9..e7dd85cbab 100644 --- a/daemon/image_history.go +++ b/daemon/image_history.go @@ -69,7 +69,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e h.ID = id.String() var tags []string - for _, r := range daemon.stores[platform].referenceStore.References(id.Digest()) { + for _, r := range daemon.referenceStore.References(id.Digest()) { if _, ok := r.(reference.NamedTagged); ok { tags = append(tags, reference.FamiliarString(r)) } diff --git a/daemon/image_inspect.go b/daemon/image_inspect.go index 3baf265dab..fefb93c6e1 100644 --- a/daemon/image_inspect.go +++ b/daemon/image_inspect.go @@ -24,7 +24,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { platform = runtime.GOOS } - refs := daemon.stores[platform].referenceStore.References(img.ID().Digest()) + refs := daemon.referenceStore.References(img.ID().Digest()) repoTags := []string{} repoDigests := []string{} for _, ref := range refs { diff --git a/daemon/image_pull.go b/daemon/image_pull.go index 0f59f77711..aef1876bad 100644 --- a/daemon/image_pull.go +++ b/daemon/image_pull.go @@ -74,7 +74,7 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. ImageEventLogger: daemon.LogImageEvent, MetadataStore: daemon.stores[platform].distributionMetadataStore, ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore), - ReferenceStore: daemon.stores[platform].referenceStore, + ReferenceStore: daemon.referenceStore, }, DownloadManager: daemon.downloadManager, Schema2Types: distribution.ImageTypes, diff --git a/daemon/image_push.go b/daemon/image_push.go index c2e5967b19..b558073379 100644 --- a/daemon/image_push.go +++ b/daemon/image_push.go @@ -56,7 +56,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead ImageEventLogger: daemon.LogImageEvent, MetadataStore: daemon.stores[platform].distributionMetadataStore, ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore), - ReferenceStore: daemon.stores[platform].referenceStore, + ReferenceStore: daemon.referenceStore, }, ConfigMediaType: schema2.MediaTypeImageConfig, LayerStore: distribution.NewLayerProviderFromStore(daemon.stores[platform].layerStore), diff --git a/daemon/image_tag.go b/daemon/image_tag.go index 5f28daed0a..0c1d761c87 100644 --- a/daemon/image_tag.go +++ b/daemon/image_tag.go @@ -28,7 +28,7 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) error { // TagImageWithReference adds the given reference to the image ID provided. func (daemon *Daemon) TagImageWithReference(imageID image.ID, platform string, newTag reference.Named) error { - if err := daemon.stores[platform].referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { + if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { return err } diff --git a/daemon/images.go b/daemon/images.go index 548c3a5fc8..9be6e0ec30 100644 --- a/daemon/images.go +++ b/daemon/images.go @@ -149,7 +149,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs newImage := newImage(img, size) - for _, ref := range daemon.stores[platform].referenceStore.References(id.Digest()) { + for _, ref := range daemon.referenceStore.References(id.Digest()) { if imageFilters.Include("reference") { var found bool var matchErr error diff --git a/daemon/prune.go b/daemon/prune.go index 7df922d6b5..66eca2b6e5 100644 --- a/daemon/prune.go +++ b/daemon/prune.go @@ -221,7 +221,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args return nil, ctx.Err() default: dgst := digest.Digest(id) - if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { + if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { continue } if !until.IsZero() && img.Created.After(until) { @@ -252,7 +252,7 @@ deleteImagesLoop: } deletedImages := []types.ImageDeleteResponseItem{} - refs := daemon.stores[platform].referenceStore.References(dgst) + refs := daemon.referenceStore.References(dgst) if len(refs) > 0 { shouldDelete := !danglingOnly if !shouldDelete { diff --git a/reference/store.go b/reference/store.go index b98f9579db..bde5951927 100644 --- a/reference/store.go +++ b/reference/store.go @@ -26,7 +26,7 @@ type Association struct { ID digest.Digest } -// Store provides the set of methods which can operate on a tag store. +// Store provides the set of methods which can operate on a reference store. type Store interface { References(id digest.Digest) []reference.Named ReferencesByName(ref reference.Named) []Association @@ -46,9 +46,6 @@ type store struct { // referencesByIDCache is a cache of references indexed by ID, to speed // up References. referencesByIDCache map[digest.Digest]map[string]reference.Named - // platform is the container target platform for this store (which may be - // different to the host operating system - platform string } // Repository maps tags to digests. The key is a stringified Reference, @@ -73,7 +70,7 @@ func (a lexicalAssociations) Less(i, j int) bool { // NewReferenceStore creates a new reference store, tied to a file path where // the set of references are serialized in JSON format. -func NewReferenceStore(jsonPath, platform string) (Store, error) { +func NewReferenceStore(jsonPath string) (Store, error) { abspath, err := filepath.Abs(jsonPath) if err != nil { return nil, err @@ -83,7 +80,6 @@ func NewReferenceStore(jsonPath, platform string) (Store, error) { jsonPath: abspath, Repositories: make(map[string]repository), referencesByIDCache: make(map[digest.Digest]map[string]reference.Named), - platform: platform, } // Load the json file if it exists, otherwise create it. if err := store.reload(); os.IsNotExist(err) { diff --git a/reference/store_test.go b/reference/store_test.go index 2c796e76f9..8f0ff6304e 100644 --- a/reference/store_test.go +++ b/reference/store_test.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" "path/filepath" - "runtime" "strings" "testing" @@ -41,7 +40,7 @@ func TestLoad(t *testing.T) { } jsonFile.Close() - store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) + store, err := NewReferenceStore(jsonFile.Name()) if err != nil { t.Fatalf("error creating tag store: %v", err) } @@ -70,7 +69,7 @@ func TestSave(t *testing.T) { jsonFile.Close() defer os.RemoveAll(jsonFile.Name()) - store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) + store, err := NewReferenceStore(jsonFile.Name()) if err != nil { t.Fatalf("error creating tag store: %v", err) } @@ -112,7 +111,7 @@ func TestAddDeleteGet(t *testing.T) { jsonFile.Close() defer os.RemoveAll(jsonFile.Name()) - store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) + store, err := NewReferenceStore(jsonFile.Name()) if err != nil { t.Fatalf("error creating tag store: %v", err) } @@ -329,7 +328,7 @@ func TestInvalidTags(t *testing.T) { tmpDir, err := ioutil.TempDir("", "tag-store-test") defer os.RemoveAll(tmpDir) - store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"), runtime.GOOS) + store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json")) if err != nil { t.Fatalf("error creating tag store: %v", err) }