From 9c902364fb97faeca8257ed683bdef27615d9f1f Mon Sep 17 00:00:00 2001 From: John Starks Date: Fri, 20 May 2016 13:23:01 -0700 Subject: [PATCH 1/2] Revendor docker/distribution Signed-off-by: John Starks --- hack/vendor.sh | 2 +- vendor/src/github.com/docker/distribution/AUTHORS | 12 ++++++++++++ vendor/src/github.com/docker/distribution/Dockerfile | 10 +++++----- vendor/src/github.com/docker/distribution/blobs.go | 3 +++ .../docker/distribution/manifest/schema2/manifest.go | 5 ++++- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index b041665556..8e03548f9e 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -51,7 +51,7 @@ clone git github.com/boltdb/bolt v1.2.1 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages -clone git github.com/docker/distribution 9ec0d742d69f77caa4dd5f49ceb70c3067d39f30 +clone git github.com/docker/distribution 5bbf65499960b184fe8e0f045397375e1a6722b8 clone git github.com/vbatts/tar-split v0.9.11 # get go-zfs packages diff --git a/vendor/src/github.com/docker/distribution/AUTHORS b/vendor/src/github.com/docker/distribution/AUTHORS index 0857b62fc9..70d525999b 100644 --- a/vendor/src/github.com/docker/distribution/AUTHORS +++ b/vendor/src/github.com/docker/distribution/AUTHORS @@ -1,4 +1,5 @@ Aaron Lehmann +Aaron Schlesinger Aaron Vinson Adam Enger Adrian Mouat @@ -7,13 +8,16 @@ Alex Chan Alex Elman amitshukla Amy Lindburg +Andrew Hsu Andrew Meredith Andrew T Nguyen Andrey Kostov Andy Goldstein +Anis Elleuch Anton Tiurin Antonio Mercado Antonio Murdaca +Arien Holthuizen Arnaud Porterie Arthur Baars Asuka Suzuki @@ -27,6 +31,7 @@ burnettk Carson A Chris Dillon Daisuke Fujita +Daniel Huhn Darren Shepherd Dave Trombley Dave Tucker @@ -41,6 +46,7 @@ DJ Enriquez Donald Huang Doug Davis Eric Yang +Fabio Huser farmerworking Felix Yan Florentin Raud @@ -57,8 +63,10 @@ Jack Griffin Jason Freidman Jeff Nickoloff Jessie Frazelle +jhaohai Jianqing Wang John Starks +Jon Johnson Jon Poler Jonathan Boulle Jordan Liggitt @@ -92,17 +100,20 @@ Olivier Gambier Olivier Jacques Omer Cohen Patrick Devine +Phil Estes Philip Misiowiec Richard Scothern Rodolfo Carvalho Rusty Conover Sean Boran Sebastiaan van Stijn +Serge Dubrouski Sharif Nassar Shawn Falkner-Horine Shreyas Karnik Simon Thulbourn Spencer Rinehart +Stefan Majewsky Stefan Weil Stephen J Day Sungho Moon @@ -114,6 +125,7 @@ Thomas Sjögren Tianon Gravi Tibor Vass Tonis Tiigi +Tony Holdstock-Brown Trevor Pounds Troels Thomsen Vincent Batts diff --git a/vendor/src/github.com/docker/distribution/Dockerfile b/vendor/src/github.com/docker/distribution/Dockerfile index abb3e3bbf8..fa9cd4627e 100644 --- a/vendor/src/github.com/docker/distribution/Dockerfile +++ b/vendor/src/github.com/docker/distribution/Dockerfile @@ -1,8 +1,4 @@ -FROM golang:1.6 - -RUN apt-get update && \ - apt-get install -y apache2-utils && \ - rm -rf /var/lib/apt/lists/* +FROM golang:1.6-alpine ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution ENV DOCKER_BUILDTAGS include_oss include_gcs @@ -10,6 +6,10 @@ ENV DOCKER_BUILDTAGS include_oss include_gcs WORKDIR $DISTRIBUTION_DIR COPY . $DISTRIBUTION_DIR COPY cmd/registry/config-dev.yml /etc/docker/registry/config.yml + +RUN set -ex \ + && apk add --no-cache make git + RUN make PREFIX=/go clean binaries VOLUME ["/var/lib/registry"] diff --git a/vendor/src/github.com/docker/distribution/blobs.go b/vendor/src/github.com/docker/distribution/blobs.go index 1765e9f740..9f572bfabb 100644 --- a/vendor/src/github.com/docker/distribution/blobs.go +++ b/vendor/src/github.com/docker/distribution/blobs.go @@ -69,6 +69,9 @@ type Descriptor struct { // against against this digest. Digest digest.Digest `json:"digest,omitempty"` + // URLs contains the source URLs of this content. + URLs []string `json:"urls,omitempty"` + // NOTE: Before adding a field here, please ensure that all // other options have been exhausted. Much of the type relationships // depend on the simplicity of this type. diff --git a/vendor/src/github.com/docker/distribution/manifest/schema2/manifest.go b/vendor/src/github.com/docker/distribution/manifest/schema2/manifest.go index 8d378e990b..355b5ad4ea 100644 --- a/vendor/src/github.com/docker/distribution/manifest/schema2/manifest.go +++ b/vendor/src/github.com/docker/distribution/manifest/schema2/manifest.go @@ -20,6 +20,10 @@ const ( // MediaTypeLayer is the mediaType used for layers referenced by the // manifest. MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" + + // MediaTypeForeignLayer is the mediaType used for layers that must be + // downloaded from foreign URLs. + MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" ) var ( @@ -63,7 +67,6 @@ type Manifest struct { // References returnes the descriptors of this manifests references. func (m Manifest) References() []distribution.Descriptor { return m.Layers - } // Target returns the target of this signed manifest. From 05bd04350b8348b3c3bbe3156420257313e4e804 Mon Sep 17 00:00:00 2001 From: John Starks Date: Wed, 25 May 2016 19:11:51 -0700 Subject: [PATCH 2/2] Support layers from external URLs This is used to support downloading Windows base images from Microsoft servers. Signed-off-by: John Starks --- distribution/pull_v2.go | 42 +++++++++++-------- distribution/pull_v2_unix.go | 7 ++++ distribution/pull_v2_windows.go | 36 ++++++++++++++++ distribution/push_v2.go | 7 ++++ distribution/xfer/download.go | 13 +++++- distribution/xfer/download_test.go | 5 +++ image/tarexport/load.go | 15 ++++--- image/tarexport/save.go | 66 +++++++++++++++++++----------- image/tarexport/tarexport.go | 10 +++-- layer/filestore.go | 30 ++++++++++++++ layer/layer.go | 12 ++++++ layer/layer_store.go | 13 ++++++ layer/layer_unix.go | 6 +++ layer/layer_windows.go | 6 +++ layer/ro_layer.go | 7 ++++ layer/ro_layer_windows.go | 9 ++++ 16 files changed, 231 insertions(+), 53 deletions(-) create mode 100644 layer/ro_layer_windows.go diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 748ef422b3..256964561f 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -133,6 +133,7 @@ type v2LayerDescriptor struct { V2MetadataService *metadata.V2MetadataService tmpFile *os.File verifier digest.Verifier + foreignSrc *distribution.Descriptor } func (ld *v2LayerDescriptor) Key() string { @@ -180,9 +181,8 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre } tmpFile := ld.tmpFile - blobs := ld.repo.Blobs(ctx) - layerDownload, err := blobs.Open(ctx, ld.digest) + layerDownload, err := ld.open(ctx) if err != nil { logrus.Errorf("Error initiating layer download: %v", err) if err == distribution.ErrBlobUnknown { @@ -501,6 +501,29 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s return imageID, manifestDigest, nil } + var descriptors []xfer.DownloadDescriptor + + // Note that the order of this loop is in the direction of bottom-most + // to top-most, so that the downloads slice gets ordered correctly. + for _, d := range mfst.Layers { + layerDescriptor := &v2LayerDescriptor{ + digest: d.Digest, + repo: p.repo, + repoInfo: p.repoInfo, + V2MetadataService: p.V2MetadataService, + } + + if d.MediaType == schema2.MediaTypeForeignLayer && len(d.URLs) > 0 { + if !layer.ForeignSourceSupported() { + return "", "", errors.New("foreign layers are not supported on this OS") + } + + layerDescriptor.foreignSrc = &d + } + + descriptors = append(descriptors, layerDescriptor) + } + configChan := make(chan []byte, 1) errChan := make(chan error, 1) var cancel func() @@ -517,21 +540,6 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s configChan <- configJSON }() - var descriptors []xfer.DownloadDescriptor - - // Note that the order of this loop is in the direction of bottom-most - // to top-most, so that the downloads slice gets ordered correctly. - for _, d := range mfst.References() { - layerDescriptor := &v2LayerDescriptor{ - digest: d.Digest, - repo: p.repo, - repoInfo: p.repoInfo, - V2MetadataService: p.V2MetadataService, - } - - descriptors = append(descriptors, layerDescriptor) - } - var ( configJSON []byte // raw serialized image config unmarshalledConfig image.Image // deserialized image config diff --git a/distribution/pull_v2_unix.go b/distribution/pull_v2_unix.go index 9fbb875efc..cdd7806ad2 100644 --- a/distribution/pull_v2_unix.go +++ b/distribution/pull_v2_unix.go @@ -3,6 +3,8 @@ package distribution import ( + "github.com/docker/distribution" + "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/image" ) @@ -10,3 +12,8 @@ import ( func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error { return nil } + +func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) { + blobs := ld.repo.Blobs(ctx) + return blobs.Open(ctx, ld.digest) +} diff --git a/distribution/pull_v2_windows.go b/distribution/pull_v2_windows.go index d99434431f..3dc112d355 100644 --- a/distribution/pull_v2_windows.go +++ b/distribution/pull_v2_windows.go @@ -5,9 +5,15 @@ package distribution import ( "encoding/json" "fmt" + "net/http" + "os" + "github.com/docker/distribution" + "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/image" + "github.com/docker/docker/layer" ) func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error { @@ -28,3 +34,33 @@ func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) } return fmt.Errorf("Invalid base layer %q", v1img.Parent) } + +var _ layer.ForeignSourcer = &v2LayerDescriptor{} + +func (ld *v2LayerDescriptor) ForeignSource() *distribution.Descriptor { + return ld.foreignSrc +} + +func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) { + if ld.foreignSrc == nil { + blobs := ld.repo.Blobs(ctx) + return blobs.Open(ctx, ld.digest) + } + + var ( + err error + rsc distribution.ReadSeekCloser + ) + + // Find the first URL that results in a 200 result code. + for _, url := range ld.foreignSrc.URLs { + rsc = transport.NewHTTPReadSeeker(http.DefaultClient, url, nil) + _, err = rsc.Seek(0, os.SEEK_SET) + if err == nil { + break + } + rsc.Close() + rsc = nil + } + return rsc, err +} diff --git a/distribution/push_v2.go b/distribution/push_v2.go index e86badb472..fc745be13d 100644 --- a/distribution/push_v2.go +++ b/distribution/push_v2.go @@ -240,6 +240,13 @@ func (pd *v2PushDescriptor) DiffID() layer.DiffID { } func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { + if fs, ok := pd.layer.(layer.ForeignSourcer); ok { + if d := fs.ForeignSource(); d != nil { + progress.Update(progressOutput, pd.ID(), "Skipped foreign layer") + return *d, nil + } + } + diffID := pd.DiffID() pd.pushState.Lock() diff --git a/distribution/xfer/download.go b/distribution/xfer/download.go index 67ffc1a988..6600b244b6 100644 --- a/distribution/xfer/download.go +++ b/distribution/xfer/download.go @@ -7,6 +7,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" @@ -318,7 +319,11 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, return } - d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer) + var src *distribution.Descriptor + if fs, ok := descriptor.(layer.ForeignSourcer); ok { + src = fs.ForeignSource() + } + d.layer, err = d.layerStore.RegisterForeign(inflatedLayerData, parentLayer, src) if err != nil { select { case <-d.Transfer.Context().Done(): @@ -409,7 +414,11 @@ func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor Downloa } defer layerReader.Close() - d.layer, err = d.layerStore.Register(layerReader, parentLayer) + var src *distribution.Descriptor + if fs, ok := l.(layer.ForeignSourcer); ok { + src = fs.ForeignSource() + } + d.layer, err = d.layerStore.RegisterForeign(layerReader, parentLayer, src) if err != nil { d.err = fmt.Errorf("failed to register layer: %v", err) return diff --git a/distribution/xfer/download_test.go b/distribution/xfer/download_test.go index 5a38e3f038..f477062795 100644 --- a/distribution/xfer/download_test.go +++ b/distribution/xfer/download_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -71,6 +72,10 @@ func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer. } func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) { + return ls.RegisterForeign(reader, parentID, nil) +} + +func (ls *mockLayerStore) RegisterForeign(reader io.Reader, parentID layer.ChainID, _ *distribution.Descriptor) (layer.Layer, error) { var ( parent layer.Layer err error diff --git a/image/tarexport/load.go b/image/tarexport/load.go index efb09c573d..94efed8d94 100644 --- a/image/tarexport/load.go +++ b/image/tarexport/load.go @@ -10,6 +10,7 @@ import ( "reflect" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/docker/image" "github.com/docker/docker/image/v1" "github.com/docker/docker/layer" @@ -63,6 +64,10 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) var parentLinks []parentLink for _, m := range manifest { + if m.LayerSources != nil && !layer.ForeignSourceSupported() { + return fmt.Errorf("invalid manifest, foreign layers not supported on this operating system") + } + configPath, err := safePath(tmpDir, m.Config) if err != nil { return err @@ -92,7 +97,7 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) r.Append(diffID) newLayer, err := l.ls.Get(r.ChainID()) if err != nil { - newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput) + newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), m.LayerSources[diffID], progressOutput) if err != nil { return err } @@ -151,7 +156,7 @@ func (l *tarexporter) setParentID(id, parentID image.ID) error { return l.is.SetParent(id, parentID) } -func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressOutput progress.Output) (layer.Layer, error) { +func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc *distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) { rawTar, err := os.Open(filename) if err != nil { logrus.Debugf("Error reading embedded tar: %v", err) @@ -174,9 +179,9 @@ func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressReader := progress.NewProgressReader(inflatedLayerData, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer") - return l.ls.Register(progressReader, rootFS.ChainID()) + return l.ls.RegisterForeign(progressReader, rootFS.ChainID(), foreignSrc) } - return l.ls.Register(inflatedLayerData, rootFS.ChainID()) + return l.ls.RegisterForeign(inflatedLayerData, rootFS.ChainID(), foreignSrc) } func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error { @@ -298,7 +303,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str if err != nil { return err } - newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, progressOutput) + newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, nil, progressOutput) if err != nil { return err } diff --git a/image/tarexport/save.go b/image/tarexport/save.go index a87a7512d6..34065ba2a3 100644 --- a/image/tarexport/save.go +++ b/image/tarexport/save.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/image" "github.com/docker/docker/image/v1" @@ -131,7 +132,8 @@ func (s *saveSession) save(outStream io.Writer) error { var parentLinks []parentLink for id, imageDescr := range s.images { - if err = s.saveImage(id); err != nil { + foreignSrcs, err := s.saveImage(id) + if err != nil { return err } @@ -151,9 +153,10 @@ func (s *saveSession) save(outStream io.Writer) error { } manifest = append(manifest, manifestItem{ - Config: digest.Digest(id).Hex() + ".json", - RepoTags: repoTags, - Layers: layers, + Config: digest.Digest(id).Hex() + ".json", + RepoTags: repoTags, + Layers: layers, + LayerSources: foreignSrcs, }) parentID, _ := s.is.GetParent(id) @@ -213,18 +216,19 @@ func (s *saveSession) save(outStream io.Writer) error { return nil } -func (s *saveSession) saveImage(id image.ID) error { +func (s *saveSession) saveImage(id image.ID) (map[layer.DiffID]*distribution.Descriptor, error) { img, err := s.is.Get(id) if err != nil { - return err + return nil, err } if len(img.RootFS.DiffIDs) == 0 { - return fmt.Errorf("empty export - not implemented") + return nil, fmt.Errorf("empty export - not implemented") } var parent digest.Digest var layers []string + var foreignSrcs map[layer.DiffID]*distribution.Descriptor for i := range img.RootFS.DiffIDs { v1Img := image.V1Image{} if i == len(img.RootFS.DiffIDs)-1 { @@ -234,7 +238,7 @@ func (s *saveSession) saveImage(id image.ID) error { rootFS.DiffIDs = rootFS.DiffIDs[:i+1] v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent) if err != nil { - return err + return nil, err } v1Img.ID = v1ID.Hex() @@ -242,79 +246,91 @@ func (s *saveSession) saveImage(id image.ID) error { v1Img.Parent = parent.Hex() } - if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil { - return err + src, err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created) + if err != nil { + return nil, err } layers = append(layers, v1Img.ID) parent = v1ID + if src != nil { + if foreignSrcs == nil { + foreignSrcs = make(map[layer.DiffID]*distribution.Descriptor) + } + foreignSrcs[img.RootFS.DiffIDs[i]] = src + } } configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json") if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil { - return err + return nil, err } if err := system.Chtimes(configFile, img.Created, img.Created); err != nil { - return err + return nil, err } s.images[id].layers = layers - return nil + return foreignSrcs, nil } -func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error { +func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) (*distribution.Descriptor, error) { if _, exists := s.savedLayers[legacyImg.ID]; exists { - return nil + return nil, nil } outDir := filepath.Join(s.outDir, legacyImg.ID) if err := os.Mkdir(outDir, 0755); err != nil { - return err + return nil, err } // todo: why is this version file here? if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil { - return err + return nil, err } imageConfig, err := json.Marshal(legacyImg) if err != nil { - return err + return nil, err } if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil { - return err + return nil, err } // serialize filesystem tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName)) if err != nil { - return err + return nil, err } defer tarFile.Close() l, err := s.ls.Get(id) if err != nil { - return err + return nil, err } defer layer.ReleaseAndLog(s.ls, l) arch, err := l.TarStream() if err != nil { - return err + return nil, err } defer arch.Close() if _, err := io.Copy(tarFile, arch); err != nil { - return err + return nil, err } for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} { // todo: maybe save layer created timestamp? if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil { - return err + return nil, err } } s.savedLayers[legacyImg.ID] = struct{}{} - return nil + + var src *distribution.Descriptor + if fs, ok := l.(layer.ForeignSourcer); ok { + src = fs.ForeignSource() + } + return src, nil } diff --git a/image/tarexport/tarexport.go b/image/tarexport/tarexport.go index b1aa5b6599..07cc337224 100644 --- a/image/tarexport/tarexport.go +++ b/image/tarexport/tarexport.go @@ -1,6 +1,7 @@ package tarexport import ( + "github.com/docker/distribution" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/reference" @@ -15,10 +16,11 @@ const ( ) type manifestItem struct { - Config string - RepoTags []string - Layers []string - Parent image.ID `json:",omitempty"` + Config string + RepoTags []string + Layers []string + Parent image.ID `json:",omitempty"` + LayerSources map[layer.DiffID]*distribution.Descriptor `json:",omitempty"` } type tarexporter struct { diff --git a/layer/filestore.go b/layer/filestore.go index a0044b3663..6e11ca9a37 100644 --- a/layer/filestore.go +++ b/layer/filestore.go @@ -2,6 +2,7 @@ package layer import ( "compress/gzip" + "encoding/json" "errors" "fmt" "io" @@ -13,6 +14,7 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/pkg/ioutils" ) @@ -24,6 +26,9 @@ var ( // digest.SHA384, // Currently not used // digest.SHA512, // Currently not used } + + // ErrNoForeignSource is returned when no foreign source is set for a layer. + ErrNoForeignSource = errors.New("layer does not have a foreign source") ) type fileMetadataStore struct { @@ -98,6 +103,14 @@ func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error { return ioutil.WriteFile(filepath.Join(fm.root, "cache-id"), []byte(cacheID), 0644) } +func (fm *fileMetadataTransaction) SetForeignSource(ref distribution.Descriptor) error { + jsonRef, err := json.Marshal(ref) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(fm.root, "descriptor.json"), jsonRef, 0644) +} + func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) { f, err := os.OpenFile(filepath.Join(fm.root, "tar-split.json.gz"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -191,6 +204,23 @@ func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) { return content, nil } +func (fms *fileMetadataStore) GetForeignSource(layer ChainID) (distribution.Descriptor, error) { + content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "descriptor.json")) + if err != nil { + if os.IsNotExist(err) { + return distribution.Descriptor{}, ErrNoForeignSource + } + return distribution.Descriptor{}, err + } + + var ref distribution.Descriptor + err = json.Unmarshal(content, &ref) + if err != nil { + return distribution.Descriptor{}, err + } + return ref, err +} + func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) { fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz")) if err != nil { diff --git a/layer/layer.go b/layer/layer.go index 5100fe2dee..edef3c4eba 100644 --- a/layer/layer.go +++ b/layer/layer.go @@ -14,6 +14,7 @@ import ( "io" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/pkg/archive" ) @@ -107,6 +108,14 @@ type Layer interface { Metadata() (map[string]string, error) } +// ForeignSourcer is an interface used to describe the source of layers +// and objects representing layers, when the source is a foreign URL. +type ForeignSourcer interface { + // ForeignSource returns the descriptor for this layer if it is + // a foreign layer, or nil for ordinary layers. + ForeignSource() *distribution.Descriptor +} + // RWLayer represents a layer which is // read and writable type RWLayer interface { @@ -168,6 +177,7 @@ type MountInit func(root string) error // read-only and read-write layers. type Store interface { Register(io.Reader, ChainID) (Layer, error) + RegisterForeign(io.Reader, ChainID, *distribution.Descriptor) (Layer, error) Get(ChainID) (Layer, error) Release(Layer) ([]Metadata, error) @@ -189,6 +199,7 @@ type MetadataTransaction interface { SetParent(parent ChainID) error SetDiffID(DiffID) error SetCacheID(string) error + SetForeignSource(distribution.Descriptor) error TarSplitWriter(compressInput bool) (io.WriteCloser, error) Commit(ChainID) error @@ -208,6 +219,7 @@ type MetadataStore interface { GetParent(ChainID) (ChainID, error) GetDiffID(ChainID) (DiffID, error) GetCacheID(ChainID) (string, error) + GetForeignSource(ChainID) (distribution.Descriptor, error) TarSplitReader(ChainID) (io.ReadCloser, error) SetMountID(string, string) error diff --git a/layer/layer_store.go b/layer/layer_store.go index f18aff2145..fb87a8b793 100644 --- a/layer/layer_store.go +++ b/layer/layer_store.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -137,6 +138,13 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) { references: map[Layer]struct{}{}, } + foreignSrc, err := ls.store.GetForeignSource(layer) + if err == nil { + cl.foreignSrc = &foreignSrc + } else if err != ErrNoForeignSource { + return nil, fmt.Errorf("failed to get foreign reference for %s: %s", layer, err) + } + if parent != "" { p, err := ls.loadLayer(parent) if err != nil { @@ -228,6 +236,10 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri } func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) { + return ls.RegisterForeign(ts, parent, nil) +} + +func (ls *layerStore) RegisterForeign(ts io.Reader, parent ChainID, foreignSrc *distribution.Descriptor) (Layer, error) { // err is used to hold the error which will always trigger // cleanup of creates sources but may not be an error returned // to the caller (already exists). @@ -258,6 +270,7 @@ func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) { layer := &roLayer{ parent: p, cacheID: stringid.GenerateRandomID(), + foreignSrc: foreignSrc, referenceCount: 1, layerStore: ls, references: map[Layer]struct{}{}, diff --git a/layer/layer_unix.go b/layer/layer_unix.go index d77e2fc66e..7d6529e5a9 100644 --- a/layer/layer_unix.go +++ b/layer/layer_unix.go @@ -7,3 +7,9 @@ import "github.com/docker/docker/pkg/stringid" func (ls *layerStore) mountID(name string) string { return stringid.GenerateRandomID() } + +// ForeignSourceSupported returns whether layers downloaded from foreign sources are +// supported in this daemon. +func ForeignSourceSupported() bool { + return false +} diff --git a/layer/layer_windows.go b/layer/layer_windows.go index e20311a091..6d081649b9 100644 --- a/layer/layer_windows.go +++ b/layer/layer_windows.go @@ -96,3 +96,9 @@ func (ls *layerStore) mountID(name string) string { func (ls *layerStore) GraphDriver() graphdriver.Driver { return ls.driver } + +// ForeignSourceSupported returns whether layers downloaded from foreign sources are +// supported in this daemon. +func ForeignSourceSupported() bool { + return true +} diff --git a/layer/ro_layer.go b/layer/ro_layer.go index 92b0ea0ee2..1a180064fd 100644 --- a/layer/ro_layer.go +++ b/layer/ro_layer.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/docker/distribution" "github.com/docker/distribution/digest" ) @@ -14,6 +15,7 @@ type roLayer struct { cacheID string size int64 layerStore *layerStore + foreignSrc *distribution.Descriptor referenceCount int references map[Layer]struct{} @@ -123,6 +125,11 @@ func storeLayer(tx MetadataTransaction, layer *roLayer) error { return err } } + if layer.foreignSrc != nil { + if err := tx.SetForeignSource(*layer.foreignSrc); err != nil { + return err + } + } return nil } diff --git a/layer/ro_layer_windows.go b/layer/ro_layer_windows.go new file mode 100644 index 0000000000..797cb6d8a2 --- /dev/null +++ b/layer/ro_layer_windows.go @@ -0,0 +1,9 @@ +package layer + +import "github.com/docker/distribution" + +var _ ForeignSourcer = &roLayer{} + +func (rl *roLayer) ForeignSource() *distribution.Descriptor { + return rl.foreignSrc +}