From 4d437a29d28ec2d47ff7cf0868390b921ededa03 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 26 Jan 2016 11:20:14 -0800 Subject: [PATCH] Vendor updated distribution for resumable downloads Signed-off-by: Aaron Lehmann --- distribution/pull_v2.go | 2 +- distribution/push_v2.go | 8 +- distribution/registry.go | 8 +- hack/vendor.sh | 2 +- .../github.com/docker/distribution/.mailmap | 3 +- .../github.com/docker/distribution/AUTHORS | 9 ++ .../docker/distribution/MAINTAINERS | 5 + .../manifest/schema1/config_builder.go | 21 +-- .../manifest/schema1/reference_builder.go | 10 +- .../docker/distribution/manifests.go | 2 +- .../docker/distribution/registry.go | 7 +- .../distribution/registry/api/v2/urls.go | 30 ++-- .../registry/client/auth/session.go | 2 +- .../registry/client/repository.go | 148 +++++++++++++----- .../registry/client/transport/http_reader.go | 97 +++++++++--- 15 files changed, 256 insertions(+), 98 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 71a1c4e5fd..04d05e02f4 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -250,7 +250,7 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat p.confirmedV2 = true logrus.Debugf("Pulling ref from V2 registry: %s", ref.String()) - progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Name()) + progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+p.repo.Named().Name()) var ( imageID image.ID diff --git a/distribution/push_v2.go b/distribution/push_v2.go index e5f9de81f6..ac2017a8d0 100644 --- a/distribution/push_v2.go +++ b/distribution/push_v2.go @@ -166,7 +166,11 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, ima if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil { logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err) - builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, p.repo.Name(), ref.Tag(), img.RawJSON()) + manifestRef, err := distreference.WithTag(p.repo.Named(), ref.Tag()) + if err != nil { + return err + } + builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, img.RawJSON()) manifest, err = manifestFromBuilder(ctx, builder, descriptors) if err != nil { return err @@ -219,7 +223,7 @@ type v2PushDescriptor struct { } func (pd *v2PushDescriptor) Key() string { - return "v2push:" + pd.repo.Name() + " " + pd.layer.DiffID().String() + return "v2push:" + pd.repo.Named().Name() + " " + pd.layer.DiffID().String() } func (pd *v2PushDescriptor) ID() string { diff --git a/distribution/registry.go b/distribution/registry.go index 4f1f0ff805..6049be0466 100644 --- a/distribution/registry.go +++ b/distribution/registry.go @@ -10,6 +10,7 @@ import ( "time" "github.com/docker/distribution" + distreference "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/auth" @@ -154,7 +155,12 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end } tr := transport.NewTransport(base, modifiers...) - repo, err = client.NewRepository(ctx, repoName, endpoint.URL, tr) + repoNameRef, err := distreference.ParseNamed(repoName) + if err != nil { + return nil, foundVersion, err + } + + repo, err = client.NewRepository(ctx, repoNameRef, endpoint.URL, tr) return repo, foundVersion, err } diff --git a/hack/vendor.sh b/hack/vendor.sh index 5c91621f26..0a8b761738 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -46,7 +46,7 @@ clone git github.com/boltdb/bolt v1.1.0 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages -clone git github.com/docker/distribution c301f8ab27f4913c968b8d73a38e5dda79b9d3d7 +clone git github.com/docker/distribution ab9b433fcaf7c8319562a8b80f2720f5faca712f clone git github.com/vbatts/tar-split v0.9.11 # get desired notary commit, might also need to be updated in Dockerfile diff --git a/vendor/src/github.com/docker/distribution/.mailmap b/vendor/src/github.com/docker/distribution/.mailmap index 191e60cdac..e4e50ee793 100644 --- a/vendor/src/github.com/docker/distribution/.mailmap +++ b/vendor/src/github.com/docker/distribution/.mailmap @@ -2,6 +2,7 @@ Stephen J Day Stephen Day Stephen Day Olivier Gambier Olivier Gambier Brian Bland Brian Bland +Brian Bland Brian Bland Josh Hawn Josh Hawn Richard Scothern Richard Richard Scothern Richard Scothern @@ -11,4 +12,4 @@ Jessie Frazelle Sharif Nassar Sharif Nassar Sven Dowideit Sven Dowideit Vincent Giersch Vincent Giersch -davidli davidli \ No newline at end of file +davidli davidli diff --git a/vendor/src/github.com/docker/distribution/AUTHORS b/vendor/src/github.com/docker/distribution/AUTHORS index 4b97cd78d5..a44266b0c2 100644 --- a/vendor/src/github.com/docker/distribution/AUTHORS +++ b/vendor/src/github.com/docker/distribution/AUTHORS @@ -21,6 +21,7 @@ Ben Firshman bin liu Brian Bland burnettk +Carson A Chris Dillon Daisuke Fujita Darren Shepherd @@ -33,11 +34,13 @@ davidli Dejan Golja Derek McGowan Diogo Mónica +DJ Enriquez Donald Huang Doug Davis farmerworking Florentin Raud Frederick F. Kautz IV +gabriell nascimento harche Henri Gomez Hu Keping @@ -55,7 +58,9 @@ Josh Hawn Julien Fernandez Kelsey Hightower Kenneth Lim +Kenny Leung Li Yi +Liu Hua Louis Kottmann Luke Carpenter Mary Anthony @@ -76,7 +81,9 @@ Olivier Jacques Patrick Devine Philip Misiowiec Richard Scothern +Rodolfo Carvalho Rusty Conover +Sean Boran Sebastiaan van Stijn Sharif Nassar Shawn Falkner-Horine @@ -93,11 +100,13 @@ Thomas Sjögren Tianon Gravi Tibor Vass Tonis Tiigi +Trevor Pounds Troels Thomsen Vincent Batts Vincent Demeester Vincent Giersch W. Trevor King +weiyuan.yl xg.song xiekeyang Yann ROBERT diff --git a/vendor/src/github.com/docker/distribution/MAINTAINERS b/vendor/src/github.com/docker/distribution/MAINTAINERS index bda400150c..97f415dbfb 100644 --- a/vendor/src/github.com/docker/distribution/MAINTAINERS +++ b/vendor/src/github.com/docker/distribution/MAINTAINERS @@ -32,6 +32,11 @@ Email = "aaron.lehmann@docker.com" GitHub = "aaronlehmann" + [people.brianbland] + Name = "Brian Bland" + Email = "brian.bland@docker.com" + GitHub = "BrianBland" + [people.dmcgowan] Name = "Derek McGowan" Email = "derek@mcgstyle.net" diff --git a/vendor/src/github.com/docker/distribution/manifest/schema1/config_builder.go b/vendor/src/github.com/docker/distribution/manifest/schema1/config_builder.go index e9fe81be67..b3d1e55431 100644 --- a/vendor/src/github.com/docker/distribution/manifest/schema1/config_builder.go +++ b/vendor/src/github.com/docker/distribution/manifest/schema1/config_builder.go @@ -9,6 +9,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" + "github.com/docker/distribution/reference" "github.com/docker/libtrust" "github.com/docker/distribution/digest" @@ -39,10 +40,8 @@ type configManifestBuilder struct { // configJSON is configuration supplied when the ManifestBuilder was // created. configJSON []byte - // name is the name provided to NewConfigManifestBuilder - name string - // tag is the tag provided to NewConfigManifestBuilder - tag string + // ref contains the name and optional tag provided to NewConfigManifestBuilder. + ref reference.Named // descriptors is the set of descriptors referencing the layers. descriptors []distribution.Descriptor // emptyTarDigest is set to a valid digest if an empty tar has been @@ -54,13 +53,12 @@ type configManifestBuilder struct { // schema version from an image configuration and a set of descriptors. // It takes a BlobService so that it can add an empty tar to the blob store // if the resulting manifest needs empty layers. -func NewConfigManifestBuilder(bs distribution.BlobService, pk libtrust.PrivateKey, name, tag string, configJSON []byte) distribution.ManifestBuilder { +func NewConfigManifestBuilder(bs distribution.BlobService, pk libtrust.PrivateKey, ref reference.Named, configJSON []byte) distribution.ManifestBuilder { return &configManifestBuilder{ bs: bs, pk: pk, configJSON: configJSON, - name: name, - tag: tag, + ref: ref, } } @@ -190,12 +188,17 @@ func (mb *configManifestBuilder) Build(ctx context.Context) (m distribution.Mani history[0].V1Compatibility = string(transformedConfig) + tag := "" + if tagged, isTagged := mb.ref.(reference.Tagged); isTagged { + tag = tagged.Tag() + } + mfst := Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, - Name: mb.name, - Tag: mb.tag, + Name: mb.ref.Name(), + Tag: tag, Architecture: img.Architecture, FSLayers: fsLayerList, History: history, diff --git a/vendor/src/github.com/docker/distribution/manifest/schema1/reference_builder.go b/vendor/src/github.com/docker/distribution/manifest/schema1/reference_builder.go index 36209e3423..fc1045f9ea 100644 --- a/vendor/src/github.com/docker/distribution/manifest/schema1/reference_builder.go +++ b/vendor/src/github.com/docker/distribution/manifest/schema1/reference_builder.go @@ -8,6 +8,7 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" + "github.com/docker/distribution/reference" "github.com/docker/libtrust" ) @@ -20,13 +21,18 @@ type referenceManifestBuilder struct { // NewReferenceManifestBuilder is used to build new manifests for the current // schema version using schema1 dependencies. -func NewReferenceManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder { +func NewReferenceManifestBuilder(pk libtrust.PrivateKey, ref reference.Named, architecture string) distribution.ManifestBuilder { + tag := "" + if tagged, isTagged := ref.(reference.Tagged); isTagged { + tag = tagged.Tag() + } + return &referenceManifestBuilder{ Manifest: Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, - Name: name, + Name: ref.Name(), Tag: tag, Architecture: architecture, }, diff --git a/vendor/src/github.com/docker/distribution/manifests.go b/vendor/src/github.com/docker/distribution/manifests.go index 1acb0500d7..40c5622f08 100644 --- a/vendor/src/github.com/docker/distribution/manifests.go +++ b/vendor/src/github.com/docker/distribution/manifests.go @@ -81,7 +81,7 @@ type UnmarshalFunc func([]byte) (Manifest, Descriptor, error) var mappings = make(map[string]UnmarshalFunc, 0) -// UnmarshalManifest looks up manifest unmarshall functions based on +// UnmarshalManifest looks up manifest unmarshal functions based on // MediaType func UnmarshalManifest(ctHeader string, p []byte) (Manifest, Descriptor, error) { // Need to look up by the actual media type, not the raw contents of diff --git a/vendor/src/github.com/docker/distribution/registry.go b/vendor/src/github.com/docker/distribution/registry.go index ce5d777925..37dbb49be9 100644 --- a/vendor/src/github.com/docker/distribution/registry.go +++ b/vendor/src/github.com/docker/distribution/registry.go @@ -2,6 +2,7 @@ package distribution import ( "github.com/docker/distribution/context" + "github.com/docker/distribution/reference" ) // Scope defines the set of items that match a namespace. @@ -32,7 +33,7 @@ type Namespace interface { // Repository should return a reference to the named repository. The // registry may or may not have the repository but should always return a // reference. - Repository(ctx context.Context, name string) (Repository, error) + Repository(ctx context.Context, name reference.Named) (Repository, error) // Repositories fills 'repos' with a lexigraphically sorted catalog of repositories // up to the size of 'repos' and returns the value 'n' for the number of entries @@ -48,8 +49,8 @@ type ManifestServiceOption interface { // Repository is a named collection of manifests and layers. type Repository interface { - // Name returns the name of the repository. - Name() string + // Named returns the name of the repository. + Named() reference.Named // Manifests returns a reference to this repository's manifest service. // with the supplied options applied. diff --git a/vendor/src/github.com/docker/distribution/registry/api/v2/urls.go b/vendor/src/github.com/docker/distribution/registry/api/v2/urls.go index 6ba39cc9bd..408c7b74bb 100644 --- a/vendor/src/github.com/docker/distribution/registry/api/v2/urls.go +++ b/vendor/src/github.com/docker/distribution/registry/api/v2/urls.go @@ -5,7 +5,7 @@ import ( "net/url" "strings" - "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" "github.com/gorilla/mux" ) @@ -113,10 +113,10 @@ func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) { } // BuildTagsURL constructs a url to list the tags in the named repository. -func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { +func (ub *URLBuilder) BuildTagsURL(name reference.Named) (string, error) { route := ub.cloneRoute(RouteNameTags) - tagsURL, err := route.URL("name", name) + tagsURL, err := route.URL("name", name.Name()) if err != nil { return "", err } @@ -126,10 +126,18 @@ func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { // BuildManifestURL constructs a url for the manifest identified by name and // reference. The argument reference may be either a tag or digest. -func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { +func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) { route := ub.cloneRoute(RouteNameManifest) - manifestURL, err := route.URL("name", name, "reference", reference) + tagOrDigest := "" + switch v := ref.(type) { + case reference.Tagged: + tagOrDigest = v.Tag() + case reference.Digested: + tagOrDigest = v.Digest().String() + } + + manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest) if err != nil { return "", err } @@ -138,10 +146,10 @@ func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { } // BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { +func (ub *URLBuilder) BuildBlobURL(ref reference.Canonical) (string, error) { route := ub.cloneRoute(RouteNameBlob) - layerURL, err := route.URL("name", name, "digest", dgst.String()) + layerURL, err := route.URL("name", ref.Name(), "digest", ref.Digest().String()) if err != nil { return "", err } @@ -151,10 +159,10 @@ func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, err // BuildBlobUploadURL constructs a url to begin a blob upload in the // repository identified by name. -func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { +func (ub *URLBuilder) BuildBlobUploadURL(name reference.Named, values ...url.Values) (string, error) { route := ub.cloneRoute(RouteNameBlobUpload) - uploadURL, err := route.URL("name", name) + uploadURL, err := route.URL("name", name.Name()) if err != nil { return "", err } @@ -166,10 +174,10 @@ func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (str // including any url values. This should generally not be used by clients, as // this url is provided by server implementations during the blob upload // process. -func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { +func (ub *URLBuilder) BuildBlobUploadChunkURL(name reference.Named, uuid string, values ...url.Values) (string, error) { route := ub.cloneRoute(RouteNameBlobUploadChunk) - uploadURL, err := route.URL("name", name, "uuid", uuid) + uploadURL, err := route.URL("name", name.Name(), "uuid", uuid) if err != nil { return "", err } diff --git a/vendor/src/github.com/docker/distribution/registry/client/auth/session.go b/vendor/src/github.com/docker/distribution/registry/client/auth/session.go index 6b483c62ef..50a94a3da3 100644 --- a/vendor/src/github.com/docker/distribution/registry/client/auth/session.go +++ b/vendor/src/github.com/docker/distribution/registry/client/auth/session.go @@ -285,9 +285,9 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon } if tr.ExpiresIn < minimumTokenLifetimeSeconds { - logrus.Debugf("Increasing token expiration to: %d seconds", tr.ExpiresIn) // The default/minimum lifetime. tr.ExpiresIn = minimumTokenLifetimeSeconds + logrus.Debugf("Increasing token expiration to: %d seconds", tr.ExpiresIn) } if tr.IssuedAt.IsZero() { diff --git a/vendor/src/github.com/docker/distribution/registry/client/repository.go b/vendor/src/github.com/docker/distribution/registry/client/repository.go index d652121109..1e8c4fa98e 100644 --- a/vendor/src/github.com/docker/distribution/registry/client/repository.go +++ b/vendor/src/github.com/docker/distribution/registry/client/repository.go @@ -27,6 +27,26 @@ type Registry interface { Repositories(ctx context.Context, repos []string, last string) (n int, err error) } +// checkHTTPRedirect is a callback that can manipulate redirected HTTP +// requests. It is used to preserve Accept and Range headers. +func checkHTTPRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + + if len(via) > 0 { + for headerName, headerVals := range via[0].Header { + if headerName == "Accept" || headerName == "Range" { + for _, val := range headerVals { + req.Header.Add(headerName, val) + } + } + } + } + + return nil +} + // NewRegistry creates a registry namespace which can be used to get a listing of repositories func NewRegistry(ctx context.Context, baseURL string, transport http.RoundTripper) (Registry, error) { ub, err := v2.NewURLBuilderFromString(baseURL) @@ -35,8 +55,9 @@ func NewRegistry(ctx context.Context, baseURL string, transport http.RoundTrippe } client := &http.Client{ - Transport: transport, - Timeout: 1 * time.Minute, + Transport: transport, + Timeout: 1 * time.Minute, + CheckRedirect: checkHTTPRedirect, } return ®istry{ @@ -98,18 +119,15 @@ func (r *registry) Repositories(ctx context.Context, entries []string, last stri } // NewRepository creates a new Repository for the given repository name and base URL. -func NewRepository(ctx context.Context, name, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { - if _, err := reference.ParseNamed(name); err != nil { - return nil, err - } - +func NewRepository(ctx context.Context, name reference.Named, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { ub, err := v2.NewURLBuilderFromString(baseURL) if err != nil { return nil, err } client := &http.Client{ - Transport: transport, + Transport: transport, + CheckRedirect: checkHTTPRedirect, // TODO(dmcgowan): create cookie jar } @@ -125,21 +143,21 @@ type repository struct { client *http.Client ub *v2.URLBuilder context context.Context - name string + name reference.Named } -func (r *repository) Name() string { +func (r *repository) Named() reference.Named { return r.name } func (r *repository) Blobs(ctx context.Context) distribution.BlobStore { statter := &blobStatter{ - name: r.Name(), + name: r.name, ub: r.ub, client: r.client, } return &blobs{ - name: r.Name(), + name: r.name, ub: r.ub, client: r.client, statter: cache.NewCachedBlobStatter(memory.NewInMemoryBlobDescriptorCacheProvider(), statter), @@ -149,7 +167,7 @@ func (r *repository) Blobs(ctx context.Context) distribution.BlobStore { func (r *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { // todo(richardscothern): options should be sent over the wire return &manifests{ - name: r.Name(), + name: r.name, ub: r.ub, client: r.client, etags: make(map[string]string), @@ -161,7 +179,7 @@ func (r *repository) Tags(ctx context.Context) distribution.TagService { client: r.client, ub: r.ub, context: r.context, - name: r.Name(), + name: r.Named(), } } @@ -170,7 +188,7 @@ type tags struct { client *http.Client ub *v2.URLBuilder context context.Context - name string + name reference.Named } // All returns all tags @@ -253,7 +271,11 @@ func descriptorFromResponse(response *http.Response) (distribution.Descriptor, e // to construct a descriptor for the tag. If the registry doesn't support HEADing // a manifest, fallback to GET. func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { - u, err := t.ub.BuildManifestURL(t.name, tag) + ref, err := reference.WithTag(t.name, tag) + if err != nil { + return distribution.Descriptor{}, err + } + u, err := t.ub.BuildManifestURL(ref) if err != nil { return distribution.Descriptor{}, err } @@ -293,14 +315,18 @@ func (t *tags) Untag(ctx context.Context, tag string) error { } type manifests struct { - name string + name reference.Named ub *v2.URLBuilder client *http.Client etags map[string]string } func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { - u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) + ref, err := reference.WithDigest(ms.name, dgst) + if err != nil { + return false, err + } + u, err := ms.ub.BuildManifestURL(ref) if err != nil { return false, err } @@ -337,11 +363,19 @@ func (o etagOption) Apply(ms distribution.ManifestService) error { } func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + var ( + digestOrTag string + ref reference.Named + err error + ) - var tag string for _, option := range options { if opt, ok := option.(withTagOption); ok { - tag = opt.tag + digestOrTag = opt.tag + ref, err = reference.WithTag(ms.name, opt.tag) + if err != nil { + return nil, err + } } else { err := option.Apply(ms) if err != nil { @@ -350,14 +384,15 @@ func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...dis } } - var ref string - if tag != "" { - ref = tag - } else { - ref = dgst.String() + if digestOrTag == "" { + digestOrTag = dgst.String() + ref, err = reference.WithDigest(ms.name, dgst) + if err != nil { + return nil, err + } } - u, err := ms.ub.BuildManifestURL(ms.name, ref) + u, err := ms.ub.BuildManifestURL(ref) if err != nil { return nil, err } @@ -371,8 +406,8 @@ func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...dis req.Header.Add("Accept", t) } - if _, ok := ms.etags[ref]; ok { - req.Header.Set("If-None-Match", ms.etags[ref]) + if _, ok := ms.etags[digestOrTag]; ok { + req.Header.Set("If-None-Match", ms.etags[digestOrTag]) } resp, err := ms.client.Do(req) @@ -414,13 +449,19 @@ func (o withTagOption) Apply(m distribution.ManifestService) error { } // Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the -// tag name in order to build the correct upload URL. This state is written and read under a lock. +// tag name in order to build the correct upload URL. func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { - var tag string + ref := ms.name + var tagged bool for _, option := range options { if opt, ok := option.(withTagOption); ok { - tag = opt.tag + var err error + ref, err = reference.WithTag(ref, opt.tag) + if err != nil { + return "", err + } + tagged = true } else { err := option.Apply(ms) if err != nil { @@ -428,13 +469,24 @@ func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options . } } } - - manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag) + mediaType, p, err := m.Payload() if err != nil { return "", err } - mediaType, p, err := m.Payload() + if !tagged { + // generate a canonical digest and Put by digest + _, d, err := distribution.UnmarshalManifest(mediaType, p) + if err != nil { + return "", err + } + ref, err = reference.WithDigest(ref, d.Digest) + if err != nil { + return "", err + } + } + + manifestURL, err := ms.ub.BuildManifestURL(ref) if err != nil { return "", err } @@ -466,7 +518,11 @@ func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options . } func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error { - u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) + ref, err := reference.WithDigest(ms.name, dgst) + if err != nil { + return err + } + u, err := ms.ub.BuildManifestURL(ref) if err != nil { return err } @@ -493,7 +549,7 @@ func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error { }*/ type blobs struct { - name string + name reference.Named ub *v2.URLBuilder client *http.Client @@ -531,7 +587,11 @@ func (bs *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { } func (bs *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { - blobURL, err := bs.ub.BuildBlobURL(bs.name, dgst) + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return nil, err + } + blobURL, err := bs.ub.BuildBlobURL(ref) if err != nil { return nil, err } @@ -666,13 +726,17 @@ func (bs *blobs) Delete(ctx context.Context, dgst digest.Digest) error { } type blobStatter struct { - name string + name reference.Named ub *v2.URLBuilder client *http.Client } func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - u, err := bs.ub.BuildBlobURL(bs.name, dgst) + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + u, err := bs.ub.BuildBlobURL(ref) if err != nil { return distribution.Descriptor{}, err } @@ -720,7 +784,11 @@ func buildCatalogValues(maxEntries int, last string) url.Values { } func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error { - blobURL, err := bs.ub.BuildBlobURL(bs.name, dgst) + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return err + } + blobURL, err := bs.ub.BuildBlobURL(ref) if err != nil { return err } diff --git a/vendor/src/github.com/docker/distribution/registry/client/transport/http_reader.go b/vendor/src/github.com/docker/distribution/registry/client/transport/http_reader.go index b27b6c2379..22b0b9d69a 100644 --- a/vendor/src/github.com/docker/distribution/registry/client/transport/http_reader.go +++ b/vendor/src/github.com/docker/distribution/registry/client/transport/http_reader.go @@ -1,12 +1,22 @@ package transport import ( - "bufio" "errors" "fmt" "io" "net/http" "os" + "regexp" + "strconv" +) + +var ( + contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`) + + // ErrWrongCodeForByteRange is returned if the client sends a request + // with a Range header but the server returns a 2xx or 3xx code other + // than 206 Partial Content. + ErrWrongCodeForByteRange = errors.New("expected HTTP 206 from byte range request") ) // ReadSeekCloser combines io.ReadSeeker with io.Closer. @@ -40,8 +50,6 @@ type httpReadSeeker struct { // rc is the remote read closer. rc io.ReadCloser - // brd is a buffer for internal buffered io. - brd *bufio.Reader // readerOffset tracks the offset as of the last read. readerOffset int64 // seekOffset allows Seek to override the offset. Seek changes @@ -79,11 +87,6 @@ func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) { hrs.seekOffset += int64(n) hrs.readerOffset += int64(n) - // Simulate io.EOF error if we reach filesize. - if err == nil && hrs.size >= 0 && hrs.readerOffset >= hrs.size { - err = io.EOF - } - return n, err } @@ -92,8 +95,18 @@ func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { return 0, hrs.err } + lastReaderOffset := hrs.readerOffset + + if whence == os.SEEK_SET && hrs.rc == nil { + // If no request has been made yet, and we are seeking to an + // absolute position, set the read offset as well to avoid an + // unnecessary request. + hrs.readerOffset = offset + } + _, err := hrs.reader() if err != nil { + hrs.readerOffset = lastReaderOffset return 0, err } @@ -101,14 +114,14 @@ func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { switch whence { case os.SEEK_CUR: - newOffset += int64(offset) + newOffset += offset case os.SEEK_END: if hrs.size < 0 { return 0, errors.New("content length not known") } - newOffset = hrs.size + int64(offset) + newOffset = hrs.size + offset case os.SEEK_SET: - newOffset = int64(offset) + newOffset = offset } if newOffset < 0 { @@ -131,7 +144,6 @@ func (hrs *httpReadSeeker) Close() error { } hrs.rc = nil - hrs.brd = nil hrs.err = errors.New("httpLayer: closed") @@ -154,7 +166,7 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { } if hrs.rc != nil { - return hrs.brd, nil + return hrs.rc, nil } req, err := http.NewRequest("GET", hrs.url, nil) @@ -163,10 +175,8 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { } if hrs.readerOffset > 0 { - // TODO(stevvooe): Get this working correctly. - // If we are at different offset, issue a range request from there. - req.Header.Add("Range", "1-") + req.Header.Add("Range", fmt.Sprintf("bytes=%d-", hrs.readerOffset)) // TODO: get context in here // context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range")) } @@ -179,12 +189,55 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { // Normally would use client.SuccessStatus, but that would be a cyclic // import if resp.StatusCode >= 200 && resp.StatusCode <= 399 { - hrs.rc = resp.Body - if resp.StatusCode == http.StatusOK { + if hrs.readerOffset > 0 { + if resp.StatusCode != http.StatusPartialContent { + return nil, ErrWrongCodeForByteRange + } + + contentRange := resp.Header.Get("Content-Range") + if contentRange == "" { + return nil, errors.New("no Content-Range header found in HTTP 206 response") + } + + submatches := contentRangeRegexp.FindStringSubmatch(contentRange) + if len(submatches) < 4 { + return nil, fmt.Errorf("could not parse Content-Range header: %s", contentRange) + } + + startByte, err := strconv.ParseUint(submatches[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse start of range in Content-Range header: %s", contentRange) + } + + if startByte != uint64(hrs.readerOffset) { + return nil, fmt.Errorf("received Content-Range starting at offset %d instead of requested %d", startByte, hrs.readerOffset) + } + + endByte, err := strconv.ParseUint(submatches[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse end of range in Content-Range header: %s", contentRange) + } + + if submatches[3] == "*" { + hrs.size = -1 + } else { + size, err := strconv.ParseUint(submatches[3], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse total size in Content-Range header: %s", contentRange) + } + + if endByte+1 != size { + return nil, fmt.Errorf("range in Content-Range stops before the end of the content: %s", contentRange) + } + + hrs.size = int64(size) + } + } else if resp.StatusCode == http.StatusOK { hrs.size = resp.ContentLength } else { hrs.size = -1 } + hrs.rc = resp.Body } else { defer resp.Body.Close() if hrs.errorHandler != nil { @@ -193,11 +246,5 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) } - if hrs.brd == nil { - hrs.brd = bufio.NewReader(hrs.rc) - } else { - hrs.brd.Reset(hrs.rc) - } - - return hrs.brd, nil + return hrs.rc, nil }