Use ocischema package instead of custom handler

Previously we were re-using schema2.DeserializedManifest to handle oci
manifests. The issue lies in the fact that distribution started
validating the media type string during json deserialization. This
change broke our usage of that type.

Instead distribution now provides direct support for oci schemas, so use
that instead of our custom handlers.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

distribution/oci.go

@@ -1,29 +0,0 @@
-package distribution
-
-import (
-	"fmt"
-
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema2"
-	digest "github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-)
-
-func init() {
-	// TODO: Remove this registration if distribution is included with OCI support
-
-	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
-		m := new(schema2.DeserializedManifest)
-		err := m.UnmarshalJSON(b)
-		if err != nil {
-			return nil, distribution.Descriptor{}, err
-		}
-
-		dgst := digest.FromBytes(b)
-		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: ocispec.MediaTypeImageManifest}, err
-	}
-	err := distribution.RegisterManifestSchema(ocispec.MediaTypeImageManifest, ocischemaFunc)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to register manifest: %s", err))
-	}
-}

distribution/pull_v2.go

@@ -14,6 +14,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
@@ -410,6 +411,11 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform
 		if err != nil {
 			return false, err
 		}
+	case *ocischema.DeserializedManifest:
+		id, manifestDigest, err = p.pullOCI(ctx, ref, v, platform)
+		if err != nil {
+			return false, err
+		}
 	case *manifestlist.DeserializedManifestList:
 		id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform)
 		if err != nil {
@@ -557,24 +563,18 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
 	return imageID, manifestDigest, nil
 }
 
-func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
-	manifestDigest, err = schema2ManifestDigest(ref, mfst)
-	if err != nil {
-		return "", "", err
-	}
-
-	target := mfst.Target()
+func (p *v2Puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) {
 	if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
 		// If the image already exists locally, no need to pull
 		// anything.
-		return target.Digest, manifestDigest, nil
+		return target.Digest, nil
 	}
 
 	var descriptors []xfer.DownloadDescriptor
 
 	// Note that the order of this loop is in the direction of bottom-most
 	// to top-most, so that the downloads slice gets ordered correctly.
-	for _, d := range mfst.Layers {
+	for _, d := range layers {
 		layerDescriptor := &v2LayerDescriptor{
 			digest:            d.Digest,
 			repo:              p.repo,
@@ -629,23 +629,23 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 	if runtime.GOOS == "windows" {
 		configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
 		if err != nil {
-			return "", "", err
+			return "", err
 		}
 		if configRootFS == nil {
-			return "", "", errRootFSInvalid
+			return "", errRootFSInvalid
 		}
 		if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil {
-			return "", "", err
+			return "", err
 		}
 
 		if len(descriptors) != len(configRootFS.DiffIDs) {
-			return "", "", errRootFSMismatch
+			return "", errRootFSMismatch
 		}
 		if platform == nil {
 			// Early bath if the requested OS doesn't match that of the configuration.
 			// This avoids doing the download, only to potentially fail later.
 			if !system.IsOSSupported(configPlatform.OS) {
-				return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
+				return "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
 			}
 			layerStoreOS = configPlatform.OS
 		}
@@ -692,14 +692,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 			case <-downloadsDone:
 			case <-layerErrChan:
 			}
-			return "", "", err
+			return "", err
 		}
 	}
 
 	select {
 	case <-downloadsDone:
 	case err = <-layerErrChan:
-		return "", "", err
+		return "", err
 	}
 
 	if release != nil {
@@ -711,22 +711,40 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 		// Otherwise the image config could be referencing layers that aren't
 		// included in the manifest.
 		if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
-			return "", "", errRootFSMismatch
+			return "", errRootFSMismatch
 		}
 
 		for i := range downloadedRootFS.DiffIDs {
 			if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
-				return "", "", errRootFSMismatch
+				return "", errRootFSMismatch
 			}
 		}
 	}
 
 	imageID, err := p.config.ImageStore.Put(configJSON)
 	if err != nil {
-		return "", "", err
+		return "", err
 	}
 
-	return imageID, manifestDigest, nil
+	return imageID, nil
+}
+
+func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
+	if err != nil {
+		return "", "", err
+	}
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
+	return id, manifestDigest, err
+}
+
+func (p *v2Puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
+	if err != nil {
+		return "", "", err
+	}
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
+	return id, manifestDigest, err
 }
 
 func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) {
@@ -811,6 +829,12 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
 		if err != nil {
 			return "", "", err
 		}
+	case *ocischema.DeserializedManifest:
+		platform := toOCIPlatform(manifestMatches[0].Platform)
+		id, _, err = p.pullOCI(ctx, manifestRef, v, &platform)
+		if err != nil {
+			return "", "", err
+		}
 	default:
 		return "", "", errors.New("unsupported manifest format")
 	}

vendor/github.com/docker/distribution/manifest/ocischema/builder.go

@@ -0,0 +1,107 @@
+package ocischema
+
+import (
+	"context"
+	"errors"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Builder is a type for constructing manifests.
+type Builder struct {
+	// bs is a BlobService used to publish the configuration blob.
+	bs distribution.BlobService
+
+	// configJSON references
+	configJSON []byte
+
+	// layers is a list of layer descriptors that gets built by successive
+	// calls to AppendReference.
+	layers []distribution.Descriptor
+
+	// Annotations contains arbitrary metadata relating to the targeted content.
+	annotations map[string]string
+
+	// For testing purposes
+	mediaType string
+}
+
+// NewManifestBuilder is used to build new manifests for the current schema
+// version. It takes a BlobService so it can publish the configuration blob
+// as part of the Build process, and annotations.
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
+	mb := &Builder{
+		bs:          bs,
+		configJSON:  make([]byte, len(configJSON)),
+		annotations: annotations,
+		mediaType:   v1.MediaTypeImageManifest,
+	}
+	copy(mb.configJSON, configJSON)
+
+	return mb
+}
+
+// SetMediaType assigns the passed mediatype or error if the mediatype is not a
+// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json"
+func (mb *Builder) SetMediaType(mediaType string) error {
+	if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
+		return errors.New("invalid media type for OCI image manifest")
+	}
+
+	mb.mediaType = mediaType
+	return nil
+}
+
+// Build produces a final manifest from the given references.
+func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
+	m := Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+			MediaType:     mb.mediaType,
+		},
+		Layers:      make([]distribution.Descriptor, len(mb.layers)),
+		Annotations: mb.annotations,
+	}
+	copy(m.Layers, mb.layers)
+
+	configDigest := digest.FromBytes(mb.configJSON)
+
+	var err error
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
+	switch err {
+	case nil:
+		// Override MediaType, since Put always replaces the specified media
+		// type with application/octet-stream in the descriptor it returns.
+		m.Config.MediaType = v1.MediaTypeImageConfig
+		return FromStruct(m)
+	case distribution.ErrBlobUnknown:
+		// nop
+	default:
+		return nil, err
+	}
+
+	// Add config to the blob store
+	m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON)
+	// Override MediaType, since Put always replaces the specified media
+	// type with application/octet-stream in the descriptor it returns.
+	m.Config.MediaType = v1.MediaTypeImageConfig
+	if err != nil {
+		return nil, err
+	}
+
+	return FromStruct(m)
+}
+
+// AppendReference adds a reference to the current ManifestBuilder.
+func (mb *Builder) AppendReference(d distribution.Describable) error {
+	mb.layers = append(mb.layers, d.Descriptor())
+	return nil
+}
+
+// References returns the current references added to this builder.
+func (mb *Builder) References() []distribution.Descriptor {
+	return mb.layers
+}

vendor/github.com/docker/distribution/manifest/ocischema/manifest.go

@@ -0,0 +1,124 @@
+package ocischema
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var (
+	// SchemaVersion provides a pre-initialized version structure for this
+	// packages version of the manifest.
+	SchemaVersion = manifest.Versioned{
+		SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version
+		MediaType:     v1.MediaTypeImageManifest,
+	}
+)
+
+func init() {
+	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		m := new(DeserializedManifest)
+		err := m.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		dgst := digest.FromBytes(b)
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err
+	}
+	err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+}
+
+// Manifest defines a ocischema manifest.
+type Manifest struct {
+	manifest.Versioned
+
+	// Config references the image configuration as a blob.
+	Config distribution.Descriptor `json:"config"`
+
+	// Layers lists descriptors for the layers referenced by the
+	// configuration.
+	Layers []distribution.Descriptor `json:"layers"`
+
+	// Annotations contains arbitrary metadata for the image manifest.
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// References returns the descriptors of this manifests references.
+func (m Manifest) References() []distribution.Descriptor {
+	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
+	references = append(references, m.Config)
+	references = append(references, m.Layers...)
+	return references
+}
+
+// Target returns the target of this manifest.
+func (m Manifest) Target() distribution.Descriptor {
+	return m.Config
+}
+
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
+// It satisfies the distribution.Manifest interface.
+type DeserializedManifest struct {
+	Manifest
+
+	// canonical is the canonical byte representation of the Manifest.
+	canonical []byte
+}
+
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
+// DeserializedManifest which contains the manifest and its JSON representation.
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
+	var deserialized DeserializedManifest
+	deserialized.Manifest = m
+
+	var err error
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
+	return &deserialized, err
+}
+
+// UnmarshalJSON populates a new Manifest struct from JSON data.
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
+	m.canonical = make([]byte, len(b))
+	// store manifest in canonical
+	copy(m.canonical, b)
+
+	// Unmarshal canonical JSON into Manifest object
+	var manifest Manifest
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
+		return err
+	}
+
+	if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
+		return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
+			v1.MediaTypeImageManifest, manifest.MediaType)
+	}
+
+	m.Manifest = manifest
+
+	return nil
+}
+
+// MarshalJSON returns the contents of canonical. If canonical is empty,
+// marshals the inner contents.
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
+	if len(m.canonical) > 0 {
+		return m.canonical, nil
+	}
+
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
+}
+
+// Payload returns the raw content of the manifest. The contents can be used to
+// calculate the content identifier.
+func (m DeserializedManifest) Payload() (string, []byte, error) {
+	return v1.MediaTypeImageManifest, m.canonical, nil
+}