kotovalexarian-likes-github
/
moby--moby
mirror of https://github.com/moby/moby.git
1
0
Fork 0
Code Releases Activity

Support layers from external URLs 

This is used to support downloading Windows base images from Microsoft
servers.

Signed-off-by: John Starks <jostarks@microsoft.com>
This commit is contained in:
John Starks 2016-05-25 19:11:51 -07:00
parent 9c902364fb
commit 05bd04350b
16 changed files with 231 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
distribution/pull_v2.go
View File

@ -133,6 +133,7 @@ type v2LayerDescriptor struct {
V2MetadataService *metadata.V2MetadataService
tmpFile *os.File
verifier digest.Verifier
foreignSrc *distribution.Descriptor
}
func (ld *v2LayerDescriptor) Key() string {
@ -180,9 +181,8 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre
}
tmpFile := ld.tmpFile
blobs := ld.repo.Blobs(ctx)
layerDownload, err := blobs.Open(ctx, ld.digest)
layerDownload, err := ld.open(ctx)
if err != nil {
logrus.Errorf("Error initiating layer download: %v", err)
if err == distribution.ErrBlobUnknown {
@ -501,6 +501,29 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
return imageID, manifestDigest, nil
}
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.Layers {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
repoInfo: p.repoInfo,
V2MetadataService: p.V2MetadataService,
}
if d.MediaType == schema2.MediaTypeForeignLayer && len(d.URLs) > 0 {
if !layer.ForeignSourceSupported() {
return "", "", errors.New("foreign layers are not supported on this OS")
}
layerDescriptor.foreignSrc = &d
}
descriptors = append(descriptors, layerDescriptor)
}
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
@ -517,21 +540,6 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
configChan <- configJSON
}()
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for _, d := range mfst.References() {
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
repoInfo: p.repoInfo,
V2MetadataService: p.V2MetadataService,
}
descriptors = append(descriptors, layerDescriptor)
}
var (
configJSON []byte // raw serialized image config
unmarshalledConfig image.Image // deserialized image config

7
distribution/pull_v2_unix.go
View File

@ -3,6 +3,8 @@
package distribution
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
)
@ -10,3 +12,8 @@ import (
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
return nil
}
func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) {
blobs := ld.repo.Blobs(ctx)
return blobs.Open(ctx, ld.digest)
}

36
distribution/pull_v2_windows.go
View File

@ -5,9 +5,15 @@ package distribution
import (
"encoding/json"
"fmt"
"net/http"
"os"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
)
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
@ -28,3 +34,33 @@ func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS)
}
return fmt.Errorf("Invalid base layer %q", v1img.Parent)
}
var _ layer.ForeignSourcer = &v2LayerDescriptor{}
func (ld *v2LayerDescriptor) ForeignSource() *distribution.Descriptor {
return ld.foreignSrc
}
func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) {
if ld.foreignSrc == nil {
blobs := ld.repo.Blobs(ctx)
return blobs.Open(ctx, ld.digest)
}
var (
err error
rsc distribution.ReadSeekCloser
)
// Find the first URL that results in a 200 result code.
for _, url := range ld.foreignSrc.URLs {
rsc = transport.NewHTTPReadSeeker(http.DefaultClient, url, nil)
_, err = rsc.Seek(0, os.SEEK_SET)
if err == nil {
break
}
rsc.Close()
rsc = nil
}
return rsc, err
}

7
distribution/push_v2.go
View File

@ -240,6 +240,13 @@ func (pd *v2PushDescriptor) DiffID() layer.DiffID {
}
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
if fs, ok := pd.layer.(layer.ForeignSourcer); ok {
if d := fs.ForeignSource(); d != nil {
progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
return *d, nil
}
}
diffID := pd.DiffID()
pd.pushState.Lock()

13
distribution/xfer/download.go
View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive"
@ -318,7 +319,11 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor,
return
}
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
var src *distribution.Descriptor
if fs, ok := descriptor.(layer.ForeignSourcer); ok {
src = fs.ForeignSource()
}
d.layer, err = d.layerStore.RegisterForeign(inflatedLayerData, parentLayer, src)
if err != nil {
select {
case <-d.Transfer.Context().Done():
@ -409,7 +414,11 @@ func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor Downloa
}
defer layerReader.Close()
d.layer, err = d.layerStore.Register(layerReader, parentLayer)
var src *distribution.Descriptor
if fs, ok := l.(layer.ForeignSourcer); ok {
src = fs.ForeignSource()
}
d.layer, err = d.layerStore.RegisterForeign(layerReader, parentLayer, src)
if err != nil {
d.err = fmt.Errorf("failed to register layer: %v", err)
return

5
distribution/xfer/download_test.go
View File

@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
@ -71,6 +72,10 @@ func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.
}
func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
return ls.RegisterForeign(reader, parentID, nil)
}
func (ls *mockLayerStore) RegisterForeign(reader io.Reader, parentID layer.ChainID, _ *distribution.Descriptor) (layer.Layer, error) {
var (
parent layer.Layer
err error

15
image/tarexport/load.go
View File

@ -10,6 +10,7 @@ import (
"reflect"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
@ -63,6 +64,10 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
var parentLinks []parentLink
for _, m := range manifest {
if m.LayerSources != nil && !layer.ForeignSourceSupported() {
return fmt.Errorf("invalid manifest, foreign layers not supported on this operating system")
}
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
@ -92,7 +97,7 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
r.Append(diffID)
newLayer, err := l.ls.Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput)
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), m.LayerSources[diffID], progressOutput)
if err != nil {
return err
}
@ -151,7 +156,7 @@ func (l *tarexporter) setParentID(id, parentID image.ID) error {
return l.is.SetParent(id, parentID)
}
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressOutput progress.Output) (layer.Layer, error) {
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc *distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
rawTar, err := os.Open(filename)
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
@ -174,9 +179,9 @@ func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string,
progressReader := progress.NewProgressReader(inflatedLayerData, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
return l.ls.Register(progressReader, rootFS.ChainID())
return l.ls.RegisterForeign(progressReader, rootFS.ChainID(), foreignSrc)
}
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
return l.ls.RegisterForeign(inflatedLayerData, rootFS.ChainID(), foreignSrc)
}
func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error {
@ -298,7 +303,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, progressOutput)
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, nil, progressOutput)
if err != nil {
return err
}

66
image/tarexport/save.go
View File

@ -9,6 +9,7 @@ import (
"path/filepath"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
@ -131,7 +132,8 @@ func (s *saveSession) save(outStream io.Writer) error {
var parentLinks []parentLink
for id, imageDescr := range s.images {
if err = s.saveImage(id); err != nil {
foreignSrcs, err := s.saveImage(id)
if err != nil {
return err
}
@ -151,9 +153,10 @@ func (s *saveSession) save(outStream io.Writer) error {
}
manifest = append(manifest, manifestItem{
Config: digest.Digest(id).Hex() + ".json",
RepoTags: repoTags,
Layers: layers,
Config: digest.Digest(id).Hex() + ".json",
RepoTags: repoTags,
Layers: layers,
LayerSources: foreignSrcs,
})
parentID, _ := s.is.GetParent(id)
@ -213,18 +216,19 @@ func (s *saveSession) save(outStream io.Writer) error {
return nil
}
func (s *saveSession) saveImage(id image.ID) error {
func (s *saveSession) saveImage(id image.ID) (map[layer.DiffID]*distribution.Descriptor, error) {
img, err := s.is.Get(id)
if err != nil {
return err
return nil, err
}
if len(img.RootFS.DiffIDs) == 0 {
return fmt.Errorf("empty export - not implemented")
return nil, fmt.Errorf("empty export - not implemented")
}
var parent digest.Digest
var layers []string
var foreignSrcs map[layer.DiffID]*distribution.Descriptor
for i := range img.RootFS.DiffIDs {
v1Img := image.V1Image{}
if i == len(img.RootFS.DiffIDs)-1 {
@ -234,7 +238,7 @@ func (s *saveSession) saveImage(id image.ID) error {
rootFS.DiffIDs = rootFS.DiffIDs[:i+1]
v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent)
if err != nil {
return err
return nil, err
}
v1Img.ID = v1ID.Hex()
@ -242,79 +246,91 @@ func (s *saveSession) saveImage(id image.ID) error {
v1Img.Parent = parent.Hex()
}
if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil {
return err
src, err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created)
if err != nil {
return nil, err
}
layers = append(layers, v1Img.ID)
parent = v1ID
if src != nil {
if foreignSrcs == nil {
foreignSrcs = make(map[layer.DiffID]*distribution.Descriptor)
}
foreignSrcs[img.RootFS.DiffIDs[i]] = src
}
}
configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json")
if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil {
return err
return nil, err
}
if err := system.Chtimes(configFile, img.Created, img.Created); err != nil {
return err
return nil, err
}
s.images[id].layers = layers
return nil
return foreignSrcs, nil
}
func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error {
func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) (*distribution.Descriptor, error) {
if _, exists := s.savedLayers[legacyImg.ID]; exists {
return nil
return nil, nil
}
outDir := filepath.Join(s.outDir, legacyImg.ID)
if err := os.Mkdir(outDir, 0755); err != nil {
return err
return nil, err
}
// todo: why is this version file here?
if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
return err
return nil, err
}
imageConfig, err := json.Marshal(legacyImg)
if err != nil {
return err
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
return err
return nil, err
}
// serialize filesystem
tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
if err != nil {
return err
return nil, err
}
defer tarFile.Close()
l, err := s.ls.Get(id)
if err != nil {
return err
return nil, err
}
defer layer.ReleaseAndLog(s.ls, l)
arch, err := l.TarStream()
if err != nil {
return err
return nil, err
}
defer arch.Close()
if _, err := io.Copy(tarFile, arch); err != nil {
return err
return nil, err
}
for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
// todo: maybe save layer created timestamp?
if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
return err
return nil, err
}
}
s.savedLayers[legacyImg.ID] = struct{}{}
return nil
var src *distribution.Descriptor
if fs, ok := l.(layer.ForeignSourcer); ok {
src = fs.ForeignSource()
}
return src, nil
}

10
image/tarexport/tarexport.go
View File

@ -1,6 +1,7 @@
package tarexport
import (
"github.com/docker/distribution"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/reference"
@ -15,10 +16,11 @@ const (
)
type manifestItem struct {
Config string
RepoTags []string
Layers []string
Parent image.ID `json:",omitempty"`
Config string
RepoTags []string
Layers []string
Parent image.ID `json:",omitempty"`
LayerSources map[layer.DiffID]*distribution.Descriptor `json:",omitempty"`
}
type tarexporter struct {

30
layer/filestore.go
View File

@ -2,6 +2,7 @@ package layer
import (
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
@ -13,6 +14,7 @@ import (
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/ioutils"
)
@ -24,6 +26,9 @@ var (
// digest.SHA384, // Currently not used
// digest.SHA512, // Currently not used
}
// ErrNoForeignSource is returned when no foreign source is set for a layer.
ErrNoForeignSource = errors.New("layer does not have a foreign source")
)
type fileMetadataStore struct {
@ -98,6 +103,14 @@ func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error {
return ioutil.WriteFile(filepath.Join(fm.root, "cache-id"), []byte(cacheID), 0644)
}
func (fm *fileMetadataTransaction) SetForeignSource(ref distribution.Descriptor) error {
jsonRef, err := json.Marshal(ref)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(fm.root, "descriptor.json"), jsonRef, 0644)
}
func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) {
f, err := os.OpenFile(filepath.Join(fm.root, "tar-split.json.gz"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@ -191,6 +204,23 @@ func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) {
return content, nil
}
func (fms *fileMetadataStore) GetForeignSource(layer ChainID) (distribution.Descriptor, error) {
content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "descriptor.json"))
if err != nil {
if os.IsNotExist(err) {
return distribution.Descriptor{}, ErrNoForeignSource
}
return distribution.Descriptor{}, err
}
var ref distribution.Descriptor
err = json.Unmarshal(content, &ref)
if err != nil {
return distribution.Descriptor{}, err
}
return ref, err
}
func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) {
fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz"))
if err != nil {

12
layer/layer.go
View File

@ -14,6 +14,7 @@ import (
"io"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/archive"
)
@ -107,6 +108,14 @@ type Layer interface {
Metadata() (map[string]string, error)
}
// ForeignSourcer is an interface used to describe the source of layers
// and objects representing layers, when the source is a foreign URL.
type ForeignSourcer interface {
// ForeignSource returns the descriptor for this layer if it is
// a foreign layer, or nil for ordinary layers.
ForeignSource() *distribution.Descriptor
}
// RWLayer represents a layer which is
// read and writable
type RWLayer interface {
@ -168,6 +177,7 @@ type MountInit func(root string) error
// read-only and read-write layers.
type Store interface {
Register(io.Reader, ChainID) (Layer, error)
RegisterForeign(io.Reader, ChainID, *distribution.Descriptor) (Layer, error)
Get(ChainID) (Layer, error)
Release(Layer) ([]Metadata, error)
@ -189,6 +199,7 @@ type MetadataTransaction interface {
SetParent(parent ChainID) error
SetDiffID(DiffID) error
SetCacheID(string) error
SetForeignSource(distribution.Descriptor) error
TarSplitWriter(compressInput bool) (io.WriteCloser, error)
Commit(ChainID) error
@ -208,6 +219,7 @@ type MetadataStore interface {
GetParent(ChainID) (ChainID, error)
GetDiffID(ChainID) (DiffID, error)
GetCacheID(ChainID) (string, error)
GetForeignSource(ChainID) (distribution.Descriptor, error)
TarSplitReader(ChainID) (io.ReadCloser, error)
SetMountID(string, string) error

13
layer/layer_store.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/pkg/archive"
@ -137,6 +138,13 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
references: map[Layer]struct{}{},
}
foreignSrc, err := ls.store.GetForeignSource(layer)
if err == nil {
cl.foreignSrc = &foreignSrc
} else if err != ErrNoForeignSource {
return nil, fmt.Errorf("failed to get foreign reference for %s: %s", layer, err)
}
if parent != "" {
p, err := ls.loadLayer(parent)
if err != nil {
@ -228,6 +236,10 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
}
func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
return ls.RegisterForeign(ts, parent, nil)
}
func (ls *layerStore) RegisterForeign(ts io.Reader, parent ChainID, foreignSrc *distribution.Descriptor) (Layer, error) {
// err is used to hold the error which will always trigger
// cleanup of creates sources but may not be an error returned
// to the caller (already exists).
@ -258,6 +270,7 @@ func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
layer := &roLayer{
parent: p,
cacheID: stringid.GenerateRandomID(),
foreignSrc: foreignSrc,
referenceCount: 1,
layerStore: ls,
references: map[Layer]struct{}{},

6
layer/layer_unix.go
View File

@ -7,3 +7,9 @@ import "github.com/docker/docker/pkg/stringid"
func (ls *layerStore) mountID(name string) string {
return stringid.GenerateRandomID()
}
// ForeignSourceSupported returns whether layers downloaded from foreign sources are
// supported in this daemon.
func ForeignSourceSupported() bool {
return false
}

6
layer/layer_windows.go
View File

@ -96,3 +96,9 @@ func (ls *layerStore) mountID(name string) string {
func (ls *layerStore) GraphDriver() graphdriver.Driver {
return ls.driver
}
// ForeignSourceSupported returns whether layers downloaded from foreign sources are
// supported in this daemon.
func ForeignSourceSupported() bool {
return true
}

7
layer/ro_layer.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"io"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
)
@ -14,6 +15,7 @@ type roLayer struct {
cacheID string
size int64
layerStore *layerStore
foreignSrc *distribution.Descriptor
referenceCount int
references map[Layer]struct{}
@ -123,6 +125,11 @@ func storeLayer(tx MetadataTransaction, layer *roLayer) error {
return err
}
}
if layer.foreignSrc != nil {
if err := tx.SetForeignSource(*layer.foreignSrc); err != nil {
return err
}
}
return nil
}

9
layer/ro_layer_windows.go Normal file
View File

@ -0,0 +1,9 @@
package layer
import "github.com/docker/distribution"
var _ ForeignSourcer = &roLayer{}
func (rl *roLayer) ForeignSource() *distribution.Descriptor {
return rl.foreignSrc
}