diff --git a/daemon/graphdriver/imagerestorer.go b/daemon/graphdriver/imagerestorer.go index a952a908f0..d6592203b5 100644 --- a/daemon/graphdriver/imagerestorer.go +++ b/daemon/graphdriver/imagerestorer.go @@ -27,5 +27,5 @@ type Tagger interface { // functions without needing to import graph. type Recorder interface { Exists(id string) bool - Register(img *image.Image, layerData io.Reader) error + Register(img image.Descriptor, layerData io.Reader) error } diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go index d9a4c59ba6..dbd3defd5c 100644 --- a/daemon/graphdriver/windows/windows.go +++ b/daemon/graphdriver/windows/windows.go @@ -40,6 +40,26 @@ const ( filterDriver ) +// CustomImageDescriptor is an image descriptor for use by RestoreCustomImages +type customImageDescriptor struct { + img *image.Image +} + +// ID returns the image ID specified in the image structure. +func (img customImageDescriptor) ID() string { + return img.img.ID +} + +// Parent returns the parent ID - in this case, none +func (img customImageDescriptor) Parent() string { + return "" +} + +// MarshalConfig renders the image structure into JSON. +func (img customImageDescriptor) MarshalConfig() ([]byte, error) { + return json.Marshal(img.img) +} + // Driver represents a windows graph driver. type Driver struct { // info stores the shim driver information @@ -426,7 +446,7 @@ func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdr Size: imageData.Size, } - if err := recorder.Register(img, nil); err != nil { + if err := recorder.Register(customImageDescriptor{img}, nil); err != nil { return nil, err } diff --git a/graph/export.go b/graph/export.go index 6362e54a6d..d5fc5342e0 100644 --- a/graph/export.go +++ b/graph/export.go @@ -112,6 +112,11 @@ func (s *TagStore) ImageExport(names []string, outStream io.Writer) error { func (s *TagStore) exportImage(name, tempdir string) error { for n := name; n != ""; { + img, err := s.LookupImage(n) + if err != nil || img == nil { + return fmt.Errorf("No such image %s", n) + } + // temporary directory tmpImageDir := filepath.Join(tempdir, n) if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil { @@ -128,19 +133,17 @@ func (s *TagStore) exportImage(name, tempdir string) error { return err } + imageInspectRaw, err := json.Marshal(img) + if err != nil { + return err + } + // serialize json json, err := os.Create(filepath.Join(tmpImageDir, "json")) if err != nil { return err } - img, err := s.LookupImage(n) - if err != nil || img == nil { - return fmt.Errorf("No such image %s", n) - } - imageInspectRaw, err := s.graph.RawJSON(img.ID) - if err != nil { - return err - } + written, err := json.Write(imageInspectRaw) if err != nil { return err diff --git a/graph/graph.go b/graph/graph.go index 81cff3ed5b..cbdfeee1a1 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -31,6 +31,26 @@ import ( "github.com/vbatts/tar-split/tar/storage" ) +// v1Descriptor is a non-content-addressable image descriptor +type v1Descriptor struct { + img *image.Image +} + +// ID returns the image ID specified in the image structure. +func (img v1Descriptor) ID() string { + return img.img.ID +} + +// Parent returns the parent ID specified in the image structure. +func (img v1Descriptor) Parent() string { + return img.img.Parent +} + +// MarshalConfig renders the image structure into JSON. +func (img v1Descriptor) MarshalConfig() ([]byte, error) { + return json.Marshal(img.img) +} + // The type is used to protect pulling or building related image // layers from deleteing when filtered by dangling=true // The key of layers is the images ID which is pulling or building @@ -88,10 +108,12 @@ type Graph struct { // file names for ./graph// const ( - jsonFileName = "json" - layersizeFileName = "layersize" - digestFileName = "checksum" - tarDataFileName = "tar-data.json.gz" + jsonFileName = "json" + layersizeFileName = "layersize" + digestFileName = "checksum" + tarDataFileName = "tar-data.json.gz" + v1CompatibilityFileName = "v1Compatibility" + parentFileName = "parent" ) var ( @@ -225,7 +247,7 @@ func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, com img.ContainerConfig = *containerConfig } - if err := graph.Register(img, layerData); err != nil { + if err := graph.Register(v1Descriptor{img}, layerData); err != nil { return nil, err } return img, nil @@ -233,19 +255,26 @@ func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, com // Register imports a pre-existing image into the graph. // Returns nil if the image is already registered. -func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) { +func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) { + imgID := im.ID() - if err := image.ValidateID(img.ID); err != nil { + if err := image.ValidateID(imgID); err != nil { return err } // We need this entire operation to be atomic within the engine. Note that // this doesn't mean Register is fully safe yet. - graph.imageMutex.Lock(img.ID) - defer graph.imageMutex.Unlock(img.ID) + graph.imageMutex.Lock(imgID) + defer graph.imageMutex.Unlock(imgID) + + return graph.register(im, layerData) +} + +func (graph *Graph) register(im image.Descriptor, layerData io.Reader) (err error) { + imgID := im.ID() // Skip register if image is already registered - if graph.Exists(img.ID) { + if graph.Exists(imgID) { return nil } @@ -255,14 +284,14 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) // If any error occurs, remove the new dir from the driver. // Don't check for errors since the dir might not have been created. if err != nil { - graph.driver.Remove(img.ID) + graph.driver.Remove(imgID) } }() // Ensure that the image root does not exist on the filesystem // when it is not registered in the graph. // This is common when you switch from one graph driver to another - if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) { return err } @@ -270,7 +299,7 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) // (the graph is the source of truth). // Ignore errors, since we don't know if the driver correctly returns ErrNotExist. // (FIXME: make that mandatory for drivers). - graph.driver.Remove(img.ID) + graph.driver.Remove(imgID) tmp, err := graph.mktemp() defer os.RemoveAll(tmp) @@ -278,26 +307,32 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) return fmt.Errorf("mktemp failed: %s", err) } + parent := im.Parent() + // Create root filesystem in the driver - if err := createRootFilesystemInDriver(graph, img); err != nil { + if err := createRootFilesystemInDriver(graph, imgID, parent, layerData); err != nil { return err } // Apply the diff/layer - if err := graph.storeImage(img, layerData, tmp); err != nil { + config, err := im.MarshalConfig() + if err != nil { + return err + } + if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil { return err } // Commit - if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil { + if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil { return err } - graph.idIndex.Add(img.ID) + graph.idIndex.Add(imgID) return nil } -func createRootFilesystemInDriver(graph *Graph, img *image.Image) error { - if err := graph.driver.Create(img.ID, img.Parent); err != nil { - return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) +func createRootFilesystemInDriver(graph *Graph, id, parent string, layerData io.Reader) error { + if err := graph.driver.Create(id, parent); err != nil { + return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err) } return nil } @@ -480,6 +515,21 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) { if err := dec.Decode(img); err != nil { return nil, err } + + if img.ID == "" { + img.ID = id + } + + if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil { + img.Parent = img.ParentID.Hex() + } + + // compatibilityID for parent + parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName)) + if err == nil && len(parent) > 0 { + img.Parent = string(parent) + } + if err := image.ValidateID(img.ID); err != nil { return nil, err } @@ -513,11 +563,14 @@ func (graph *Graph) saveSize(root string, size int64) error { return nil } -// SetDigest sets the digest for the image layer to the provided value. -func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { +// SetLayerDigest sets the digest for the image layer to the provided value. +func (graph *Graph) SetLayerDigest(id string, dgst digest.Digest) error { graph.imageMutex.Lock(id) defer graph.imageMutex.Unlock(id) + return graph.setLayerDigest(id, dgst) +} +func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error { root := graph.imageRoot(id) if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil { return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err) @@ -525,11 +578,15 @@ func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { return nil } -// GetDigest gets the digest for the provide image layer id. -func (graph *Graph) GetDigest(id string) (digest.Digest, error) { +// GetLayerDigest gets the digest for the provide image layer id. +func (graph *Graph) GetLayerDigest(id string) (digest.Digest, error) { graph.imageMutex.Lock(id) defer graph.imageMutex.Unlock(id) + return graph.getLayerDigest(id) +} + +func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) { root := graph.imageRoot(id) cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName)) if err != nil { @@ -541,6 +598,76 @@ func (graph *Graph) GetDigest(id string) (digest.Digest, error) { return digest.ParseDigest(string(cs)) } +// SetV1CompatibilityConfig stores the v1Compatibility JSON data associated +// with the image in the manifest to the disk +func (graph *Graph) SetV1CompatibilityConfig(id string, data []byte) error { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.setV1CompatibilityConfig(id, data) +} +func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error { + root := graph.imageRoot(id) + return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600) +} + +// GetV1CompatibilityConfig reads the v1Compatibility JSON data for the image +// from the disk +func (graph *Graph) GetV1CompatibilityConfig(id string) ([]byte, error) { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.getV1CompatibilityConfig(id) +} + +func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) { + root := graph.imageRoot(id) + return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName)) +} + +// GenerateV1CompatibilityChain makes sure v1Compatibility JSON data exists +// for the image. If it doesn't it generates and stores it for the image and +// all of it's parents based on the image config JSON. +func (graph *Graph) GenerateV1CompatibilityChain(id string) ([]byte, error) { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + if v1config, err := graph.getV1CompatibilityConfig(id); err == nil { + return v1config, nil + } + + // generate new, store it to disk + img, err := graph.Get(id) + if err != nil { + return nil, err + } + + digestPrefix := string(digest.Canonical) + ":" + img.ID = strings.TrimPrefix(img.ID, digestPrefix) + + if img.Parent != "" { + parentConfig, err := graph.GenerateV1CompatibilityChain(img.Parent) + if err != nil { + return nil, err + } + var parent struct{ ID string } + err = json.Unmarshal(parentConfig, &parent) + if err != nil { + return nil, err + } + img.Parent = parent.ID + } + + json, err := json.Marshal(img) + if err != nil { + return nil, err + } + if err := graph.setV1CompatibilityConfig(id, json); err != nil { + return nil, err + } + return json, nil +} + // RawJSON returns the JSON representation for an image as a byte array. func (graph *Graph) RawJSON(id string) ([]byte, error) { root := graph.imageRoot(id) @@ -560,29 +687,38 @@ func jsonPath(root string) string { // storeImage stores file system layer data for the given image to the // graph's storage driver. Image metadata is stored in a file // at the specified root directory. -func (graph *Graph) storeImage(img *image.Image, layerData io.Reader, root string) (err error) { +func (graph *Graph) storeImage(id, parent string, config []byte, layerData io.Reader, root string) (err error) { + var size int64 // Store the layer. If layerData is not nil, unpack it into the new layer if layerData != nil { - if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil { + if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil { return err } } - if err := graph.saveSize(root, img.Size); err != nil { + if err := graph.saveSize(root, size); err != nil { return err } - f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil { + return err + } + + // If image is pointing to a parent via CompatibilityID write the reference to disk + img, err := image.NewImgJSON(config) if err != nil { return err } - defer f.Close() - - return json.NewEncoder(f).Encode(img) + if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() { + if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil { + return err + } + } + return nil } -func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.Reader, root string) (err error) { +func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData io.Reader, root string) (size int64, err error) { var ar io.Reader if graph.tarSplitDisabled { @@ -591,7 +727,7 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.R // this is saving the tar-split metadata mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { - return err + return 0, err } mfz := gzip.NewWriter(mf) @@ -601,24 +737,24 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.R inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { - return err + return 0, err } // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil) if err != nil { - return err + return 0, err } ar = archive.Reader(rdr) } - if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, ar); err != nil { - return err + if size, err = graph.driver.ApplyDiff(id, parent, ar); err != nil { + return 0, err } - return nil + return } func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) { diff --git a/graph/graph_test.go b/graph/graph_test.go index 3a5ca43585..79c2d099a0 100644 --- a/graph/graph_test.go +++ b/graph/graph_test.go @@ -73,7 +73,7 @@ func TestInterruptedRegister(t *testing.T) { Created: time.Now(), } w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) - graph.Register(image, badArchive) + graph.Register(v1Descriptor{image}, badArchive) if _, err := graph.Get(image.ID); err == nil { t.Fatal("Image should not exist after Register is interrupted") } @@ -82,7 +82,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(image, goodArchive); err != nil { + if err := graph.Register(v1Descriptor{image}, goodArchive); err != nil { t.Fatal(err) } } @@ -130,7 +130,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(image, archive) + err = graph.Register(v1Descriptor{image}, archive) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(img1, archive); err != nil { + if err := graph.Register(v1Descriptor{img1}, archive); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { @@ -246,9 +246,19 @@ func TestByParent(t *testing.T) { Created: time.Now(), Parent: parentImage.ID, } - _ = graph.Register(parentImage, archive1) - _ = graph.Register(childImage1, archive2) - _ = graph.Register(childImage2, archive3) + + err := graph.Register(v1Descriptor{parentImage}, archive1) + if err != nil { + t.Fatal(err) + } + err = graph.Register(v1Descriptor{childImage1}, archive2) + if err != nil { + t.Fatal(err) + } + err = graph.Register(v1Descriptor{childImage2}, archive3) + if err != nil { + t.Fatal(err) + } byParent := graph.ByParent() numChildren := len(byParent[parentImage.ID]) diff --git a/graph/load.go b/graph/load.go index a3e3551252..e78a61478c 100644 --- a/graph/load.go +++ b/graph/load.go @@ -122,7 +122,7 @@ func (s *TagStore) recursiveLoad(address, tmpImageDir string) error { } } } - if err := s.graph.Register(img, layer); err != nil { + if err := s.graph.Register(v1Descriptor{img}, layer); err != nil { return err } } diff --git a/graph/pull_v1.go b/graph/pull_v1.go index 04bb8fa2dc..754b041db1 100644 --- a/graph/pull_v1.go +++ b/graph/pull_v1.go @@ -127,7 +127,7 @@ func (p *v1Puller) pullRepository(askedTag string) error { defer func() { p.graph.Release(sessionID, imgIDs...) }() - for _, image := range repoData.ImgList { + for _, imgData := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { errors <- nil @@ -140,6 +140,11 @@ func (p *v1Puller) pullRepository(askedTag string) error { return } + if err := image.ValidateID(img.ID); err != nil { + errors <- err + return + } + // ensure no two downloads of the same image happen at the same time poolKey := "img:" + img.ID broadcaster, found := p.poolAdd("pull", poolKey) @@ -197,7 +202,7 @@ func (p *v1Puller) pullRepository(askedTag string) error { errors <- nil } - go downloadImage(image) + go downloadImage(imgData) } var lastError error @@ -317,7 +322,7 @@ func (p *v1Puller) pullImage(out io.Writer, imgID, endpoint string) (layersDownl layersDownloaded = true defer layer.Close() - err = p.graph.Register(img, + err = p.graph.Register(v1Descriptor{img}, progressreader.New(progressreader.Config{ In: layer, Out: broadcaster, diff --git a/graph/pull_v2.go b/graph/pull_v2.go index eceb70f152..95c5106801 100644 --- a/graph/pull_v2.go +++ b/graph/pull_v2.go @@ -1,10 +1,12 @@ package graph import ( + "errors" "fmt" "io" "io/ioutil" "os" + "sync" "github.com/Sirupsen/logrus" "github.com/docker/distribution" @@ -73,7 +75,8 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { } - broadcaster, found := p.poolAdd("pull", taggedName) + poolKey := "v2:" + taggedName + broadcaster, found := p.poolAdd("pull", poolKey) broadcaster.Add(p.config.OutStream) if found { // Another pull of the same repository is already taking place; just wait for it to finish @@ -83,7 +86,7 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { // This must use a closure so it captures the value of err when the // function returns, not when the 'defer' is evaluated. defer func() { - p.poolRemoveWithError("pull", taggedName, err) + p.poolRemoveWithError("pull", poolKey, err) }() var layersDownloaded bool @@ -104,7 +107,8 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { // downloadInfo is used to pass information from download to extractor type downloadInfo struct { - img *image.Image + img contentAddressableDescriptor + imgIndex int tmpFile *os.File digest digest.Digest layer distribution.ReadSeekCloser @@ -114,12 +118,66 @@ type downloadInfo struct { broadcaster *broadcaster.Buffered } +// contentAddressableDescriptor is used to pass image data from a manifest to the +// graph. +type contentAddressableDescriptor struct { + id string + parent string + strongID digest.Digest + compatibilityID string + config []byte + v1Compatibility []byte +} + +func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) { + img := contentAddressableDescriptor{ + v1Compatibility: v1Compatibility, + } + + var err error + img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent) + if err != nil { + return img, err + } + img.strongID, err = image.StrongID(img.config) + if err != nil { + return img, err + } + + unmarshalledConfig, err := image.NewImgJSON(v1Compatibility) + if err != nil { + return img, err + } + + img.compatibilityID = unmarshalledConfig.ID + img.id = img.strongID.Hex() + + return img, nil +} + +// ID returns the actual ID to be used for the downloaded image. This may be +// a computed ID. +func (img contentAddressableDescriptor) ID() string { + return img.id +} + +// Parent returns the parent ID to be used for the image. This may be a +// computed ID. +func (img contentAddressableDescriptor) Parent() string { + return img.parent +} + +// MarshalConfig renders the image structure into JSON. +func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) { + return img.config, nil +} + type errVerification struct{} func (errVerification) Error() string { return "verification failed" } func (p *v2Puller) download(di *downloadInfo) { - logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID) + logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id) blobs := p.repo.Blobs(context.Background()) @@ -151,12 +209,12 @@ func (p *v2Puller) download(di *downloadInfo) { Formatter: p.sf, Size: di.size, NewLines: false, - ID: stringid.TruncateID(di.img.ID), + ID: stringid.TruncateID(di.img.id), Action: "Downloading", }) io.Copy(di.tmpFile, reader) - di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil)) + di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil)) if !verifier.Verified() { err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest) @@ -165,9 +223,9 @@ func (p *v2Puller) download(di *downloadInfo) { return } - di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) + di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil)) - logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, di.tmpFile.Name()) + logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, di.tmpFile.Name()) di.layer = layerDownload di.err <- nil @@ -193,6 +251,17 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo logrus.Printf("Image manifest for %s has been verified", taggedName) } + // remove duplicate layers and check parent chain validity + err = fixManifestLayers(&manifest.Manifest) + if err != nil { + return false, err + } + + imgs, err := p.getImageInfos(manifest.Manifest) + if err != nil { + return false, err + } + out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) var downloads []*downloadInfo @@ -213,26 +282,32 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo }() for i := len(manifest.FSLayers) - 1; i >= 0; i-- { - img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) - if err != nil { - logrus.Debugf("error getting image v1 json: %v", err) - return false, err - } - p.graph.Retain(p.sessionID, img.ID) - layerIDs = append(layerIDs, img.ID) + img := imgs[i] + + p.graph.Retain(p.sessionID, img.id) + layerIDs = append(layerIDs, img.id) + + p.graph.imageMutex.Lock(img.id) // Check if exists - if p.graph.Exists(img.ID) { - logrus.Debugf("Image already exists: %s", img.ID) - out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil)) + if p.graph.Exists(img.id) { + if err := p.validateImageInGraph(img.id, imgs, i); err != nil { + p.graph.imageMutex.Unlock(img.id) + return false, fmt.Errorf("image validation failed: %v", err) + } + logrus.Debugf("Image already exists: %s", img.id) + p.graph.imageMutex.Unlock(img.id) continue } - out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) + p.graph.imageMutex.Unlock(img.id) + + out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil)) d := &downloadInfo{ - img: img, - poolKey: "layer:" + img.ID, - digest: manifest.FSLayers[i].BlobSum, + img: img, + imgIndex: i, + poolKey: "v2layer:" + img.id, + digest: manifest.FSLayers[i].BlobSum, // TODO: seems like this chan buffer solved hanging problem in go1.5, // this can indicate some deeper problem that somehow we never take // error from channel in loop below @@ -274,26 +349,49 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo } d.tmpFile.Seek(0, 0) - reader := progressreader.New(progressreader.Config{ - In: d.tmpFile, - Out: d.broadcaster, - Formatter: p.sf, - Size: d.size, - NewLines: false, - ID: stringid.TruncateID(d.img.ID), - Action: "Extracting", - }) + err := func() error { + reader := progressreader.New(progressreader.Config{ + In: d.tmpFile, + Out: d.broadcaster, + Formatter: p.sf, + Size: d.size, + NewLines: false, + ID: stringid.TruncateID(d.img.id), + Action: "Extracting", + }) - err = p.graph.Register(d.img, reader) + p.graph.imageMutex.Lock(d.img.id) + defer p.graph.imageMutex.Unlock(d.img.id) + + // Must recheck the data on disk if any exists. + // This protects against races where something + // else is written to the graph under this ID + // after attemptIDReuse. + if p.graph.Exists(d.img.id) { + if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil { + return fmt.Errorf("image validation failed: %v", err) + } + } + + if err := p.graph.register(d.img, reader); err != nil { + return err + } + + if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil { + return err + } + + if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil { + return err + } + + return nil + }() if err != nil { return false, err } - if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { - return false, err - } - - d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) + d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil)) d.broadcaster.Close() tagUpdated = true } @@ -424,3 +522,217 @@ func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (ver } return verified, nil } + +// fixManifestLayers removes repeated layers from the manifest and checks the +// correctness of the parent chain. +func fixManifestLayers(m *manifest.Manifest) error { + images := make([]*image.Image, len(m.FSLayers)) + for i := range m.FSLayers { + img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility)) + if err != nil { + return err + } + images[i] = img + if err := image.ValidateID(img.ID); err != nil { + return err + } + } + + if images[len(images)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image.") + } + + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + + var lastID string + for _, img := range images { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + + // backwards loop so that we keep the remaining indexes after removing items + for i := len(images) - 2; i >= 0; i-- { + if images[i].ID == images[i+1].ID { // repeated ID. remove and continue + m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...) + m.History = append(m.History[:i], m.History[i+1:]...) + } else if images[i].Parent != images[i+1].ID { + return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent) + } + } + + return nil +} + +// getImageInfos returns an imageinfo struct for every image in the manifest. +// These objects contain both calculated strongIDs and compatibilityIDs found +// in v1Compatibility object. +func (p *v2Puller) getImageInfos(m manifest.Manifest) ([]contentAddressableDescriptor, error) { + imgs := make([]contentAddressableDescriptor, len(m.FSLayers)) + + var parent digest.Digest + for i := len(imgs) - 1; i >= 0; i-- { + var err error + imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent) + if err != nil { + return nil, err + } + parent = imgs[i].strongID + } + + p.attemptIDReuse(imgs) + + return imgs, nil +} + +var idReuseLock sync.Mutex + +// attemptIDReuse does a best attempt to match verified compatibilityIDs +// already in the graph with the computed strongIDs so we can keep using them. +// This process will never fail but may just return the strongIDs if none of +// the compatibilityIDs exists or can be verified. If the strongIDs themselves +// fail verification, we deterministically generate alternate IDs to use until +// we find one that's available or already exists with the correct data. +func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) { + // This function needs to be protected with a global lock, because it + // locks multiple IDs at once, and there's no good way to make sure + // the locking happens a deterministic order. + idReuseLock.Lock() + defer idReuseLock.Unlock() + + idMap := make(map[string]struct{}) + for _, img := range imgs { + idMap[img.id] = struct{}{} + idMap[img.compatibilityID] = struct{}{} + + if p.graph.Exists(img.compatibilityID) { + if _, err := p.graph.GenerateV1CompatibilityChain(img.compatibilityID); err != nil { + logrus.Debugf("Migration v1Compatibility generation error: %v", err) + return + } + } + } + for id := range idMap { + p.graph.imageMutex.Lock(id) + defer p.graph.imageMutex.Unlock(id) + } + + // continueReuse controls whether the function will try to find + // existing layers on disk under the old v1 IDs, to avoid repulling + // them. The hashes are checked to ensure these layers are okay to + // use. continueReuse starts out as true, but is set to false if + // the code encounters something that doesn't match the expected hash. + continueReuse := true + + for i := len(imgs) - 1; i >= 0; i-- { + if p.graph.Exists(imgs[i].id) { + // Found an image in the graph under the strongID. Validate the + // image before using it. + if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil { + continueReuse = false + logrus.Debugf("not using existing strongID: %v", err) + + // The strong ID existed in the graph but didn't + // validate successfully. We can't use the strong ID + // because it didn't validate successfully. Treat the + // graph like a hash table with probing... compute + // SHA256(id) until we find an ID that either doesn't + // already exist in the graph, or has existing content + // that validates successfully. + for { + if err := p.tryNextID(imgs, i, idMap); err != nil { + logrus.Debug(err.Error()) + } else { + break + } + } + } + continue + } + + if continueReuse { + compatibilityID := imgs[i].compatibilityID + if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil { + logrus.Debugf("stopping ID reuse: %v", err) + continueReuse = false + } else { + // The compatibility ID exists in the graph and was + // validated. Use it. + imgs[i].id = compatibilityID + } + } + } + + // fix up the parents of the images + for i := 0; i < len(imgs); i++ { + if i == len(imgs)-1 { // Base layer + imgs[i].parent = "" + } else { + imgs[i].parent = imgs[i+1].id + } + } +} + +// validateImageInGraph checks that an image in the graph has the expected +// strongID. id is the entry in the graph to check, imgs is the slice of +// images being processed (for access to the parent), and i is the index +// into this slice which the graph entry should be checked against. +func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error { + img, err := p.graph.Get(id) + if err != nil { + return fmt.Errorf("missing: %v", err) + } + layerID, err := p.graph.getLayerDigest(id) + if err != nil { + return fmt.Errorf("digest: %v", err) + } + var parentID digest.Digest + if i != len(imgs)-1 { + if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID + return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id) + } + parentID = imgs[i+1].strongID + } else if img.Parent != "" { + return fmt.Errorf("unexpected parent: %v", img.Parent) + } + + v1Config, err := p.graph.getV1CompatibilityConfig(img.ID) + if err != nil { + return fmt.Errorf("v1Compatibility: %v %v", img.ID, err) + } + + json, err := image.MakeImageConfig(v1Config, layerID, parentID) + if err != nil { + return fmt.Errorf("make config: %v", err) + } + + if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID { + logrus.Debugf("Validated %v as %v", dgst, id) + } else { + return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err) + } + + // All clear + return nil +} + +func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error { + nextID, _ := digest.FromBytes([]byte(imgs[i].id)) + imgs[i].id = nextID.Hex() + + if _, exists := idMap[imgs[i].id]; !exists { + p.graph.imageMutex.Lock(imgs[i].id) + defer p.graph.imageMutex.Unlock(imgs[i].id) + } + + if p.graph.Exists(imgs[i].id) { + if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil { + return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err) + } + } + return nil +} diff --git a/graph/pull_v2_test.go b/graph/pull_v2_test.go new file mode 100644 index 0000000000..a1777b81b8 --- /dev/null +++ b/graph/pull_v2_test.go @@ -0,0 +1,100 @@ +package graph + +import ( + "reflect" + "strings" + "testing" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" +) + +// TestFixManifestLayers checks that fixManifestLayers removes a duplicate +// layer, and that it makes no changes to the manifest when called a second +// time, after the duplicate is removed. +func TestFixManifestLayers(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + duplicateLayerManifestExpectedOutput := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err != nil { + t.Fatalf("unexpected error from fixManifestLayers: %v", err) + } + + if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) { + t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest") + } + + // Run fixManifestLayers again and confirm that it doesn't change the + // manifest (which no longer has duplicate layers). + if err := fixManifestLayers(&duplicateLayerManifest); err != nil { + t.Fatalf("unexpected error from fixManifestLayers: %v", err) + } + + if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) { + t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)") + } +} + +// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails +// if the base layer configuration specifies a parent. +func TestFixManifestLayersBaseLayerParent(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") { + t.Fatalf("expected an invalid parent ID error from fixManifestLayers") + } +} + +// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails +// if an image configuration specifies a parent that doesn't directly follow +// that (deduplicated) image in the image history. +func TestFixManifestLayersBadParent(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") { + t.Fatalf("expected an invalid parent ID error from fixManifestLayers") + } +} diff --git a/graph/push_v1.go b/graph/push_v1.go index c3ce7eeb96..0f3e1593a5 100644 --- a/graph/push_v1.go +++ b/graph/push_v1.go @@ -8,6 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/image" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/streamformatter" @@ -127,7 +128,7 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [ continue } // If the image does not have a tag it still needs to be sent to the - // registry with an empty tag so that it is accociated with the repository + // registry with an empty tag so that it is associated with the repository imageIndex = append(imageIndex, ®istry.ImgData{ ID: id, Tag: "", @@ -137,8 +138,9 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [ } type imagePushData struct { - id string - endpoint string + id string + compatibilityID string + endpoint string } // lookupImageOnEndpoint checks the specified endpoint to see if an image exists @@ -146,7 +148,7 @@ type imagePushData struct { func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) { defer wg.Done() for image := range images { - if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil { + if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil { logrus.Errorf("Error in LookupRemoteImage: %s", err) imagesToPush <- image.id continue @@ -180,9 +182,14 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags pushes <- shouldPush }() for _, id := range imageIDs { + compatibilityID, err := p.getV1ID(id) + if err != nil { + return err + } imageData <- imagePushData{ - id: id, - endpoint: endpoint, + id: id, + compatibilityID: compatibilityID, + endpoint: endpoint, } } // close the channel to notify the workers that there will be no more images to check. @@ -202,7 +209,11 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags } for _, tag := range tags[id] { p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag)) - if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil { + compatibilityID, err := p.getV1ID(id) + if err != nil { + return err + } + if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil { return err } } @@ -224,6 +235,12 @@ func (p *v1Pusher) pushRepository(tag string) error { logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo) for _, data := range imageIndex { logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) + + // convert IDs to compatibilityIDs, imageIndex only used in registry calls + data.ID, err = p.getV1ID(data.ID) + if err != nil { + return err + } } if _, found := p.poolAdd("push", p.repoInfo.LocalName); found { @@ -253,20 +270,27 @@ func (p *v1Pusher) pushRepository(tag string) error { } func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { - jsonRaw, err := p.graph.RawJSON(imgID) + jsonRaw, err := p.getV1Config(imgID) if err != nil { return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) } p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil)) + compatibilityID, err := p.getV1ID(imgID) + if err != nil { + return "", err + } + + // General rule is to use ID for graph accesses and compatibilityID for + // calls to session.registry() imgData := ®istry.ImgData{ - ID: imgID, + ID: compatibilityID, } // Send the json if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil { if err == registry.ErrAlreadyExists { - p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) + p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil)) return "", nil } return "", err @@ -279,7 +303,7 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { defer os.RemoveAll(layerData.Name()) // Send the layer - logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) + logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size) checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID, progressreader.New(progressreader.Config{ @@ -288,7 +312,7 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { Formatter: p.sf, Size: layerData.Size, NewLines: false, - ID: stringid.TruncateID(imgData.ID), + ID: stringid.TruncateID(imgID), Action: "Pushing", }), ep, jsonRaw) if err != nil { @@ -301,6 +325,30 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { return "", err } - p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil)) + p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil)) return imgData.Checksum, nil } + +// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID +// is read from from the v1Compatibility config file in the disk. +func (p *v1Pusher) getV1ID(id string) (string, error) { + jsonData, err := p.getV1Config(id) + if err != nil { + return "", err + } + img, err := image.NewImgJSON(jsonData) + if err != nil { + return "", err + } + return img.ID, nil +} + +// getV1Config returns v1Compatibility config for the image in the graph. If +// there is no v1Compatibility file on disk for the image +func (p *v1Pusher) getV1Config(id string) ([]byte, error) { + jsonData, err := p.graph.GenerateV1CompatibilityChain(id) + if err != nil { + return nil, err + } + return jsonData, nil +} diff --git a/graph/push_v2.go b/graph/push_v2.go index 5051ad1437..70cb2e42a0 100644 --- a/graph/push_v2.go +++ b/graph/push_v2.go @@ -138,13 +138,8 @@ func (p *v2Pusher) pushV2Tag(tag string) error { } } - jsonData, err := p.graph.RawJSON(layer.ID) - if err != nil { - return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) - } - var exists bool - dgst, err := p.graph.GetDigest(layer.ID) + dgst, err := p.graph.GetLayerDigest(layer.ID) switch err { case nil: if p.layersPushed[dgst] { @@ -178,13 +173,19 @@ func (p *v2Pusher) pushV2Tag(tag string) error { return err } else if pushDigest != dgst { // Cache new checksum - if err := p.graph.SetDigest(layer.ID, pushDigest); err != nil { + if err := p.graph.SetLayerDigest(layer.ID, pushDigest); err != nil { return err } dgst = pushDigest } } + // read v1Compatibility config, generate new if needed + jsonData, err := p.graph.GenerateV1CompatibilityChain(layer.ID) + if err != nil { + return err + } + m.FSLayers = append(m.FSLayers, manifest.FSLayer{BlobSum: dgst}) m.History = append(m.History, manifest.History{V1Compatibility: string(jsonData)}) diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index e2485a0e1f..f3fc9e2a01 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -82,7 +82,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img := &image.Image{ID: testOfficialImageID} - if err := graph.Register(img, officialArchive); err != nil { + if err := graph.Register(v1Descriptor{img}, officialArchive); err != nil { t.Fatal(err) } if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil { @@ -93,7 +93,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img = &image.Image{ID: testPrivateImageID} - if err := graph.Register(img, privateArchive); err != nil { + if err := graph.Register(v1Descriptor{img}, privateArchive); err != nil { t.Fatal(err) } if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil { diff --git a/image/fixtures/post1.9/expected_computed_id b/image/fixtures/post1.9/expected_computed_id new file mode 100644 index 0000000000..cba6d81f4e --- /dev/null +++ b/image/fixtures/post1.9/expected_computed_id @@ -0,0 +1 @@ +sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105 diff --git a/image/fixtures/post1.9/expected_config b/image/fixtures/post1.9/expected_config new file mode 100644 index 0000000000..ae27bdd429 --- /dev/null +++ b/image/fixtures/post1.9/expected_config @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} diff --git a/image/fixtures/post1.9/layer_id b/image/fixtures/post1.9/layer_id new file mode 100644 index 0000000000..ded2db28e7 --- /dev/null +++ b/image/fixtures/post1.9/layer_id @@ -0,0 +1 @@ +sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/post1.9/parent_id b/image/fixtures/post1.9/parent_id new file mode 100644 index 0000000000..7d524f80c2 --- /dev/null +++ b/image/fixtures/post1.9/parent_id @@ -0,0 +1 @@ +sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/post1.9/v1compatibility b/image/fixtures/post1.9/v1compatibility new file mode 100644 index 0000000000..d6697c2b68 --- /dev/null +++ b/image/fixtures/post1.9/v1compatibility @@ -0,0 +1 @@ +{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"} diff --git a/image/fixtures/pre1.9/expected_computed_id b/image/fixtures/pre1.9/expected_computed_id new file mode 100644 index 0000000000..c27b0b6a20 --- /dev/null +++ b/image/fixtures/pre1.9/expected_computed_id @@ -0,0 +1 @@ +sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89 diff --git a/image/fixtures/pre1.9/expected_config b/image/fixtures/pre1.9/expected_config new file mode 100644 index 0000000000..83fc30487a --- /dev/null +++ b/image/fixtures/pre1.9/expected_config @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} diff --git a/image/fixtures/pre1.9/layer_id b/image/fixtures/pre1.9/layer_id new file mode 100644 index 0000000000..ded2db28e7 --- /dev/null +++ b/image/fixtures/pre1.9/layer_id @@ -0,0 +1 @@ +sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/pre1.9/parent_id b/image/fixtures/pre1.9/parent_id new file mode 100644 index 0000000000..7d524f80c2 --- /dev/null +++ b/image/fixtures/pre1.9/parent_id @@ -0,0 +1 @@ +sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/pre1.9/v1compatibility b/image/fixtures/pre1.9/v1compatibility new file mode 100644 index 0000000000..af96e82506 --- /dev/null +++ b/image/fixtures/pre1.9/v1compatibility @@ -0,0 +1 @@ +{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0} diff --git a/image/image.go b/image/image.go index e069262eac..89799160da 100644 --- a/image/image.go +++ b/image/image.go @@ -2,19 +2,38 @@ package image import ( "encoding/json" + "fmt" "regexp" "time" + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" derr "github.com/docker/docker/errors" + "github.com/docker/docker/pkg/version" "github.com/docker/docker/runconfig" ) var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) +// noFallbackMinVersion is the minimum version for which v1compatibility +// information will not be marshaled through the Image struct to remove +// blank fields. +var noFallbackMinVersion = version.Version("1.8.3") + +// Descriptor provides the information necessary to register an image in +// the graph. +type Descriptor interface { + ID() string + Parent() string + MarshalConfig() ([]byte, error) +} + // Image stores the image configuration. +// All fields in this struct must be marked `omitempty` to keep getting +// predictable hashes from the old `v1Compatibility` configuration. type Image struct { // ID a unique 64 character identifier of the image - ID string `json:"id"` + ID string `json:"id,omitempty"` // Parent id of the image Parent string `json:"parent,omitempty"` // Comment user added comment @@ -36,7 +55,11 @@ type Image struct { // OS is the operating system used to build and run the image OS string `json:"os,omitempty"` // Size is the total size of the image including all layers it is composed of - Size int64 + Size int64 `json:",omitempty"` // capitalized for backwards compatibility + // ParentID specifies the strong, content address of the parent configuration. + ParentID digest.Digest `json:"parent_id,omitempty"` + // LayerID provides the content address of the associated layer. + LayerID digest.Digest `json:"layer_id,omitempty"` } // NewImgJSON creates an Image configuration from json. @@ -57,3 +80,70 @@ func ValidateID(id string) error { } return nil } + +// MakeImageConfig returns immutable configuration JSON for image based on the +// v1Compatibility object, layer digest and parent StrongID. SHA256() of this +// config is the new image ID (strongID). +func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) { + + // Detect images created after 1.8.3 + img, err := NewImgJSON(v1Compatibility) + if err != nil { + return nil, err + } + useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion) + + if useFallback { + // Fallback for pre-1.8.3. Calculate base config based on Image struct + // so that fields with default values added by Docker will use same ID + logrus.Debugf("Using fallback hash for %v", layerID) + + v1Compatibility, err = json.Marshal(img) + if err != nil { + return nil, err + } + } + + var c map[string]*json.RawMessage + if err := json.Unmarshal(v1Compatibility, &c); err != nil { + return nil, err + } + + if err := layerID.Validate(); err != nil { + return nil, fmt.Errorf("invalid layerID: %v", err) + } + + c["layer_id"] = rawJSON(layerID) + + if parentID != "" { + if err := parentID.Validate(); err != nil { + return nil, fmt.Errorf("invalid parentID %v", err) + } + c["parent_id"] = rawJSON(parentID) + } + + delete(c, "id") + delete(c, "parent") + delete(c, "Size") // Size is calculated from data on disk and is inconsitent + + return json.Marshal(c) +} + +// StrongID returns image ID for the config JSON. +func StrongID(configJSON []byte) (digest.Digest, error) { + digester := digest.Canonical.New() + if _, err := digester.Hash().Write(configJSON); err != nil { + return "", err + } + dgst := digester.Digest() + logrus.Debugf("H(%v) = %v", string(configJSON), dgst) + return dgst, nil +} + +func rawJSON(value interface{}) *json.RawMessage { + jsonval, err := json.Marshal(value) + if err != nil { + return nil + } + return (*json.RawMessage)(&jsonval) +} diff --git a/image/image_test.go b/image/image_test.go new file mode 100644 index 0000000000..77d92c4490 --- /dev/null +++ b/image/image_test.go @@ -0,0 +1,55 @@ +package image + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/docker/distribution/digest" +) + +var fixtures = []string{ + "fixtures/pre1.9", + "fixtures/post1.9", +} + +func loadFixtureFile(t *testing.T, path string) []byte { + fileData, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("error opening %s: %v", path, err) + } + + return bytes.TrimSpace(fileData) +} + +// TestMakeImageConfig makes sure that MakeImageConfig returns the expected +// canonical JSON for a reference Image. +func TestMakeImageConfig(t *testing.T) { + for _, fixture := range fixtures { + v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility") + expectedConfig := loadFixtureFile(t, fixture+"/expected_config") + layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id")) + parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id")) + + json, err := MakeImageConfig(v1Compatibility, layerID, parentID) + if err != nil { + t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err) + } + if !bytes.Equal(json, expectedConfig) { + t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json) + } + } +} + +// TestGetStrongID makes sure that GetConfigJSON returns the expected +// hash for a reference Image. +func TestGetStrongID(t *testing.T) { + for _, fixture := range fixtures { + expectedConfig := loadFixtureFile(t, fixture+"/expected_config") + expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id")) + + if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID { + t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err) + } + } +} diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index e4e9d23fce..ce74c7d7f2 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,7 +1,11 @@ package main import ( + "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" "regexp" "strings" "time" @@ -159,3 +163,254 @@ func (s *DockerHubPullSuite) TestPullClientDisconnect(c *check.C) { time.Sleep(500 * time.Millisecond) } } + +type idAndParent struct { + ID string + Parent string +} + +func inspectImage(c *check.C, imageRef string) idAndParent { + out, _ := dockerCmd(c, "inspect", imageRef) + var inspectOutput []idAndParent + err := json.Unmarshal([]byte(out), &inspectOutput) + if err != nil { + c.Fatal(err) + } + + return inspectOutput[0] +} + +func imageID(c *check.C, imageRef string) string { + return inspectImage(c, imageRef).ID +} + +func imageParent(c *check.C, imageRef string) string { + return inspectImage(c, imageRef).Parent +} + +// TestPullMigration verifies that pulling an image based on layers +// that already exists locally will reuse those existing layers. +func (s *DockerRegistrySuite) TestPullMigration(c *check.C) { + repoName := privateRegistryURL + "/dockercli/migration" + + baseImage := repoName + ":base" + _, err := buildImage(baseImage, fmt.Sprintf(` + FROM scratch + ENV IMAGE base + CMD echo %s + `, baseImage), true) + if err != nil { + c.Fatal(err) + } + + baseIDBeforePush := imageID(c, baseImage) + baseParentBeforePush := imageParent(c, baseImage) + + derivedImage := repoName + ":derived" + _, err = buildImage(derivedImage, fmt.Sprintf(` + FROM %s + CMD echo %s + `, baseImage, derivedImage), true) + if err != nil { + c.Fatal(err) + } + + derivedIDBeforePush := imageID(c, derivedImage) + + dockerCmd(c, "push", derivedImage) + + // Remove derived image from the local store + dockerCmd(c, "rmi", derivedImage) + + // Repull + dockerCmd(c, "pull", derivedImage) + + // Check that the parent of this pulled image is the original base + // image + derivedIDAfterPull1 := imageID(c, derivedImage) + derivedParentAfterPull1 := imageParent(c, derivedImage) + + if derivedIDAfterPull1 == derivedIDBeforePush { + c.Fatal("image's ID should have changed on after deleting and pulling") + } + + if derivedParentAfterPull1 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) + } + + // Confirm that repushing and repulling does not change the computed ID + dockerCmd(c, "push", derivedImage) + dockerCmd(c, "rmi", derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull2 := imageID(c, derivedImage) + derivedParentAfterPull2 := imageParent(c, derivedImage) + + if derivedIDAfterPull2 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + + if derivedParentAfterPull2 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) + } + + // Remove everything, repull, and make sure everything uses computed IDs + dockerCmd(c, "rmi", baseImage, derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull3 := imageID(c, derivedImage) + derivedParentAfterPull3 := imageParent(c, derivedImage) + derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3) + + if derivedIDAfterPull3 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a second repull") + } + + if derivedParentAfterPull3 == baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush) + } + + if derivedGrandparentAfterPull3 == baseParentBeforePush { + c.Fatal("base image's parent ID should have been rewritten on pull") + } +} + +// TestPullMigrationRun verifies that pulling an image based on layers +// that already exists locally will result in an image that runs properly. +func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) { + type idAndParent struct { + ID string + Parent string + } + + derivedImage := privateRegistryURL + "/dockercli/migration-run" + baseImage := "busybox" + + _, err := buildImage(derivedImage, fmt.Sprintf(` + FROM %s + RUN dd if=/dev/zero of=/file bs=1024 count=1024 + CMD echo %s + `, baseImage, derivedImage), true) + if err != nil { + c.Fatal(err) + } + + baseIDBeforePush := imageID(c, baseImage) + derivedIDBeforePush := imageID(c, derivedImage) + + dockerCmd(c, "push", derivedImage) + + // Remove derived image from the local store + dockerCmd(c, "rmi", derivedImage) + + // Repull + dockerCmd(c, "pull", derivedImage) + + // Check that this pulled image is based on the original base image + derivedIDAfterPull1 := imageID(c, derivedImage) + derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage)) + + if derivedIDAfterPull1 == derivedIDBeforePush { + c.Fatal("image's ID should have changed on after deleting and pulling") + } + + if derivedParentAfterPull1 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) + } + + // Make sure the image runs correctly + out, _ := dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } + + // Confirm that repushing and repulling does not change the computed ID + dockerCmd(c, "push", derivedImage) + dockerCmd(c, "rmi", derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull2 := imageID(c, derivedImage) + derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage)) + + if derivedIDAfterPull2 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + + if derivedParentAfterPull2 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) + } + + // Make sure the image still runs + out, _ = dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } +} + +// TestPullConflict provides coverage of the situation where a computed +// strongID conflicts with some unverifiable data in the graph. +func (s *DockerRegistrySuite) TestPullConflict(c *check.C) { + repoName := privateRegistryURL + "/dockercli/conflict" + + _, err := buildImage(repoName, ` + FROM scratch + ENV IMAGE conflict + CMD echo conflict + `, true) + if err != nil { + c.Fatal(err) + } + + dockerCmd(c, "push", repoName) + + // Pull to make it content-addressable + dockerCmd(c, "rmi", repoName) + dockerCmd(c, "pull", repoName) + + IDBeforeLoad := imageID(c, repoName) + + // Load/save to turn this into an unverified image with the same ID + tmpDir, err := ioutil.TempDir("", "conflict-save-output") + if err != nil { + c.Errorf("failed to create temporary directory: %s", err) + } + defer os.RemoveAll(tmpDir) + + tarFile := filepath.Join(tmpDir, "repo.tar") + + dockerCmd(c, "save", "-o", tarFile, repoName) + dockerCmd(c, "rmi", repoName) + dockerCmd(c, "load", "-i", tarFile) + + // Check that the the ID is the same after save/load. + IDAfterLoad := imageID(c, repoName) + + if IDAfterLoad != IDBeforeLoad { + c.Fatal("image's ID should be the same after save/load") + } + + // Repull + dockerCmd(c, "pull", repoName) + + // Check that the ID is now different because of the conflict. + IDAfterPull1 := imageID(c, repoName) + + // Expect the new ID to be SHA256(oldID) + expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad)) + if err != nil { + c.Fatalf("digest error: %v", err) + } + expectedID := expectedIDDigest.Hex() + if IDAfterPull1 != expectedID { + c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1) + } + + // A second pull should use the new ID again. + dockerCmd(c, "pull", repoName) + + IDAfterPull2 := imageID(c, repoName) + + if IDAfterPull2 != IDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repull") + } +} diff --git a/runconfig/config.go b/runconfig/config.go index 48646c1ec4..08f146d1b9 100644 --- a/runconfig/config.go +++ b/runconfig/config.go @@ -12,6 +12,8 @@ import ( // It should hold only portable information about the container. // Here, "portable" means "independent from the host we are running on". // Non-portable information *should* appear in HostConfig. +// All fields added to this struct must be marked `omitempty` to keep getting +// predictable hashes from the old `v1Compatibility` configuration. type Config struct { Hostname string // Hostname Domainname string // Domainname @@ -19,7 +21,8 @@ type Config struct { AttachStdin bool // Attach the standard input, makes possible user interaction AttachStdout bool // Attach the standard output AttachStderr bool // Attach the standard error - ExposedPorts map[nat.Port]struct{} // List of exposed ports + ExposedPorts map[nat.Port]struct{} `json:",omitempty"` // List of exposed ports + PublishService string `json:",omitempty"` // Name of the network service exposed by the container Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin StdinOnce bool // If true, close stdin after the 1 attached client disconnects. @@ -29,11 +32,11 @@ type Config struct { Volumes map[string]struct{} // List of volumes (mounts) used for the container WorkingDir string // Current directory (PWD) in the command will be launched Entrypoint *stringutils.StrSlice // Entrypoint to run when starting the container - NetworkDisabled bool // Is network disabled - MacAddress string // Mac Address of the container + NetworkDisabled bool `json:",omitempty"` // Is network disabled + MacAddress string `json:",omitempty"` // Mac Address of the container OnBuild []string // ONBUILD metadata that were defined on the image Dockerfile Labels map[string]string // List of labels set to this container - StopSignal string // Signal to stop a container + StopSignal string `json:",omitempty"` // Signal to stop a container } // DecodeContainerConfig decodes a json encoded config into a ContainerConfigWrapper