Calculate hash based image IDs on pull

Generate a hash chain involving the image configuration, layer digests,
and parent image hashes. Use the digests to compute IDs for each image
in a manifest, instead of using the remotely specified IDs.

To avoid breaking users' caches, check for images already in the graph
under old IDs, and avoid repulling an image if the version on disk under
the legacy ID ends up with the same digest that was computed from the
manifest for that image.

When a calculated ID already exists in the graph but can't be verified,
continue trying SHA256(digest) until a suitable ID is found.

"save" and "load" are not changed to use a similar scheme. "load" will
preserve the IDs present in the tar file.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

daemon/graphdriver/imagerestorer.go

@@ -27,5 +27,5 @@ type Tagger interface {
 // functions without needing to import graph.
 type Recorder interface {
 	Exists(id string) bool
-	Register(img *image.Image, layerData io.Reader) error
+	Register(img image.Descriptor, layerData io.Reader) error
 }

daemon/graphdriver/windows/windows.go

@@ -40,6 +40,26 @@ const (
 	filterDriver
 )
 
+// CustomImageDescriptor is an image descriptor for use by RestoreCustomImages
+type customImageDescriptor struct {
+	img *image.Image
+}
+
+// ID returns the image ID specified in the image structure.
+func (img customImageDescriptor) ID() string {
+	return img.img.ID
+}
+
+// Parent returns the parent ID - in this case, none
+func (img customImageDescriptor) Parent() string {
+	return ""
+}
+
+// MarshalConfig renders the image structure into JSON.
+func (img customImageDescriptor) MarshalConfig() ([]byte, error) {
+	return json.Marshal(img.img)
+}
+
 // Driver represents a windows graph driver.
 type Driver struct {
 	// info stores the shim driver information
@@ -426,7 +446,7 @@ func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdr
 				Size:          imageData.Size,
 			}
 
-			if err := recorder.Register(img, nil); err != nil {
+			if err := recorder.Register(customImageDescriptor{img}, nil); err != nil {
 				return nil, err
 			}
 

graph/export.go

@@ -112,6 +112,11 @@ func (s *TagStore) ImageExport(names []string, outStream io.Writer) error {
 
 func (s *TagStore) exportImage(name, tempdir string) error {
 	for n := name; n != ""; {
+		img, err := s.LookupImage(n)
+		if err != nil || img == nil {
+			return fmt.Errorf("No such image %s", n)
+		}
+
 		// temporary directory
 		tmpImageDir := filepath.Join(tempdir, n)
 		if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil {
@@ -128,19 +133,17 @@ func (s *TagStore) exportImage(name, tempdir string) error {
 			return err
 		}
 
+		imageInspectRaw, err := json.Marshal(img)
+		if err != nil {
+			return err
+		}
+
 		// serialize json
 		json, err := os.Create(filepath.Join(tmpImageDir, "json"))
 		if err != nil {
 			return err
 		}
-		img, err := s.LookupImage(n)
+
-		if err != nil || img == nil {
-			return fmt.Errorf("No such image %s", n)
-		}
-		imageInspectRaw, err := s.graph.RawJSON(img.ID)
-		if err != nil {
-			return err
-		}
 		written, err := json.Write(imageInspectRaw)
 		if err != nil {
 			return err

graph/graph.go

@@ -31,6 +31,26 @@ import (
 	"github.com/vbatts/tar-split/tar/storage"
 )
 
+// v1Descriptor is a non-content-addressable image descriptor
+type v1Descriptor struct {
+	img *image.Image
+}
+
+// ID returns the image ID specified in the image structure.
+func (img v1Descriptor) ID() string {
+	return img.img.ID
+}
+
+// Parent returns the parent ID specified in the image structure.
+func (img v1Descriptor) Parent() string {
+	return img.img.Parent
+}
+
+// MarshalConfig renders the image structure into JSON.
+func (img v1Descriptor) MarshalConfig() ([]byte, error) {
+	return json.Marshal(img.img)
+}
+
 // The type is used to protect pulling or building related image
 // layers from deleteing when filtered by dangling=true
 // The key of layers is the images ID which is pulling or building
@@ -88,10 +108,12 @@ type Graph struct {
 
 // file names for ./graph/<ID>/
 const (
-	jsonFileName      = "json"
+	jsonFileName            = "json"
-	layersizeFileName = "layersize"
+	layersizeFileName       = "layersize"
-	digestFileName    = "checksum"
+	digestFileName          = "checksum"
-	tarDataFileName   = "tar-data.json.gz"
+	tarDataFileName         = "tar-data.json.gz"
+	v1CompatibilityFileName = "v1Compatibility"
+	parentFileName          = "parent"
 )
 
 var (
@@ -225,7 +247,7 @@ func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, com
 		img.ContainerConfig = *containerConfig
 	}
 
-	if err := graph.Register(img, layerData); err != nil {
+	if err := graph.Register(v1Descriptor{img}, layerData); err != nil {
 		return nil, err
 	}
 	return img, nil
@@ -233,19 +255,26 @@ func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, com
 
 // Register imports a pre-existing image into the graph.
 // Returns nil if the image is already registered.
-func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) {
+func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) {
+	imgID := im.ID()
 
-	if err := image.ValidateID(img.ID); err != nil {
+	if err := image.ValidateID(imgID); err != nil {
 		return err
 	}
 
 	// We need this entire operation to be atomic within the engine. Note that
 	// this doesn't mean Register is fully safe yet.
-	graph.imageMutex.Lock(img.ID)
+	graph.imageMutex.Lock(imgID)
-	defer graph.imageMutex.Unlock(img.ID)
+	defer graph.imageMutex.Unlock(imgID)
+
+	return graph.register(im, layerData)
+}
+
+func (graph *Graph) register(im image.Descriptor, layerData io.Reader) (err error) {
+	imgID := im.ID()
 
 	// Skip register if image is already registered
-	if graph.Exists(img.ID) {
+	if graph.Exists(imgID) {
 		return nil
 	}
 
@@ -255,14 +284,14 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error)
 		// If any error occurs, remove the new dir from the driver.
 		// Don't check for errors since the dir might not have been created.
 		if err != nil {
-			graph.driver.Remove(img.ID)
+			graph.driver.Remove(imgID)
 		}
 	}()
 
 	// Ensure that the image root does not exist on the filesystem
 	// when it is not registered in the graph.
 	// This is common when you switch from one graph driver to another
-	if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
+	if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 
@@ -270,7 +299,7 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error)
 	// (the graph is the source of truth).
 	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
 	// (FIXME: make that mandatory for drivers).
-	graph.driver.Remove(img.ID)
+	graph.driver.Remove(imgID)
 
 	tmp, err := graph.mktemp()
 	defer os.RemoveAll(tmp)
@@ -278,26 +307,32 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error)
 		return fmt.Errorf("mktemp failed: %s", err)
 	}
 
+	parent := im.Parent()
+
 	// Create root filesystem in the driver
-	if err := createRootFilesystemInDriver(graph, img); err != nil {
+	if err := createRootFilesystemInDriver(graph, imgID, parent, layerData); err != nil {
 		return err
 	}
 
 	// Apply the diff/layer
-	if err := graph.storeImage(img, layerData, tmp); err != nil {
+	config, err := im.MarshalConfig()
+	if err != nil {
+		return err
+	}
+	if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil {
 		return err
 	}
 	// Commit
-	if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
+	if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil {
 		return err
 	}
-	graph.idIndex.Add(img.ID)
+	graph.idIndex.Add(imgID)
 	return nil
 }
 
-func createRootFilesystemInDriver(graph *Graph, img *image.Image) error {
+func createRootFilesystemInDriver(graph *Graph, id, parent string, layerData io.Reader) error {
-	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
+	if err := graph.driver.Create(id, parent); err != nil {
-		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
+		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err)
 	}
 	return nil
 }
@@ -480,6 +515,21 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) {
 	if err := dec.Decode(img); err != nil {
 		return nil, err
 	}
+
+	if img.ID == "" {
+		img.ID = id
+	}
+
+	if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
+		img.Parent = img.ParentID.Hex()
+	}
+
+	// compatibilityID for parent
+	parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
+	if err == nil && len(parent) > 0 {
+		img.Parent = string(parent)
+	}
+
 	if err := image.ValidateID(img.ID); err != nil {
 		return nil, err
 	}
@@ -513,11 +563,14 @@ func (graph *Graph) saveSize(root string, size int64) error {
 	return nil
 }
 
-// SetDigest sets the digest for the image layer to the provided value.
+// SetLayerDigest sets the digest for the image layer to the provided value.
-func (graph *Graph) SetDigest(id string, dgst digest.Digest) error {
+func (graph *Graph) SetLayerDigest(id string, dgst digest.Digest) error {
 	graph.imageMutex.Lock(id)
 	defer graph.imageMutex.Unlock(id)
 
+	return graph.setLayerDigest(id, dgst)
+}
+func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error {
 	root := graph.imageRoot(id)
 	if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil {
 		return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err)
@@ -525,11 +578,15 @@ func (graph *Graph) SetDigest(id string, dgst digest.Digest) error {
 	return nil
 }
 
-// GetDigest gets the digest for the provide image layer id.
+// GetLayerDigest gets the digest for the provide image layer id.
-func (graph *Graph) GetDigest(id string) (digest.Digest, error) {
+func (graph *Graph) GetLayerDigest(id string) (digest.Digest, error) {
 	graph.imageMutex.Lock(id)
 	defer graph.imageMutex.Unlock(id)
 
+	return graph.getLayerDigest(id)
+}
+
+func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) {
 	root := graph.imageRoot(id)
 	cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName))
 	if err != nil {
@@ -541,6 +598,76 @@ func (graph *Graph) GetDigest(id string) (digest.Digest, error) {
 	return digest.ParseDigest(string(cs))
 }
 
+// SetV1CompatibilityConfig stores the v1Compatibility JSON data associated
+// with the image in the manifest to the disk
+func (graph *Graph) SetV1CompatibilityConfig(id string, data []byte) error {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.setV1CompatibilityConfig(id, data)
+}
+func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error {
+	root := graph.imageRoot(id)
+	return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600)
+}
+
+// GetV1CompatibilityConfig reads the v1Compatibility JSON data for the image
+// from the disk
+func (graph *Graph) GetV1CompatibilityConfig(id string) ([]byte, error) {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.getV1CompatibilityConfig(id)
+}
+
+func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) {
+	root := graph.imageRoot(id)
+	return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName))
+}
+
+// GenerateV1CompatibilityChain makes sure v1Compatibility JSON data exists
+// for the image. If it doesn't it generates and stores it for the image and
+// all of it's parents based on the image config JSON.
+func (graph *Graph) GenerateV1CompatibilityChain(id string) ([]byte, error) {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	if v1config, err := graph.getV1CompatibilityConfig(id); err == nil {
+		return v1config, nil
+	}
+
+	// generate new, store it to disk
+	img, err := graph.Get(id)
+	if err != nil {
+		return nil, err
+	}
+
+	digestPrefix := string(digest.Canonical) + ":"
+	img.ID = strings.TrimPrefix(img.ID, digestPrefix)
+
+	if img.Parent != "" {
+		parentConfig, err := graph.GenerateV1CompatibilityChain(img.Parent)
+		if err != nil {
+			return nil, err
+		}
+		var parent struct{ ID string }
+		err = json.Unmarshal(parentConfig, &parent)
+		if err != nil {
+			return nil, err
+		}
+		img.Parent = parent.ID
+	}
+
+	json, err := json.Marshal(img)
+	if err != nil {
+		return nil, err
+	}
+	if err := graph.setV1CompatibilityConfig(id, json); err != nil {
+		return nil, err
+	}
+	return json, nil
+}
+
 // RawJSON returns the JSON representation for an image as a byte array.
 func (graph *Graph) RawJSON(id string) ([]byte, error) {
 	root := graph.imageRoot(id)
@@ -560,29 +687,38 @@ func jsonPath(root string) string {
 // storeImage stores file system layer data for the given image to the
 // graph's storage driver. Image metadata is stored in a file
 // at the specified root directory.
-func (graph *Graph) storeImage(img *image.Image, layerData io.Reader, root string) (err error) {
+func (graph *Graph) storeImage(id, parent string, config []byte, layerData io.Reader, root string) (err error) {
+	var size int64
 	// Store the layer. If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil {
+		if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
 			return err
 		}
 	}
 
-	if err := graph.saveSize(root, img.Size); err != nil {
+	if err := graph.saveSize(root, size); err != nil {
 		return err
 	}
 
-	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
+		return err
+	}
+
+	// If image is pointing to a parent via CompatibilityID write the reference to disk
+	img, err := image.NewImgJSON(config)
 	if err != nil {
 		return err
 	}
 
-	defer f.Close()
+	if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() {
-
+		if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
-	return json.NewEncoder(f).Encode(img)
+			return err
+		}
+	}
+	return nil
 }
 
-func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.Reader, root string) (err error) {
+func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData io.Reader, root string) (size int64, err error) {
 	var ar io.Reader
 
 	if graph.tarSplitDisabled {
@@ -591,7 +727,7 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.R
 		// this is saving the tar-split metadata
 		mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
 		if err != nil {
-			return err
+			return 0, err
 		}
 
 		mfz := gzip.NewWriter(mf)
@@ -601,24 +737,24 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.R
 
 		inflatedLayerData, err := archive.DecompressStream(layerData)
 		if err != nil {
-			return err
+			return 0, err
 		}
 
 		// we're passing nil here for the file putter, because the ApplyDiff will
 		// handle the extraction of the archive
 		rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
 		if err != nil {
-			return err
+			return 0, err
 		}
 
 		ar = archive.Reader(rdr)
 	}
 
-	if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, ar); err != nil {
+	if size, err = graph.driver.ApplyDiff(id, parent, ar); err != nil {
-		return err
+		return 0, err
 	}
 
-	return nil
+	return
 }
 
 func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) {

graph/graph_test.go

@@ -73,7 +73,7 @@ func TestInterruptedRegister(t *testing.T) {
 		Created: time.Now(),
 	}
 	w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
-	graph.Register(image, badArchive)
+	graph.Register(v1Descriptor{image}, badArchive)
 	if _, err := graph.Get(image.ID); err == nil {
 		t.Fatal("Image should not exist after Register is interrupted")
 	}
@@ -82,7 +82,7 @@ func TestInterruptedRegister(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := graph.Register(image, goodArchive); err != nil {
+	if err := graph.Register(v1Descriptor{image}, goodArchive); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -130,7 +130,7 @@ func TestRegister(t *testing.T) {
 		Comment: "testing",
 		Created: time.Now(),
 	}
-	err = graph.Register(image, archive)
+	err = graph.Register(v1Descriptor{image}, archive)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,7 +212,7 @@ func TestDelete(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Test delete twice (pull -> rm -> pull -> rm)
-	if err := graph.Register(img1, archive); err != nil {
+	if err := graph.Register(v1Descriptor{img1}, archive); err != nil {
 		t.Fatal(err)
 	}
 	if err := graph.Delete(img1.ID); err != nil {
@@ -246,9 +246,19 @@ func TestByParent(t *testing.T) {
 		Created: time.Now(),
 		Parent:  parentImage.ID,
 	}
-	_ = graph.Register(parentImage, archive1)
+
-	_ = graph.Register(childImage1, archive2)
+	err := graph.Register(v1Descriptor{parentImage}, archive1)
-	_ = graph.Register(childImage2, archive3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = graph.Register(v1Descriptor{childImage1}, archive2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = graph.Register(v1Descriptor{childImage2}, archive3)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	byParent := graph.ByParent()
 	numChildren := len(byParent[parentImage.ID])

graph/load.go

@@ -122,7 +122,7 @@ func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
 				}
 			}
 		}
-		if err := s.graph.Register(img, layer); err != nil {
+		if err := s.graph.Register(v1Descriptor{img}, layer); err != nil {
 			return err
 		}
 	}

graph/pull_v1.go

@@ -127,7 +127,7 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 	defer func() {
 		p.graph.Release(sessionID, imgIDs...)
 	}()
-	for _, image := range repoData.ImgList {
+	for _, imgData := range repoData.ImgList {
 		downloadImage := func(img *registry.ImgData) {
 			if askedTag != "" && img.Tag != askedTag {
 				errors <- nil
@@ -140,6 +140,11 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 				return
 			}
 
+			if err := image.ValidateID(img.ID); err != nil {
+				errors <- err
+				return
+			}
+
 			// ensure no two downloads of the same image happen at the same time
 			poolKey := "img:" + img.ID
 			broadcaster, found := p.poolAdd("pull", poolKey)
@@ -197,7 +202,7 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 			errors <- nil
 		}
 
-		go downloadImage(image)
+		go downloadImage(imgData)
 	}
 
 	var lastError error
@@ -317,7 +322,7 @@ func (p *v1Puller) pullImage(out io.Writer, imgID, endpoint string) (layersDownl
 				layersDownloaded = true
 				defer layer.Close()
 
-				err = p.graph.Register(img,
+				err = p.graph.Register(v1Descriptor{img},
 					progressreader.New(progressreader.Config{
 						In:        layer,
 						Out:       broadcaster,

graph/pull_v2.go

@@ -1,10 +1,12 @@
 package graph
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"sync"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution"
@@ -73,7 +75,8 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) {
 
 	}
 
-	broadcaster, found := p.poolAdd("pull", taggedName)
+	poolKey := "v2:" + taggedName
+	broadcaster, found := p.poolAdd("pull", poolKey)
 	broadcaster.Add(p.config.OutStream)
 	if found {
 		// Another pull of the same repository is already taking place; just wait for it to finish
@@ -83,7 +86,7 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) {
 	// This must use a closure so it captures the value of err when the
 	// function returns, not when the 'defer' is evaluated.
 	defer func() {
-		p.poolRemoveWithError("pull", taggedName, err)
+		p.poolRemoveWithError("pull", poolKey, err)
 	}()
 
 	var layersDownloaded bool
@@ -104,7 +107,8 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) {
 
 // downloadInfo is used to pass information from download to extractor
 type downloadInfo struct {
-	img         *image.Image
+	img         contentAddressableDescriptor
+	imgIndex    int
 	tmpFile     *os.File
 	digest      digest.Digest
 	layer       distribution.ReadSeekCloser
@@ -114,12 +118,66 @@ type downloadInfo struct {
 	broadcaster *broadcaster.Buffered
 }
 
+// contentAddressableDescriptor is used to pass image data from a manifest to the
+// graph.
+type contentAddressableDescriptor struct {
+	id              string
+	parent          string
+	strongID        digest.Digest
+	compatibilityID string
+	config          []byte
+	v1Compatibility []byte
+}
+
+func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) {
+	img := contentAddressableDescriptor{
+		v1Compatibility: v1Compatibility,
+	}
+
+	var err error
+	img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent)
+	if err != nil {
+		return img, err
+	}
+	img.strongID, err = image.StrongID(img.config)
+	if err != nil {
+		return img, err
+	}
+
+	unmarshalledConfig, err := image.NewImgJSON(v1Compatibility)
+	if err != nil {
+		return img, err
+	}
+
+	img.compatibilityID = unmarshalledConfig.ID
+	img.id = img.strongID.Hex()
+
+	return img, nil
+}
+
+// ID returns the actual ID to be used for the downloaded image. This may be
+// a computed ID.
+func (img contentAddressableDescriptor) ID() string {
+	return img.id
+}
+
+// Parent returns the parent ID to be used for the image. This may be a
+// computed ID.
+func (img contentAddressableDescriptor) Parent() string {
+	return img.parent
+}
+
+// MarshalConfig renders the image structure into JSON.
+func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) {
+	return img.config, nil
+}
+
 type errVerification struct{}
 
 func (errVerification) Error() string { return "verification failed" }
 
 func (p *v2Puller) download(di *downloadInfo) {
-	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID)
+	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id)
 
 	blobs := p.repo.Blobs(context.Background())
 
@@ -151,12 +209,12 @@ func (p *v2Puller) download(di *downloadInfo) {
 		Formatter: p.sf,
 		Size:      di.size,
 		NewLines:  false,
-		ID:        stringid.TruncateID(di.img.ID),
+		ID:        stringid.TruncateID(di.img.id),
 		Action:    "Downloading",
 	})
 	io.Copy(di.tmpFile, reader)
 
-	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil))
+	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil))
 
 	if !verifier.Verified() {
 		err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
@@ -165,9 +223,9 @@ func (p *v2Puller) download(di *downloadInfo) {
 		return
 	}
 
-	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil))
+	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
 
-	logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, di.tmpFile.Name())
+	logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, di.tmpFile.Name())
 	di.layer = layerDownload
 
 	di.err <- nil
@@ -193,6 +251,17 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo
 		logrus.Printf("Image manifest for %s has been verified", taggedName)
 	}
 
+	// remove duplicate layers and check parent chain validity
+	err = fixManifestLayers(&manifest.Manifest)
+	if err != nil {
+		return false, err
+	}
+
+	imgs, err := p.getImageInfos(manifest.Manifest)
+	if err != nil {
+		return false, err
+	}
+
 	out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
 
 	var downloads []*downloadInfo
@@ -213,26 +282,32 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo
 	}()
 
 	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
-		img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
+		img := imgs[i]
-		if err != nil {
+
-			logrus.Debugf("error getting image v1 json: %v", err)
+		p.graph.Retain(p.sessionID, img.id)
-			return false, err
+		layerIDs = append(layerIDs, img.id)
-		}
+
-		p.graph.Retain(p.sessionID, img.ID)
+		p.graph.imageMutex.Lock(img.id)
-		layerIDs = append(layerIDs, img.ID)
 
 		// Check if exists
-		if p.graph.Exists(img.ID) {
+		if p.graph.Exists(img.id) {
-			logrus.Debugf("Image already exists: %s", img.ID)
+			if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
-			out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil))
+				p.graph.imageMutex.Unlock(img.id)
+				return false, fmt.Errorf("image validation failed: %v", err)
+			}
+			logrus.Debugf("Image already exists: %s", img.id)
+			p.graph.imageMutex.Unlock(img.id)
 			continue
 		}
-		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
+		p.graph.imageMutex.Unlock(img.id)
+
+		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
 
 		d := &downloadInfo{
-			img:     img,
+			img:      img,
-			poolKey: "layer:" + img.ID,
+			imgIndex: i,
-			digest:  manifest.FSLayers[i].BlobSum,
+			poolKey:  "v2layer:" + img.id,
+			digest:   manifest.FSLayers[i].BlobSum,
 			// TODO: seems like this chan buffer solved hanging problem in go1.5,
 			// this can indicate some deeper problem that somehow we never take
 			// error from channel in loop below
@@ -274,26 +349,49 @@ func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bo
 		}
 
 		d.tmpFile.Seek(0, 0)
-		reader := progressreader.New(progressreader.Config{
+		err := func() error {
-			In:        d.tmpFile,
+			reader := progressreader.New(progressreader.Config{
-			Out:       d.broadcaster,
+				In:        d.tmpFile,
-			Formatter: p.sf,
+				Out:       d.broadcaster,
-			Size:      d.size,
+				Formatter: p.sf,
-			NewLines:  false,
+				Size:      d.size,
-			ID:        stringid.TruncateID(d.img.ID),
+				NewLines:  false,
-			Action:    "Extracting",
+				ID:        stringid.TruncateID(d.img.id),
-		})
+				Action:    "Extracting",
+			})
 
-		err = p.graph.Register(d.img, reader)
+			p.graph.imageMutex.Lock(d.img.id)
+			defer p.graph.imageMutex.Unlock(d.img.id)
+
+			// Must recheck the data on disk if any exists.
+			// This protects against races where something
+			// else is written to the graph under this ID
+			// after attemptIDReuse.
+			if p.graph.Exists(d.img.id) {
+				if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil {
+					return fmt.Errorf("image validation failed: %v", err)
+				}
+			}
+
+			if err := p.graph.register(d.img, reader); err != nil {
+				return err
+			}
+
+			if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
+				return err
+			}
+
+			if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
+				return err
+			}
+
+			return nil
+		}()
 		if err != nil {
 			return false, err
 		}
 
-		if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
+		d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
-			return false, err
-		}
-
-		d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
 		d.broadcaster.Close()
 		tagUpdated = true
 	}
@@ -424,3 +522,217 @@ func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (ver
 	}
 	return verified, nil
 }
+
+// fixManifestLayers removes repeated layers from the manifest and checks the
+// correctness of the parent chain.
+func fixManifestLayers(m *manifest.Manifest) error {
+	images := make([]*image.Image, len(m.FSLayers))
+	for i := range m.FSLayers {
+		img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
+		if err != nil {
+			return err
+		}
+		images[i] = img
+		if err := image.ValidateID(img.ID); err != nil {
+			return err
+		}
+	}
+
+	if images[len(images)-1].Parent != "" {
+		return errors.New("Invalid parent ID in the base layer of the image.")
+	}
+
+	// check general duplicates to error instead of a deadlock
+	idmap := make(map[string]struct{})
+
+	var lastID string
+	for _, img := range images {
+		// skip IDs that appear after each other, we handle those later
+		if _, exists := idmap[img.ID]; img.ID != lastID && exists {
+			return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
+		}
+		lastID = img.ID
+		idmap[lastID] = struct{}{}
+	}
+
+	// backwards loop so that we keep the remaining indexes after removing items
+	for i := len(images) - 2; i >= 0; i-- {
+		if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
+			m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
+			m.History = append(m.History[:i], m.History[i+1:]...)
+		} else if images[i].Parent != images[i+1].ID {
+			return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
+		}
+	}
+
+	return nil
+}
+
+// getImageInfos returns an imageinfo struct for every image in the manifest.
+// These objects contain both calculated strongIDs and compatibilityIDs found
+// in v1Compatibility object.
+func (p *v2Puller) getImageInfos(m manifest.Manifest) ([]contentAddressableDescriptor, error) {
+	imgs := make([]contentAddressableDescriptor, len(m.FSLayers))
+
+	var parent digest.Digest
+	for i := len(imgs) - 1; i >= 0; i-- {
+		var err error
+		imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent)
+		if err != nil {
+			return nil, err
+		}
+		parent = imgs[i].strongID
+	}
+
+	p.attemptIDReuse(imgs)
+
+	return imgs, nil
+}
+
+var idReuseLock sync.Mutex
+
+// attemptIDReuse does a best attempt to match verified compatibilityIDs
+// already in the graph with the computed strongIDs so we can keep using them.
+// This process will never fail but may just return the strongIDs if none of
+// the compatibilityIDs exists or can be verified. If the strongIDs themselves
+// fail verification, we deterministically generate alternate IDs to use until
+// we find one that's available or already exists with the correct data.
+func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) {
+	// This function needs to be protected with a global lock, because it
+	// locks multiple IDs at once, and there's no good way to make sure
+	// the locking happens a deterministic order.
+	idReuseLock.Lock()
+	defer idReuseLock.Unlock()
+
+	idMap := make(map[string]struct{})
+	for _, img := range imgs {
+		idMap[img.id] = struct{}{}
+		idMap[img.compatibilityID] = struct{}{}
+
+		if p.graph.Exists(img.compatibilityID) {
+			if _, err := p.graph.GenerateV1CompatibilityChain(img.compatibilityID); err != nil {
+				logrus.Debugf("Migration v1Compatibility generation error: %v", err)
+				return
+			}
+		}
+	}
+	for id := range idMap {
+		p.graph.imageMutex.Lock(id)
+		defer p.graph.imageMutex.Unlock(id)
+	}
+
+	// continueReuse controls whether the function will try to find
+	// existing layers on disk under the old v1 IDs, to avoid repulling
+	// them. The hashes are checked to ensure these layers are okay to
+	// use. continueReuse starts out as true, but is set to false if
+	// the code encounters something that doesn't match the expected hash.
+	continueReuse := true
+
+	for i := len(imgs) - 1; i >= 0; i-- {
+		if p.graph.Exists(imgs[i].id) {
+			// Found an image in the graph under the strongID. Validate the
+			// image before using it.
+			if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
+				continueReuse = false
+				logrus.Debugf("not using existing strongID: %v", err)
+
+				// The strong ID existed in the graph but didn't
+				// validate successfully. We can't use the strong ID
+				// because it didn't validate successfully. Treat the
+				// graph like a hash table with probing... compute
+				// SHA256(id) until we find an ID that either doesn't
+				// already exist in the graph, or has existing content
+				// that validates successfully.
+				for {
+					if err := p.tryNextID(imgs, i, idMap); err != nil {
+						logrus.Debug(err.Error())
+					} else {
+						break
+					}
+				}
+			}
+			continue
+		}
+
+		if continueReuse {
+			compatibilityID := imgs[i].compatibilityID
+			if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil {
+				logrus.Debugf("stopping ID reuse: %v", err)
+				continueReuse = false
+			} else {
+				// The compatibility ID exists in the graph and was
+				// validated. Use it.
+				imgs[i].id = compatibilityID
+			}
+		}
+	}
+
+	// fix up the parents of the images
+	for i := 0; i < len(imgs); i++ {
+		if i == len(imgs)-1 { // Base layer
+			imgs[i].parent = ""
+		} else {
+			imgs[i].parent = imgs[i+1].id
+		}
+	}
+}
+
+// validateImageInGraph checks that an image in the graph has the expected
+// strongID. id is the entry in the graph to check, imgs is the slice of
+// images being processed (for access to the parent), and i is the index
+// into this slice which the graph entry should be checked against.
+func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error {
+	img, err := p.graph.Get(id)
+	if err != nil {
+		return fmt.Errorf("missing: %v", err)
+	}
+	layerID, err := p.graph.getLayerDigest(id)
+	if err != nil {
+		return fmt.Errorf("digest: %v", err)
+	}
+	var parentID digest.Digest
+	if i != len(imgs)-1 {
+		if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID
+			return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id)
+		}
+		parentID = imgs[i+1].strongID
+	} else if img.Parent != "" {
+		return fmt.Errorf("unexpected parent: %v", img.Parent)
+	}
+
+	v1Config, err := p.graph.getV1CompatibilityConfig(img.ID)
+	if err != nil {
+		return fmt.Errorf("v1Compatibility: %v %v", img.ID, err)
+	}
+
+	json, err := image.MakeImageConfig(v1Config, layerID, parentID)
+	if err != nil {
+		return fmt.Errorf("make config: %v", err)
+	}
+
+	if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID {
+		logrus.Debugf("Validated %v as %v", dgst, id)
+	} else {
+		return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err)
+	}
+
+	// All clear
+	return nil
+}
+
+func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error {
+	nextID, _ := digest.FromBytes([]byte(imgs[i].id))
+	imgs[i].id = nextID.Hex()
+
+	if _, exists := idMap[imgs[i].id]; !exists {
+		p.graph.imageMutex.Lock(imgs[i].id)
+		defer p.graph.imageMutex.Unlock(imgs[i].id)
+	}
+
+	if p.graph.Exists(imgs[i].id) {
+		if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
+			return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err)
+		}
+	}
+	return nil
+}

graph/pull_v2_test.go

@@ -0,0 +1,100 @@
+package graph
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+)
+
+// TestFixManifestLayers checks that fixManifestLayers removes a duplicate
+// layer, and that it makes no changes to the manifest when called a second
+// time, after the duplicate is removed.
+func TestFixManifestLayers(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	duplicateLayerManifestExpectedOutput := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
+		t.Fatalf("unexpected error from fixManifestLayers: %v", err)
+	}
+
+	if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
+		t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest")
+	}
+
+	// Run fixManifestLayers again and confirm that it doesn't change the
+	// manifest (which no longer has duplicate layers).
+	if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
+		t.Fatalf("unexpected error from fixManifestLayers: %v", err)
+	}
+
+	if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
+		t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)")
+	}
+}
+
+// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails
+// if the base layer configuration specifies a parent.
+func TestFixManifestLayersBaseLayerParent(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") {
+		t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
+	}
+}
+
+// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails
+// if an image configuration specifies a parent that doesn't directly follow
+// that (deduplicated) image in the image history.
+func TestFixManifestLayersBadParent(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") {
+		t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
+	}
+}

graph/push_v1.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/registry/client/transport"
+	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/streamformatter"
@@ -127,7 +128,7 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [
 			continue
 		}
 		// If the image does not have a tag it still needs to be sent to the
-		// registry with an empty tag so that it is accociated with the repository
+		// registry with an empty tag so that it is associated with the repository
 		imageIndex = append(imageIndex, &registry.ImgData{
 			ID:  id,
 			Tag: "",
@@ -137,8 +138,9 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [
 }
 
 type imagePushData struct {
-	id       string
+	id              string
-	endpoint string
+	compatibilityID string
+	endpoint        string
 }
 
 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists
@@ -146,7 +148,7 @@ type imagePushData struct {
 func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
 	defer wg.Done()
 	for image := range images {
-		if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil {
+		if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil {
 			logrus.Errorf("Error in LookupRemoteImage: %s", err)
 			imagesToPush <- image.id
 			continue
@@ -180,9 +182,14 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags
 		pushes <- shouldPush
 	}()
 	for _, id := range imageIDs {
+		compatibilityID, err := p.getV1ID(id)
+		if err != nil {
+			return err
+		}
 		imageData <- imagePushData{
-			id:       id,
+			id:              id,
-			endpoint: endpoint,
+			compatibilityID: compatibilityID,
+			endpoint:        endpoint,
 		}
 	}
 	// close the channel to notify the workers that there will be no more images to check.
@@ -202,7 +209,11 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags
 		}
 		for _, tag := range tags[id] {
 			p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
-			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil {
+			compatibilityID, err := p.getV1ID(id)
+			if err != nil {
+				return err
+			}
+			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil {
 				return err
 			}
 		}
@@ -224,6 +235,12 @@ func (p *v1Pusher) pushRepository(tag string) error {
 	logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
 	for _, data := range imageIndex {
 		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
+
+		// convert IDs to compatibilityIDs, imageIndex only used in registry calls
+		data.ID, err = p.getV1ID(data.ID)
+		if err != nil {
+			return err
+		}
 	}
 
 	if _, found := p.poolAdd("push", p.repoInfo.LocalName); found {
@@ -253,20 +270,27 @@ func (p *v1Pusher) pushRepository(tag string) error {
 }
 
 func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
-	jsonRaw, err := p.graph.RawJSON(imgID)
+	jsonRaw, err := p.getV1Config(imgID)
 	if err != nil {
 		return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
 	}
 	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
 
+	compatibilityID, err := p.getV1ID(imgID)
+	if err != nil {
+		return "", err
+	}
+
+	// General rule is to use ID for graph accesses and compatibilityID for
+	// calls to session.registry()
 	imgData := &registry.ImgData{
-		ID: imgID,
+		ID: compatibilityID,
 	}
 
 	// Send the json
 	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
 		if err == registry.ErrAlreadyExists {
-			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil))
+			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil))
 			return "", nil
 		}
 		return "", err
@@ -279,7 +303,7 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
 	defer os.RemoveAll(layerData.Name())
 
 	// Send the layer
-	logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size)
+	logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size)
 
 	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
 		progressreader.New(progressreader.Config{
@@ -288,7 +312,7 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
 			Formatter: p.sf,
 			Size:      layerData.Size,
 			NewLines:  false,
-			ID:        stringid.TruncateID(imgData.ID),
+			ID:        stringid.TruncateID(imgID),
 			Action:    "Pushing",
 		}), ep, jsonRaw)
 	if err != nil {
@@ -301,6 +325,30 @@ func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
 		return "", err
 	}
 
-	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil))
+	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil))
 	return imgData.Checksum, nil
 }
+
+// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID
+// is read from from the v1Compatibility config file in the disk.
+func (p *v1Pusher) getV1ID(id string) (string, error) {
+	jsonData, err := p.getV1Config(id)
+	if err != nil {
+		return "", err
+	}
+	img, err := image.NewImgJSON(jsonData)
+	if err != nil {
+		return "", err
+	}
+	return img.ID, nil
+}
+
+// getV1Config returns v1Compatibility config for the image in the graph. If
+// there is no v1Compatibility file on disk for the image
+func (p *v1Pusher) getV1Config(id string) ([]byte, error) {
+	jsonData, err := p.graph.GenerateV1CompatibilityChain(id)
+	if err != nil {
+		return nil, err
+	}
+	return jsonData, nil
+}

graph/push_v2.go

@@ -138,13 +138,8 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 			}
 		}
 
-		jsonData, err := p.graph.RawJSON(layer.ID)
-		if err != nil {
-			return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err)
-		}
-
 		var exists bool
-		dgst, err := p.graph.GetDigest(layer.ID)
+		dgst, err := p.graph.GetLayerDigest(layer.ID)
 		switch err {
 		case nil:
 			if p.layersPushed[dgst] {
@@ -178,13 +173,19 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 				return err
 			} else if pushDigest != dgst {
 				// Cache new checksum
-				if err := p.graph.SetDigest(layer.ID, pushDigest); err != nil {
+				if err := p.graph.SetLayerDigest(layer.ID, pushDigest); err != nil {
 					return err
 				}
 				dgst = pushDigest
 			}
 		}
 
+		// read v1Compatibility config, generate new if needed
+		jsonData, err := p.graph.GenerateV1CompatibilityChain(layer.ID)
+		if err != nil {
+			return err
+		}
+
 		m.FSLayers = append(m.FSLayers, manifest.FSLayer{BlobSum: dgst})
 		m.History = append(m.History, manifest.History{V1Compatibility: string(jsonData)})
 

graph/tags_unit_test.go

@@ -82,7 +82,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 	img := &image.Image{ID: testOfficialImageID}
-	if err := graph.Register(img, officialArchive); err != nil {
+	if err := graph.Register(v1Descriptor{img}, officialArchive); err != nil {
 		t.Fatal(err)
 	}
 	if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil {
@@ -93,7 +93,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 	img = &image.Image{ID: testPrivateImageID}
-	if err := graph.Register(img, privateArchive); err != nil {
+	if err := graph.Register(v1Descriptor{img}, privateArchive); err != nil {
 		t.Fatal(err)
 	}
 	if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil {

image/fixtures/post1.9/expected_computed_id

@@ -0,0 +1 @@
+sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105

image/fixtures/post1.9/expected_config

@@ -0,0 +1 @@
+{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

image/fixtures/post1.9/layer_id

@@ -0,0 +1 @@
+sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/post1.9/parent_id

@@ -0,0 +1 @@
+sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/post1.9/v1compatibility

@@ -0,0 +1 @@
+{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}

image/fixtures/pre1.9/expected_computed_id

@@ -0,0 +1 @@
+sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89

image/fixtures/pre1.9/expected_config

@@ -0,0 +1 @@
+{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

image/fixtures/pre1.9/layer_id

@@ -0,0 +1 @@
+sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/pre1.9/parent_id

@@ -0,0 +1 @@
+sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/pre1.9/v1compatibility

@@ -0,0 +1 @@
+{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0}

image/image.go

@@ -2,19 +2,38 @@ package image
 
 import (
 	"encoding/json"
+	"fmt"
 	"regexp"
 	"time"
 
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
 	derr "github.com/docker/docker/errors"
+	"github.com/docker/docker/pkg/version"
 	"github.com/docker/docker/runconfig"
 )
 
 var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
 
+// noFallbackMinVersion is the minimum version for which v1compatibility
+// information will not be marshaled through the Image struct to remove
+// blank fields.
+var noFallbackMinVersion = version.Version("1.8.3")
+
+// Descriptor provides the information necessary to register an image in
+// the graph.
+type Descriptor interface {
+	ID() string
+	Parent() string
+	MarshalConfig() ([]byte, error)
+}
+
 // Image stores the image configuration.
+// All fields in this struct must be marked `omitempty` to keep getting
+// predictable hashes from the old `v1Compatibility` configuration.
 type Image struct {
 	// ID a unique 64 character identifier of the image
-	ID string `json:"id"`
+	ID string `json:"id,omitempty"`
 	// Parent id of the image
 	Parent string `json:"parent,omitempty"`
 	// Comment user added comment
@@ -36,7 +55,11 @@ type Image struct {
 	// OS is the operating system used to build and run the image
 	OS string `json:"os,omitempty"`
 	// Size is the total size of the image including all layers it is composed of
-	Size int64
+	Size int64 `json:",omitempty"` // capitalized for backwards compatibility
+	// ParentID specifies the strong, content address of the parent configuration.
+	ParentID digest.Digest `json:"parent_id,omitempty"`
+	// LayerID provides the content address of the associated layer.
+	LayerID digest.Digest `json:"layer_id,omitempty"`
 }
 
 // NewImgJSON creates an Image configuration from json.
@@ -57,3 +80,70 @@ func ValidateID(id string) error {
 	}
 	return nil
 }
+
+// MakeImageConfig returns immutable configuration JSON for image based on the
+// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
+// config is the new image ID (strongID).
+func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
+
+	// Detect images created after 1.8.3
+	img, err := NewImgJSON(v1Compatibility)
+	if err != nil {
+		return nil, err
+	}
+	useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
+
+	if useFallback {
+		// Fallback for pre-1.8.3. Calculate base config based on Image struct
+		// so that fields with default values added by Docker will use same ID
+		logrus.Debugf("Using fallback hash for %v", layerID)
+
+		v1Compatibility, err = json.Marshal(img)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var c map[string]*json.RawMessage
+	if err := json.Unmarshal(v1Compatibility, &c); err != nil {
+		return nil, err
+	}
+
+	if err := layerID.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid layerID: %v", err)
+	}
+
+	c["layer_id"] = rawJSON(layerID)
+
+	if parentID != "" {
+		if err := parentID.Validate(); err != nil {
+			return nil, fmt.Errorf("invalid parentID %v", err)
+		}
+		c["parent_id"] = rawJSON(parentID)
+	}
+
+	delete(c, "id")
+	delete(c, "parent")
+	delete(c, "Size") // Size is calculated from data on disk and is inconsitent
+
+	return json.Marshal(c)
+}
+
+// StrongID returns image ID for the config JSON.
+func StrongID(configJSON []byte) (digest.Digest, error) {
+	digester := digest.Canonical.New()
+	if _, err := digester.Hash().Write(configJSON); err != nil {
+		return "", err
+	}
+	dgst := digester.Digest()
+	logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
+	return dgst, nil
+}
+
+func rawJSON(value interface{}) *json.RawMessage {
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	return (*json.RawMessage)(&jsonval)
+}

image/image_test.go

@@ -0,0 +1,55 @@
+package image
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+var fixtures = []string{
+	"fixtures/pre1.9",
+	"fixtures/post1.9",
+}
+
+func loadFixtureFile(t *testing.T, path string) []byte {
+	fileData, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Fatalf("error opening %s: %v", path, err)
+	}
+
+	return bytes.TrimSpace(fileData)
+}
+
+// TestMakeImageConfig makes sure that MakeImageConfig returns the expected
+// canonical JSON for a reference Image.
+func TestMakeImageConfig(t *testing.T) {
+	for _, fixture := range fixtures {
+		v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility")
+		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
+		layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id"))
+		parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id"))
+
+		json, err := MakeImageConfig(v1Compatibility, layerID, parentID)
+		if err != nil {
+			t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err)
+		}
+		if !bytes.Equal(json, expectedConfig) {
+			t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json)
+		}
+	}
+}
+
+// TestGetStrongID makes sure that GetConfigJSON returns the expected
+// hash for a reference Image.
+func TestGetStrongID(t *testing.T) {
+	for _, fixture := range fixtures {
+		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
+		expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id"))
+
+		if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID {
+			t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err)
+		}
+	}
+}

integration-cli/docker_cli_pull_test.go

@@ -1,7 +1,11 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
@@ -159,3 +163,254 @@ func (s *DockerHubPullSuite) TestPullClientDisconnect(c *check.C) {
 		time.Sleep(500 * time.Millisecond)
 	}
 }
+
+type idAndParent struct {
+	ID     string
+	Parent string
+}
+
+func inspectImage(c *check.C, imageRef string) idAndParent {
+	out, _ := dockerCmd(c, "inspect", imageRef)
+	var inspectOutput []idAndParent
+	err := json.Unmarshal([]byte(out), &inspectOutput)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	return inspectOutput[0]
+}
+
+func imageID(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).ID
+}
+
+func imageParent(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).Parent
+}
+
+// TestPullMigration verifies that pulling an image based on layers
+// that already exists locally will reuse those existing layers.
+func (s *DockerRegistrySuite) TestPullMigration(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/migration"
+
+	baseImage := repoName + ":base"
+	_, err := buildImage(baseImage, fmt.Sprintf(`
+	    FROM scratch
+	    ENV IMAGE base
+	    CMD echo %s
+	`, baseImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	baseParentBeforePush := imageParent(c, baseImage)
+
+	derivedImage := repoName + ":derived"
+	_, err = buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that the parent of this pulled image is the original base
+	// image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Remove everything, repull, and make sure everything uses computed IDs
+	dockerCmd(c, "rmi", baseImage, derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull3 := imageID(c, derivedImage)
+	derivedParentAfterPull3 := imageParent(c, derivedImage)
+	derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3)
+
+	if derivedIDAfterPull3 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a second repull")
+	}
+
+	if derivedParentAfterPull3 == baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush)
+	}
+
+	if derivedGrandparentAfterPull3 == baseParentBeforePush {
+		c.Fatal("base image's parent ID should have been rewritten on pull")
+	}
+}
+
+// TestPullMigrationRun verifies that pulling an image based on layers
+// that already exists locally will result in an image that runs properly.
+func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) {
+	type idAndParent struct {
+		ID     string
+		Parent string
+	}
+
+	derivedImage := privateRegistryURL + "/dockercli/migration-run"
+	baseImage := "busybox"
+
+	_, err := buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    RUN dd if=/dev/zero of=/file bs=1024 count=1024
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that this pulled image is based on the original base image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Make sure the image runs correctly
+	out, _ := dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Make sure the image still runs
+	out, _ = dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+}
+
+// TestPullConflict provides coverage of the situation where a computed
+// strongID conflicts with some unverifiable data in the graph.
+func (s *DockerRegistrySuite) TestPullConflict(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/conflict"
+
+	_, err := buildImage(repoName, `
+	    FROM scratch
+	    ENV IMAGE conflict
+	    CMD echo conflict
+	`, true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	dockerCmd(c, "push", repoName)
+
+	// Pull to make it content-addressable
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "pull", repoName)
+
+	IDBeforeLoad := imageID(c, repoName)
+
+	// Load/save to turn this into an unverified image with the same ID
+	tmpDir, err := ioutil.TempDir("", "conflict-save-output")
+	if err != nil {
+		c.Errorf("failed to create temporary directory: %s", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tarFile := filepath.Join(tmpDir, "repo.tar")
+
+	dockerCmd(c, "save", "-o", tarFile, repoName)
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "load", "-i", tarFile)
+
+	// Check that the the ID is the same after save/load.
+	IDAfterLoad := imageID(c, repoName)
+
+	if IDAfterLoad != IDBeforeLoad {
+		c.Fatal("image's ID should be the same after save/load")
+	}
+
+	// Repull
+	dockerCmd(c, "pull", repoName)
+
+	// Check that the ID is now different because of the conflict.
+	IDAfterPull1 := imageID(c, repoName)
+
+	// Expect the new ID to be SHA256(oldID)
+	expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad))
+	if err != nil {
+		c.Fatalf("digest error: %v", err)
+	}
+	expectedID := expectedIDDigest.Hex()
+	if IDAfterPull1 != expectedID {
+		c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1)
+	}
+
+	// A second pull should use the new ID again.
+	dockerCmd(c, "pull", repoName)
+
+	IDAfterPull2 := imageID(c, repoName)
+
+	if IDAfterPull2 != IDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repull")
+	}
+}

runconfig/config.go

@@ -12,6 +12,8 @@ import (
 // It should hold only portable information about the container.
 // Here, "portable" means "independent from the host we are running on".
 // Non-portable information *should* appear in HostConfig.
+// All fields added to this struct must be marked `omitempty` to keep getting
+// predictable hashes from the old `v1Compatibility` configuration.
 type Config struct {
 	Hostname        string                // Hostname
 	Domainname      string                // Domainname
@@ -19,7 +21,8 @@ type Config struct {
 	AttachStdin     bool                  // Attach the standard input, makes possible user interaction
 	AttachStdout    bool                  // Attach the standard output
 	AttachStderr    bool                  // Attach the standard error
-	ExposedPorts    map[nat.Port]struct{} // List of exposed ports
+	ExposedPorts    map[nat.Port]struct{} `json:",omitempty"` // List of exposed ports
+	PublishService  string                `json:",omitempty"` // Name of the network service exposed by the container
 	Tty             bool                  // Attach standard streams to a tty, including stdin if it is not closed.
 	OpenStdin       bool                  // Open stdin
 	StdinOnce       bool                  // If true, close stdin after the 1 attached client disconnects.
@@ -29,11 +32,11 @@ type Config struct {
 	Volumes         map[string]struct{}   // List of volumes (mounts) used for the container
 	WorkingDir      string                // Current directory (PWD) in the command will be launched
 	Entrypoint      *stringutils.StrSlice // Entrypoint to run when starting the container
-	NetworkDisabled bool                  // Is network disabled
+	NetworkDisabled bool                  `json:",omitempty"` // Is network disabled
-	MacAddress      string                // Mac Address of the container
+	MacAddress      string                `json:",omitempty"` // Mac Address of the container
 	OnBuild         []string              // ONBUILD metadata that were defined on the image Dockerfile
 	Labels          map[string]string     // List of labels set to this container
-	StopSignal      string                // Signal to stop a container
+	StopSignal      string                `json:",omitempty"` // Signal to stop a container
 }
 
 // DecodeContainerConfig decodes a json encoded config into a ContainerConfigWrapper