From e9f6f1a930b74c6b06f257eac838d062bbdd0d16 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 29 Jan 2015 21:28:20 -0800 Subject: [PATCH 1/2] Store tar checksum in separate file Fixes #10432 Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/manifest.go | 10 +++++- graph/manifest_test.go | 82 ++++++++++++++++++++++++++++++++++++++++++ image/image.go | 18 ++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 graph/manifest_test.go diff --git a/graph/manifest.go b/graph/manifest.go index 6bebb7e5e8..68c3dd91f7 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -92,7 +92,10 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } } - checksum := layer.Checksum + checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) + if err != nil { + return nil, fmt.Errorf("Error getting image checksum: %s", err) + } if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() { archive, err := layer.TarLayer() if err != nil { @@ -108,6 +111,11 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } checksum = tarSum.Sum(nil) + + // Save checksum value + if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), checksum); err != nil { + return nil, err + } } jsonData, err := layer.RawJson() diff --git a/graph/manifest_test.go b/graph/manifest_test.go new file mode 100644 index 0000000000..6084ace776 --- /dev/null +++ b/graph/manifest_test.go @@ -0,0 +1,82 @@ +package graph + +import ( + "encoding/json" + "os" + "testing" + + "github.com/docker/docker/image" + "github.com/docker/docker/registry" + "github.com/docker/docker/utils" +) + +const ( + testManifestImageName = "testapp" + testManifestImageID = "d821b739e8834ec89ac4469266c3d11515da88fdcbcbdddcbcddb636f54fdde9" + testManifestImageIDShort = "d821b739e883" + testManifestTag = "manifesttest" +) + +func TestManifestTarsumCache(t *testing.T) { + tmp, err := utils.TestDirectory("") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + store := mkTestTagStore(tmp, t) + defer store.graph.driver.Cleanup() + + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + img := &image.Image{ID: testManifestImageID} + if err := store.graph.Register(img, archive); err != nil { + t.Fatal(err) + } + if err := store.Set(testManifestImageName, testManifestTag, testManifestImageID, false); err != nil { + t.Fatal(err) + } + + if cs, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)); err != nil { + t.Fatal(err) + } else if cs != "" { + t.Fatalf("Non-empty checksum file after register") + } + + // Generate manifest + payload, err := store.newManifest(testManifestImageName, testManifestImageName, testManifestTag) + if err != nil { + t.Fatal(err) + } + + manifestChecksum, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)) + if err != nil { + t.Fatal(err) + } + + var manifest registry.ManifestData + if err := json.Unmarshal(payload, &manifest); err != nil { + t.Fatalf("error unmarshalling manifest: %s", err) + } + + if len(manifest.FSLayers) != 1 { + t.Fatalf("Unexpected number of layers, expecting 1: %d", len(manifest.FSLayers)) + } + + if manifest.FSLayers[0].BlobSum != manifestChecksum { + t.Fatalf("Unexpected blob sum, expecting %q, got %q", manifestChecksum, manifest.FSLayers[0].BlobSum) + } + + if len(manifest.History) != 1 { + t.Fatalf("Unexpected number of layer history, expecting 1: %d", len(manifest.History)) + } + + v1compat, err := img.RawJson() + if err != nil { + t.Fatal(err) + } + if manifest.History[0].V1Compatibility != string(v1compat) { + t.Fatalf("Unexpected json value\nExpected:\n%s\nActual:\n%s", v1compat, manifest.History[0].V1Compatibility) + } +} diff --git a/image/image.go b/image/image.go index 2187653b03..01300069f4 100644 --- a/image/image.go +++ b/image/image.go @@ -114,6 +114,24 @@ func (img *Image) SaveSize(root string) error { return nil } +func (img *Image) SaveCheckSum(root, checksum string) error { + if err := ioutil.WriteFile(path.Join(root, "checksum"), []byte(checksum), 0600); err != nil { + return fmt.Errorf("Error storing checksum in %s/checksum: %s", root, err) + } + return nil +} + +func (img *Image) GetCheckSum(root string) (string, error) { + cs, err := ioutil.ReadFile(path.Join(root, "checksum")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + return string(cs), err +} + func jsonPath(root string) string { return path.Join(root, "json") } From 3414307306dc1780f80c5ca5e9dd7a8822e27eec Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 30 Jan 2015 09:37:50 -0800 Subject: [PATCH 2/2] Remove Checksum field from image.Image struct The checksum is now being stored in a separate file beside the image JSON file. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- graph/service.go | 1 - image/image.go | 1 - 2 files changed, 2 deletions(-) diff --git a/graph/service.go b/graph/service.go index 675e12a1a9..354c57eaff 100644 --- a/graph/service.go +++ b/graph/service.go @@ -151,7 +151,6 @@ func (s *TagStore) CmdLookup(job *engine.Job) engine.Status { out.Set("Os", image.OS) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) - out.Set("Checksum", image.Checksum) if _, err = out.WriteTo(job.Stdout); err != nil { return job.Error(err) } diff --git a/image/image.go b/image/image.go index 01300069f4..27554d972a 100644 --- a/image/image.go +++ b/image/image.go @@ -31,7 +31,6 @@ type Image struct { Config *runconfig.Config `json:"config,omitempty"` Architecture string `json:"architecture,omitempty"` OS string `json:"os,omitempty"` - Checksum string `json:"checksum"` Size int64 graph Graph