mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
Changes cross-repository blob mounting to a blob Create option
Also renames BlobSumService to V2MetadataService, BlobSum to V2Metadata Signed-off-by: Brian Bland <brian.bland@docker.com>
This commit is contained in:
Brian Bland
parent
7289c7218e
commit
6309947718
15 changed files with 359 additions and 305 deletions
|
|
@ -1,137 +0,0 @@
|
|||
package metadata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
// BlobSumService maps layer IDs to a set of known blobsums for
|
||||
// the layer.
|
||||
type BlobSumService struct {
|
||||
store Store
|
||||
}
|
||||
|
||||
// BlobSum contains the digest and source repository information for a layer.
|
||||
type BlobSum struct {
|
||||
Digest digest.Digest
|
||||
SourceRepository string
|
||||
}
|
||||
|
||||
// maxBlobSums is the number of blobsums to keep per layer DiffID.
|
||||
const maxBlobSums = 50
|
||||
|
||||
// NewBlobSumService creates a new blobsum mapping service.
|
||||
func NewBlobSumService(store Store) *BlobSumService {
|
||||
return &BlobSumService{
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
func (blobserv *BlobSumService) diffIDNamespace() string {
|
||||
return "blobsum-storage"
|
||||
}
|
||||
|
||||
func (blobserv *BlobSumService) blobSumNamespace() string {
|
||||
return "blobsum-lookup"
|
||||
}
|
||||
|
||||
func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
|
||||
return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
|
||||
}
|
||||
|
||||
func (blobserv *BlobSumService) blobSumKey(blobsum BlobSum) string {
|
||||
return string(blobsum.Digest.Algorithm()) + "/" + blobsum.Digest.Hex()
|
||||
}
|
||||
|
||||
// GetBlobSums finds the blobsums associated with a layer DiffID.
|
||||
func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]BlobSum, error) {
|
||||
jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var blobsums []BlobSum
|
||||
if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blobsums, nil
|
||||
}
|
||||
|
||||
// GetDiffID finds a layer DiffID from a blobsum hash.
|
||||
func (blobserv *BlobSumService) GetDiffID(blobsum BlobSum) (layer.DiffID, error) {
|
||||
diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum))
|
||||
if err != nil {
|
||||
return layer.DiffID(""), err
|
||||
}
|
||||
|
||||
return layer.DiffID(diffIDBytes), nil
|
||||
}
|
||||
|
||||
// Add associates a blobsum with a layer DiffID. If too many blobsums are
|
||||
// present, the oldest one is dropped.
|
||||
func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum BlobSum) error {
|
||||
oldBlobSums, err := blobserv.GetBlobSums(diffID)
|
||||
if err != nil {
|
||||
oldBlobSums = nil
|
||||
}
|
||||
newBlobSums := make([]BlobSum, 0, len(oldBlobSums)+1)
|
||||
|
||||
// Copy all other blobsums to new slice
|
||||
for _, oldSum := range oldBlobSums {
|
||||
if oldSum != blobsum {
|
||||
newBlobSums = append(newBlobSums, oldSum)
|
||||
}
|
||||
}
|
||||
|
||||
newBlobSums = append(newBlobSums, blobsum)
|
||||
|
||||
if len(newBlobSums) > maxBlobSums {
|
||||
newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:]
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newBlobSums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID))
|
||||
}
|
||||
|
||||
// Remove unassociates a blobsum from a layer DiffID.
|
||||
func (blobserv *BlobSumService) Remove(blobsum BlobSum) error {
|
||||
diffID, err := blobserv.GetDiffID(blobsum)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldBlobSums, err := blobserv.GetBlobSums(diffID)
|
||||
if err != nil {
|
||||
oldBlobSums = nil
|
||||
}
|
||||
newBlobSums := make([]BlobSum, 0, len(oldBlobSums))
|
||||
|
||||
// Copy all other blobsums to new slice
|
||||
for _, oldSum := range oldBlobSums {
|
||||
if oldSum != blobsum {
|
||||
newBlobSums = append(newBlobSums, oldSum)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newBlobSums) == 0 {
|
||||
return blobserv.store.Delete(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID))
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newBlobSums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes)
|
||||
}
|
||||
137
distribution/metadata/v2_metadata_service.go
Normal file
137
distribution/metadata/v2_metadata_service.go
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
package metadata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
// V2MetadataService maps layer IDs to a set of known metadata for
|
||||
// the layer.
|
||||
type V2MetadataService struct {
|
||||
store Store
|
||||
}
|
||||
|
||||
// V2Metadata contains the digest and source repository information for a layer.
|
||||
type V2Metadata struct {
|
||||
Digest digest.Digest
|
||||
SourceRepository string
|
||||
}
|
||||
|
||||
// maxMetadata is the number of metadata entries to keep per layer DiffID.
|
||||
const maxMetadata = 50
|
||||
|
||||
// NewV2MetadataService creates a new diff ID to v2 metadata mapping service.
|
||||
func NewV2MetadataService(store Store) *V2MetadataService {
|
||||
return &V2MetadataService{
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) diffIDNamespace() string {
|
||||
return "v2metadata-by-diffid"
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) digestNamespace() string {
|
||||
return "diffid-by-digest"
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) diffIDKey(diffID layer.DiffID) string {
|
||||
return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
|
||||
}
|
||||
|
||||
func (serv *V2MetadataService) digestKey(dgst digest.Digest) string {
|
||||
return string(dgst.Algorithm()) + "/" + dgst.Hex()
|
||||
}
|
||||
|
||||
// GetMetadata finds the metadata associated with a layer DiffID.
|
||||
func (serv *V2MetadataService) GetMetadata(diffID layer.DiffID) ([]V2Metadata, error) {
|
||||
jsonBytes, err := serv.store.Get(serv.diffIDNamespace(), serv.diffIDKey(diffID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var metadata []V2Metadata
|
||||
if err := json.Unmarshal(jsonBytes, &metadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// GetDiffID finds a layer DiffID from a digest.
|
||||
func (serv *V2MetadataService) GetDiffID(dgst digest.Digest) (layer.DiffID, error) {
|
||||
diffIDBytes, err := serv.store.Get(serv.digestNamespace(), serv.digestKey(dgst))
|
||||
if err != nil {
|
||||
return layer.DiffID(""), err
|
||||
}
|
||||
|
||||
return layer.DiffID(diffIDBytes), nil
|
||||
}
|
||||
|
||||
// Add associates metadata with a layer DiffID. If too many metadata entries are
|
||||
// present, the oldest one is dropped.
|
||||
func (serv *V2MetadataService) Add(diffID layer.DiffID, metadata V2Metadata) error {
|
||||
oldMetadata, err := serv.GetMetadata(diffID)
|
||||
if err != nil {
|
||||
oldMetadata = nil
|
||||
}
|
||||
newMetadata := make([]V2Metadata, 0, len(oldMetadata)+1)
|
||||
|
||||
// Copy all other metadata to new slice
|
||||
for _, oldMeta := range oldMetadata {
|
||||
if oldMeta != metadata {
|
||||
newMetadata = append(newMetadata, oldMeta)
|
||||
}
|
||||
}
|
||||
|
||||
newMetadata = append(newMetadata, metadata)
|
||||
|
||||
if len(newMetadata) > maxMetadata {
|
||||
newMetadata = newMetadata[len(newMetadata)-maxMetadata:]
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return serv.store.Set(serv.digestNamespace(), serv.digestKey(metadata.Digest), []byte(diffID))
|
||||
}
|
||||
|
||||
// Remove unassociates a metadata entry from a layer DiffID.
|
||||
func (serv *V2MetadataService) Remove(metadata V2Metadata) error {
|
||||
diffID, err := serv.GetDiffID(metadata.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldMetadata, err := serv.GetMetadata(diffID)
|
||||
if err != nil {
|
||||
oldMetadata = nil
|
||||
}
|
||||
newMetadata := make([]V2Metadata, 0, len(oldMetadata))
|
||||
|
||||
// Copy all other metadata to new slice
|
||||
for _, oldMeta := range oldMetadata {
|
||||
if oldMeta != metadata {
|
||||
newMetadata = append(newMetadata, oldMeta)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newMetadata) == 0 {
|
||||
return serv.store.Delete(serv.diffIDNamespace(), serv.diffIDKey(diffID))
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes)
|
||||
}
|
||||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
func TestBlobSumService(t *testing.T) {
|
||||
func TestV2MetadataService(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
|
||||
if err != nil {
|
||||
t.Fatalf("could not create temp dir: %v", err)
|
||||
|
|
@ -23,41 +23,41 @@ func TestBlobSumService(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("could not create metadata store: %v", err)
|
||||
}
|
||||
blobSumService := NewBlobSumService(metadataStore)
|
||||
V2MetadataService := NewV2MetadataService(metadataStore)
|
||||
|
||||
tooManyBlobSums := make([]BlobSum, 100)
|
||||
tooManyBlobSums := make([]V2Metadata, 100)
|
||||
for i := range tooManyBlobSums {
|
||||
randDigest := randomDigest()
|
||||
tooManyBlobSums[i] = BlobSum{Digest: randDigest}
|
||||
tooManyBlobSums[i] = V2Metadata{Digest: randDigest}
|
||||
}
|
||||
|
||||
testVectors := []struct {
|
||||
diffID layer.DiffID
|
||||
blobsums []BlobSum
|
||||
metadata []V2Metadata
|
||||
}{
|
||||
{
|
||||
diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
||||
blobsums: []BlobSum{
|
||||
metadata: []V2Metadata{
|
||||
{Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
||||
},
|
||||
},
|
||||
{
|
||||
diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
||||
blobsums: []BlobSum{
|
||||
metadata: []V2Metadata{
|
||||
{Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")},
|
||||
{Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")},
|
||||
},
|
||||
},
|
||||
{
|
||||
diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
|
||||
blobsums: tooManyBlobSums,
|
||||
metadata: tooManyBlobSums,
|
||||
},
|
||||
}
|
||||
|
||||
// Set some associations
|
||||
for _, vec := range testVectors {
|
||||
for _, blobsum := range vec.blobsums {
|
||||
err := blobSumService.Add(vec.diffID, blobsum)
|
||||
for _, blobsum := range vec.metadata {
|
||||
err := V2MetadataService.Add(vec.diffID, blobsum)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Set: %v", err)
|
||||
}
|
||||
|
|
@ -66,37 +66,37 @@ func TestBlobSumService(t *testing.T) {
|
|||
|
||||
// Check the correct values are read back
|
||||
for _, vec := range testVectors {
|
||||
blobsums, err := blobSumService.GetBlobSums(vec.diffID)
|
||||
metadata, err := V2MetadataService.GetMetadata(vec.diffID)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Get: %v", err)
|
||||
}
|
||||
expectedBlobsums := len(vec.blobsums)
|
||||
if expectedBlobsums > 50 {
|
||||
expectedBlobsums = 50
|
||||
expectedMetadataEntries := len(vec.metadata)
|
||||
if expectedMetadataEntries > 50 {
|
||||
expectedMetadataEntries = 50
|
||||
}
|
||||
if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
|
||||
if !reflect.DeepEqual(metadata, vec.metadata[len(vec.metadata)-expectedMetadataEntries:len(vec.metadata)]) {
|
||||
t.Fatal("Get returned incorrect layer ID")
|
||||
}
|
||||
}
|
||||
|
||||
// Test GetBlobSums on a nonexistent entry
|
||||
_, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
||||
// Test GetMetadata on a nonexistent entry
|
||||
_, err = V2MetadataService.GetMetadata(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error looking up nonexistent entry")
|
||||
}
|
||||
|
||||
// Test GetDiffID on a nonexistent entry
|
||||
_, err = blobSumService.GetDiffID(BlobSum{Digest: digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")})
|
||||
_, err = V2MetadataService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error looking up nonexistent entry")
|
||||
}
|
||||
|
||||
// Overwrite one of the entries and read it back
|
||||
err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0])
|
||||
err = V2MetadataService.Add(testVectors[1].diffID, testVectors[0].metadata[0])
|
||||
if err != nil {
|
||||
t.Fatalf("error calling Add: %v", err)
|
||||
}
|
||||
diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0])
|
||||
diffID, err := V2MetadataService.GetDiffID(testVectors[0].metadata[0].Digest)
|
||||
if err != nil {
|
||||
t.Fatalf("error calling GetDiffID: %v", err)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue