Add distribution package

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

distribution/metadata/blobsum_service.go

@@ -0,0 +1,100 @@
+package metadata
+
+import (
+	"encoding/json"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+// BlobSumService maps layer IDs to a set of known blobsums for
+// the layer.
+type BlobSumService struct {
+	store Store
+}
+
+// maxBlobSums is the number of blobsums to keep per layer DiffID.
+const maxBlobSums = 5
+
+// NewBlobSumService creates a new blobsum mapping service.
+func NewBlobSumService(store Store) *BlobSumService {
+	return &BlobSumService{
+		store: store,
+	}
+}
+
+func (blobserv *BlobSumService) diffIDNamespace() string {
+	return "blobsum-storage"
+}
+
+func (blobserv *BlobSumService) blobSumNamespace() string {
+	return "blobsum-lookup"
+}
+
+func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
+	return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
+}
+
+func (blobserv *BlobSumService) blobSumKey(blobsum digest.Digest) string {
+	return string(blobsum.Algorithm()) + "/" + blobsum.Hex()
+}
+
+// GetBlobSums finds the blobsums associated with a layer DiffID.
+func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Digest, error) {
+	jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID))
+	if err != nil {
+		return nil, err
+	}
+
+	var blobsums []digest.Digest
+	if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
+		return nil, err
+	}
+
+	return blobsums, nil
+}
+
+// GetDiffID finds a layer DiffID from a blobsum hash.
+func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, error) {
+	diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum))
+	if err != nil {
+		return layer.DiffID(""), err
+	}
+
+	return layer.DiffID(diffIDBytes), nil
+}
+
+// Add associates a blobsum with a layer DiffID. If too many blobsums are
+// present, the oldest one is dropped.
+func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) error {
+	oldBlobSums, err := blobserv.GetBlobSums(diffID)
+	if err != nil {
+		oldBlobSums = nil
+	}
+	newBlobSums := make([]digest.Digest, 0, len(oldBlobSums)+1)
+
+	// Copy all other blobsums to new slice
+	for _, oldSum := range oldBlobSums {
+		if oldSum != blobsum {
+			newBlobSums = append(newBlobSums, oldSum)
+		}
+	}
+
+	newBlobSums = append(newBlobSums, blobsum)
+
+	if len(newBlobSums) > maxBlobSums {
+		newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:]
+	}
+
+	jsonBytes, err := json.Marshal(newBlobSums)
+	if err != nil {
+		return err
+	}
+
+	err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes)
+	if err != nil {
+		return err
+	}
+
+	return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID))
+}

distribution/metadata/blobsum_service_test.go

@@ -0,0 +1,105 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+func TestBlobSumService(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
+	if err != nil {
+		t.Fatalf("could not create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	metadataStore, err := NewFSMetadataStore(tmpDir)
+	if err != nil {
+		t.Fatalf("could not create metadata store: %v", err)
+	}
+	blobSumService := NewBlobSumService(metadataStore)
+
+	testVectors := []struct {
+		diffID   layer.DiffID
+		blobsums []digest.Digest
+	}{
+		{
+			diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+			},
+		},
+		{
+			diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+				digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
+			},
+		},
+		{
+			diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+				digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
+				digest.Digest("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"),
+				digest.Digest("sha256:8902a7ca89aabbb868835260912159026637634090dd8899eee969523252236e"),
+				digest.Digest("sha256:c84364306344ccc48532c52ff5209236273525231dddaaab53262322352883aa"),
+				digest.Digest("sha256:aa7583bbc87532a8352bbb72520a821b3623523523a8352523a52352aaa888fe"),
+			},
+		},
+	}
+
+	// Set some associations
+	for _, vec := range testVectors {
+		for _, blobsum := range vec.blobsums {
+			err := blobSumService.Add(vec.diffID, blobsum)
+			if err != nil {
+				t.Fatalf("error calling Set: %v", err)
+			}
+		}
+	}
+
+	// Check the correct values are read back
+	for _, vec := range testVectors {
+		blobsums, err := blobSumService.GetBlobSums(vec.diffID)
+		if err != nil {
+			t.Fatalf("error calling Get: %v", err)
+		}
+		expectedBlobsums := len(vec.blobsums)
+		if expectedBlobsums > 5 {
+			expectedBlobsums = 5
+		}
+		if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
+			t.Fatal("Get returned incorrect layer ID")
+		}
+	}
+
+	// Test GetBlobSums on a nonexistent entry
+	_, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Test GetDiffID on a nonexistent entry
+	_, err = blobSumService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Overwrite one of the entries and read it back
+	err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0])
+	if err != nil {
+		t.Fatalf("error calling Add: %v", err)
+	}
+	diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0])
+	if err != nil {
+		t.Fatalf("error calling GetDiffID: %v", err)
+	}
+	if diffID != testVectors[1].diffID {
+		t.Fatal("GetDiffID returned incorrect diffID")
+	}
+}

distribution/metadata/metadata.go

@@ -0,0 +1,65 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// Store implements a K/V store for mapping distribution-related IDs
+// to on-disk layer IDs and image IDs. The namespace identifies the type of
+// mapping (i.e. "v1ids" or "artifacts"). MetadataStore is goroutine-safe.
+type Store interface {
+	// Get retrieves data by namespace and key.
+	Get(namespace string, key string) ([]byte, error)
+	// Set writes data indexed by namespace and key.
+	Set(namespace, key string, value []byte) error
+}
+
+// FSMetadataStore uses the filesystem to associate metadata with layer and
+// image IDs.
+type FSMetadataStore struct {
+	sync.RWMutex
+	basePath string
+}
+
+// NewFSMetadataStore creates a new filesystem-based metadata store.
+func NewFSMetadataStore(basePath string) (*FSMetadataStore, error) {
+	if err := os.MkdirAll(basePath, 0700); err != nil {
+		return nil, err
+	}
+	return &FSMetadataStore{
+		basePath: basePath,
+	}, nil
+}
+
+func (store *FSMetadataStore) path(namespace, key string) string {
+	return filepath.Join(store.basePath, namespace, key)
+}
+
+// Get retrieves data by namespace and key. The data is read from a file named
+// after the key, stored in the namespace's directory.
+func (store *FSMetadataStore) Get(namespace string, key string) ([]byte, error) {
+	store.RLock()
+	defer store.RUnlock()
+
+	return ioutil.ReadFile(store.path(namespace, key))
+}
+
+// Set writes data indexed by namespace and key. The data is written to a file
+// named after the key, stored in the namespace's directory.
+func (store *FSMetadataStore) Set(namespace, key string, value []byte) error {
+	store.Lock()
+	defer store.Unlock()
+
+	path := store.path(namespace, key)
+	tempFilePath := path + ".tmp"
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(tempFilePath, value, 0644); err != nil {
+		return err
+	}
+	return os.Rename(tempFilePath, path)
+}

distribution/metadata/v1_id_service.go

@@ -0,0 +1,44 @@
+package metadata
+
+import (
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+)
+
+// V1IDService maps v1 IDs to layers on disk.
+type V1IDService struct {
+	store Store
+}
+
+// NewV1IDService creates a new V1 ID mapping service.
+func NewV1IDService(store Store) *V1IDService {
+	return &V1IDService{
+		store: store,
+	}
+}
+
+// namespace returns the namespace used by this service.
+func (idserv *V1IDService) namespace() string {
+	return "v1id"
+}
+
+// Get finds a layer by its V1 ID.
+func (idserv *V1IDService) Get(v1ID, registry string) (layer.ChainID, error) {
+	if err := v1.ValidateID(v1ID); err != nil {
+		return layer.ChainID(""), err
+	}
+
+	idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
+	if err != nil {
+		return layer.ChainID(""), err
+	}
+	return layer.ChainID(idBytes), nil
+}
+
+// Set associates an image with a V1 ID.
+func (idserv *V1IDService) Set(v1ID, registry string, id layer.ChainID) error {
+	if err := v1.ValidateID(v1ID); err != nil {
+		return err
+	}
+	return idserv.store.Set(idserv.namespace(), registry+","+v1ID, []byte(id))
+}

distribution/metadata/v1_id_service_test.go

@@ -0,0 +1,83 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/layer"
+)
+
+func TestV1IDService(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "v1-id-service-test")
+	if err != nil {
+		t.Fatalf("could not create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	metadataStore, err := NewFSMetadataStore(tmpDir)
+	if err != nil {
+		t.Fatalf("could not create metadata store: %v", err)
+	}
+	v1IDService := NewV1IDService(metadataStore)
+
+	testVectors := []struct {
+		registry string
+		v1ID     string
+		layerID  layer.ChainID
+	}{
+		{
+			registry: "registry1",
+			v1ID:     "f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937",
+			layerID:  layer.ChainID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+		},
+		{
+			registry: "registry2",
+			v1ID:     "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
+			layerID:  layer.ChainID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
+		},
+		{
+			registry: "registry1",
+			v1ID:     "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
+			layerID:  layer.ChainID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
+		},
+	}
+
+	// Set some associations
+	for _, vec := range testVectors {
+		err := v1IDService.Set(vec.v1ID, vec.registry, vec.layerID)
+		if err != nil {
+			t.Fatalf("error calling Set: %v", err)
+		}
+	}
+
+	// Check the correct values are read back
+	for _, vec := range testVectors {
+		layerID, err := v1IDService.Get(vec.v1ID, vec.registry)
+		if err != nil {
+			t.Fatalf("error calling Get: %v", err)
+		}
+		if layerID != vec.layerID {
+			t.Fatal("Get returned incorrect layer ID")
+		}
+	}
+
+	// Test Get on a nonexistent entry
+	_, err = v1IDService.Get("82379823067823853223359023576437723560923756b03560378f4497753917", "registry1")
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Overwrite one of the entries and read it back
+	err = v1IDService.Set(testVectors[0].v1ID, testVectors[0].registry, testVectors[1].layerID)
+	if err != nil {
+		t.Fatalf("error calling Set: %v", err)
+	}
+	layerID, err := v1IDService.Get(testVectors[0].v1ID, testVectors[0].registry)
+	if err != nil {
+		t.Fatalf("error calling Get: %v", err)
+	}
+	if layerID != testVectors[1].layerID {
+		t.Fatal("Get returned incorrect layer ID")
+	}
+}