Add tag store

The tag store associates tags and digests with image IDs. This
functionality used to be part of graph package. This commit splits it
off into a self-contained package with a simple interface.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

tag/store.go

@@ -0,0 +1,282 @@
+package tag
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/image"
+)
+
+// DefaultTag defines the default tag used when performing images related actions and no tag string is specified
+const DefaultTag = "latest"
+
+var (
+	// ErrDoesNotExist is returned if a reference is not found in the
+	// store.
+	ErrDoesNotExist = errors.New("reference does not exist")
+)
+
+// An Association is a tuple associating a reference with an image ID.
+type Association struct {
+	Ref     reference.Named
+	ImageID image.ID
+}
+
+// Store provides the set of methods which can operate on a tag store.
+type Store interface {
+	References(id image.ID) []reference.Named
+	ReferencesByName(ref reference.Named) []Association
+	Add(ref reference.Named, id image.ID, force bool) error
+	Delete(ref reference.Named) (bool, error)
+	Get(ref reference.Named) (image.ID, error)
+}
+
+type store struct {
+	mu sync.RWMutex
+	// jsonPath is the path to the file where the serialized tag data is
+	// stored.
+	jsonPath string
+	// Repositories is a map of repositories, indexed by name.
+	Repositories map[string]repository
+	// referencesByIDCache is a cache of references indexed by ID, to speed
+	// up References.
+	referencesByIDCache map[image.ID]map[string]reference.Named
+}
+
+// Repository maps tags to image IDs. The key is a a stringified Reference,
+// including the repository name.
+type repository map[string]image.ID
+
+func defaultTagIfNameOnly(ref reference.Named) reference.Named {
+	switch ref.(type) {
+	case reference.Tagged:
+		return ref
+	case reference.Digested:
+		return ref
+	default:
+		// Should never fail
+		ref, _ = reference.WithTag(ref, DefaultTag)
+		return ref
+	}
+}
+
+// NewTagStore creates a new tag store, tied to a file path where the set of
+// tags is serialized in JSON format.
+func NewTagStore(jsonPath string) (Store, error) {
+	abspath, err := filepath.Abs(jsonPath)
+	if err != nil {
+		return nil, err
+	}
+
+	store := &store{
+		jsonPath:            abspath,
+		Repositories:        make(map[string]repository),
+		referencesByIDCache: make(map[image.ID]map[string]reference.Named),
+	}
+	// Load the json file if it exists, otherwise create it.
+	if err := store.reload(); os.IsNotExist(err) {
+		if err := store.save(); err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+	return store, nil
+}
+
+// Add adds a tag or digest to the store. If force is set to true, existing
+// references can be overwritten. This only works for tags, not digests.
+func (store *store) Add(ref reference.Named, id image.ID, force bool) error {
+	ref = defaultTagIfNameOnly(ref)
+
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	repository, exists := store.Repositories[ref.Name()]
+	if !exists || repository == nil {
+		repository = make(map[string]image.ID)
+		store.Repositories[ref.Name()] = repository
+	}
+
+	refStr := ref.String()
+	oldID, exists := repository[refStr]
+
+	if exists {
+		// force only works for tags
+		if digested, isDigest := ref.(reference.Digested); isDigest {
+			return fmt.Errorf("Cannot overwrite digest %s", digested.Digest().String())
+		}
+
+		if !force {
+			return fmt.Errorf("Conflict: Tag %s is already set to image %s, if you want to replace it, please use -f option", ref.String(), oldID.String())
+		}
+
+		if store.referencesByIDCache[oldID] != nil {
+			delete(store.referencesByIDCache[oldID], refStr)
+			if len(store.referencesByIDCache[oldID]) == 0 {
+				delete(store.referencesByIDCache, oldID)
+			}
+		}
+	}
+
+	repository[refStr] = id
+	if store.referencesByIDCache[id] == nil {
+		store.referencesByIDCache[id] = make(map[string]reference.Named)
+	}
+	store.referencesByIDCache[id][refStr] = ref
+
+	return store.save()
+}
+
+// Delete deletes a reference from the store. It returns true if a deletion
+// happened, or false otherwise.
+func (store *store) Delete(ref reference.Named) (bool, error) {
+	ref = defaultTagIfNameOnly(ref)
+
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	repoName := ref.Name()
+
+	repository, exists := store.Repositories[repoName]
+	if !exists {
+		return false, ErrDoesNotExist
+	}
+
+	refStr := ref.String()
+	if id, exists := repository[refStr]; exists {
+		delete(repository, refStr)
+		if len(repository) == 0 {
+			delete(store.Repositories, repoName)
+		}
+		if store.referencesByIDCache[id] != nil {
+			delete(store.referencesByIDCache[id], refStr)
+			if len(store.referencesByIDCache[id]) == 0 {
+				delete(store.referencesByIDCache, id)
+			}
+		}
+		return true, store.save()
+	}
+
+	return false, ErrDoesNotExist
+}
+
+// Get retrieves an item from the store by reference.
+func (store *store) Get(ref reference.Named) (image.ID, error) {
+	ref = defaultTagIfNameOnly(ref)
+
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	repository, exists := store.Repositories[ref.Name()]
+	if !exists || repository == nil {
+		return "", ErrDoesNotExist
+	}
+
+	id, exists := repository[ref.String()]
+	if !exists {
+		return "", ErrDoesNotExist
+	}
+
+	return id, nil
+}
+
+// References returns a slice of references to the given image ID. The slice
+// will be nil if there are no references to this image ID.
+func (store *store) References(id image.ID) []reference.Named {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	// Convert the internal map to an array for two reasons:
+	// 1) We must not return a mutable reference.
+	// 2) It would be ugly to expose the extraneous map keys to callers.
+
+	var references []reference.Named
+	for _, ref := range store.referencesByIDCache[id] {
+		references = append(references, ref)
+	}
+
+	return references
+}
+
+// ReferencesByName returns the references for a given repository name.
+// If there are no references known for this repository name,
+// ReferencesByName returns nil.
+func (store *store) ReferencesByName(ref reference.Named) []Association {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	repository, exists := store.Repositories[ref.Name()]
+	if !exists {
+		return nil
+	}
+
+	var associations []Association
+	for refStr, refID := range repository {
+		ref, err := reference.ParseNamed(refStr)
+		if err != nil {
+			// Should never happen
+			return nil
+		}
+		associations = append(associations,
+			Association{
+				Ref:     ref,
+				ImageID: refID,
+			})
+	}
+
+	return associations
+}
+
+func (store *store) save() error {
+	// Store the json
+	jsonData, err := json.Marshal(store)
+	if err != nil {
+		return err
+	}
+
+	tempFilePath := store.jsonPath + ".tmp"
+
+	if err := ioutil.WriteFile(tempFilePath, jsonData, 0600); err != nil {
+		return err
+	}
+
+	if err := os.Rename(tempFilePath, store.jsonPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *store) reload() error {
+	f, err := os.Open(store.jsonPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&store); err != nil {
+		return err
+	}
+
+	for _, repository := range store.Repositories {
+		for refStr, refID := range repository {
+			ref, err := reference.ParseNamed(refStr)
+			if err != nil {
+				// Should never happen
+				continue
+			}
+			if store.referencesByIDCache[refID] == nil {
+				store.referencesByIDCache[refID] = make(map[string]reference.Named)
+			}
+			store.referencesByIDCache[refID][refStr] = ref
+		}
+	}
+
+	return nil
+}

tag/store_test.go

@@ -0,0 +1,328 @@
+package tag
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/image"
+)
+
+var (
+	saveLoadTestCases = map[string]image.ID{
+		"registry:5000/foobar:HEAD":                                                        "sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6",
+		"registry:5000/foobar:alternate":                                                   "sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793",
+		"registry:5000/foobar:latest":                                                      "sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b",
+		"registry:5000/foobar:master":                                                      "sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc",
+		"jess/hollywood:latest":                                                            "sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe",
+		"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6": "sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c",
+		"busybox:latest": "sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c",
+	}
+
+	marshalledSaveLoadTestCases = []byte(`{"Repositories":{"busybox":{"busybox:latest":"sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c"},"jess/hollywood":{"jess/hollywood:latest":"sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe"},"registry":{"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6":"sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c"},"registry:5000/foobar":{"registry:5000/foobar:HEAD":"sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6","registry:5000/foobar:alternate":"sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793","registry:5000/foobar:latest":"sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b","registry:5000/foobar:master":"sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc"}}}`)
+)
+
+func TestLoad(t *testing.T) {
+	jsonFile, err := ioutil.TempFile("", "tag-store-test")
+	if err != nil {
+		t.Fatalf("error creating temp file: %v", err)
+	}
+	defer os.RemoveAll(jsonFile.Name())
+
+	// Write canned json to the temp file
+	_, err = jsonFile.Write(marshalledSaveLoadTestCases)
+	if err != nil {
+		t.Fatalf("error writing to temp file: %v", err)
+	}
+	jsonFile.Close()
+
+	store, err := NewTagStore(jsonFile.Name())
+	if err != nil {
+		t.Fatalf("error creating tag store: %v", err)
+	}
+
+	for refStr, expectedID := range saveLoadTestCases {
+		ref, err := reference.ParseNamed(refStr)
+		if err != nil {
+			t.Fatalf("failed to parse reference: %v", err)
+		}
+		id, err := store.Get(ref)
+		if err != nil {
+			t.Fatalf("could not find reference %s: %v", refStr, err)
+		}
+		if id != expectedID {
+			t.Fatalf("expected %s - got %s", expectedID, id)
+		}
+	}
+}
+
+func TestSave(t *testing.T) {
+	jsonFile, err := ioutil.TempFile("", "tag-store-test")
+	if err != nil {
+		t.Fatalf("error creating temp file: %v", err)
+	}
+	_, err = jsonFile.Write([]byte(`{}`))
+	jsonFile.Close()
+	defer os.RemoveAll(jsonFile.Name())
+
+	store, err := NewTagStore(jsonFile.Name())
+	if err != nil {
+		t.Fatalf("error creating tag store: %v", err)
+	}
+
+	for refStr, id := range saveLoadTestCases {
+		ref, err := reference.ParseNamed(refStr)
+		if err != nil {
+			t.Fatalf("failed to parse reference: %v", err)
+		}
+		err = store.Add(ref, id, false)
+		if err != nil {
+			t.Fatalf("could not add reference %s: %v", refStr, err)
+		}
+	}
+
+	jsonBytes, err := ioutil.ReadFile(jsonFile.Name())
+	if err != nil {
+		t.Fatalf("could not read json file: %v", err)
+	}
+
+	if !bytes.Equal(jsonBytes, marshalledSaveLoadTestCases) {
+		t.Fatalf("save output did not match expectations\nexpected:\n%s\ngot:\n%s", marshalledSaveLoadTestCases, jsonBytes)
+	}
+}
+
+type LexicalRefs []reference.Named
+
+func (a LexicalRefs) Len() int           { return len(a) }
+func (a LexicalRefs) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a LexicalRefs) Less(i, j int) bool { return a[i].String() < a[j].String() }
+
+type LexicalAssociations []Association
+
+func (a LexicalAssociations) Len() int           { return len(a) }
+func (a LexicalAssociations) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a LexicalAssociations) Less(i, j int) bool { return a[i].Ref.String() < a[j].Ref.String() }
+
+func TestAddDeleteGet(t *testing.T) {
+	jsonFile, err := ioutil.TempFile("", "tag-store-test")
+	if err != nil {
+		t.Fatalf("error creating temp file: %v", err)
+	}
+	_, err = jsonFile.Write([]byte(`{}`))
+	jsonFile.Close()
+	defer os.RemoveAll(jsonFile.Name())
+
+	store, err := NewTagStore(jsonFile.Name())
+	if err != nil {
+		t.Fatalf("error creating tag store: %v", err)
+	}
+
+	testImageID1 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9c")
+	testImageID2 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9d")
+	testImageID3 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9e")
+
+	// Try adding a reference with no tag or digest
+	nameOnly, err := reference.WithName("username/repo")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(nameOnly, testImageID1, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	// Add a few references
+	ref1, err := reference.ParseNamed("username/repo1:latest")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(ref1, testImageID1, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	ref2, err := reference.ParseNamed("username/repo1:old")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(ref2, testImageID2, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	ref3, err := reference.ParseNamed("username/repo1:alias")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(ref3, testImageID1, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	ref4, err := reference.ParseNamed("username/repo2:latest")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(ref4, testImageID2, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	ref5, err := reference.ParseNamed("username/repo3@sha256:58153dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if err = store.Add(ref5, testImageID2, false); err != nil {
+		t.Fatalf("error adding to store: %v", err)
+	}
+
+	// Attempt to overwrite with force == false
+	if err = store.Add(ref4, testImageID3, false); err == nil || !strings.HasPrefix(err.Error(), "Conflict:") {
+		t.Fatalf("did not get expected error on overwrite attempt - got %v", err)
+	}
+	// Repeat to overwrite with force == true
+	if err = store.Add(ref4, testImageID3, true); err != nil {
+		t.Fatalf("failed to force tag overwrite: %v", err)
+	}
+
+	// Check references so far
+	id, err := store.Get(nameOnly)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID1 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
+	}
+
+	id, err = store.Get(ref1)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID1 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
+	}
+
+	id, err = store.Get(ref2)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID2 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID2.String())
+	}
+
+	id, err = store.Get(ref3)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID1 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
+	}
+
+	id, err = store.Get(ref4)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID3 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
+	}
+
+	id, err = store.Get(ref5)
+	if err != nil {
+		t.Fatalf("Get returned error: %v", err)
+	}
+	if id != testImageID2 {
+		t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
+	}
+
+	// Get should return ErrDoesNotExist for a nonexistent repo
+	nonExistRepo, err := reference.ParseNamed("username/nonexistrepo:latest")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if _, err = store.Get(nonExistRepo); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Get")
+	}
+
+	// Get should return ErrDoesNotExist for a nonexistent tag
+	nonExistTag, err := reference.ParseNamed("username/repo1:nonexist")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	if _, err = store.Get(nonExistTag); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Get")
+	}
+
+	// Check References
+	refs := store.References(testImageID1)
+	sort.Sort(LexicalRefs(refs))
+	if len(refs) != 3 {
+		t.Fatal("unexpected number of references")
+	}
+	if refs[0].String() != ref3.String() {
+		t.Fatalf("unexpected reference: %v", refs[0].String())
+	}
+	if refs[1].String() != ref1.String() {
+		t.Fatalf("unexpected reference: %v", refs[1].String())
+	}
+	if refs[2].String() != nameOnly.String()+":latest" {
+		t.Fatalf("unexpected reference: %v", refs[2].String())
+	}
+
+	// Check ReferencesByName
+	repoName, err := reference.WithName("username/repo1")
+	if err != nil {
+		t.Fatalf("could not parse reference: %v", err)
+	}
+	associations := store.ReferencesByName(repoName)
+	sort.Sort(LexicalAssociations(associations))
+	if len(associations) != 3 {
+		t.Fatal("unexpected number of associations")
+	}
+	if associations[0].Ref.String() != ref3.String() {
+		t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
+	}
+	if associations[0].ImageID != testImageID1 {
+		t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
+	}
+	if associations[1].Ref.String() != ref1.String() {
+		t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
+	}
+	if associations[1].ImageID != testImageID1 {
+		t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
+	}
+	if associations[2].Ref.String() != ref2.String() {
+		t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
+	}
+	if associations[2].ImageID != testImageID2 {
+		t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
+	}
+
+	// Delete should return ErrDoesNotExist for a nonexistent repo
+	if _, err = store.Delete(nonExistRepo); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Delete")
+	}
+
+	// Delete should return ErrDoesNotExist for a nonexistent tag
+	if _, err = store.Delete(nonExistTag); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Delete")
+	}
+
+	// Delete a few references
+	if deleted, err := store.Delete(ref1); err != nil || deleted != true {
+		t.Fatal("Delete failed")
+	}
+	if _, err := store.Get(ref1); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Get")
+	}
+	if deleted, err := store.Delete(ref5); err != nil || deleted != true {
+		t.Fatal("Delete failed")
+	}
+	if _, err := store.Get(ref5); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Get")
+	}
+	if deleted, err := store.Delete(nameOnly); err != nil || deleted != true {
+		t.Fatal("Delete failed")
+	}
+	if _, err := store.Get(nameOnly); err != ErrDoesNotExist {
+		t.Fatal("Expected ErrDoesNotExist from Get")
+	}
+}