Add tag store

The tag store associates tags and digests with image IDs. This
functionality used to be part of graph package. This commit splits it
off into a self-contained package with a simple interface.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
This commit is contained in:
Aaron Lehmann 2015-11-18 14:16:21 -08:00
parent 500e77bad0
commit 7de380c5c6
2 changed files with 610 additions and 0 deletions

282
tag/store.go Normal file
View File

@ -0,0 +1,282 @@
package tag
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
)
// DefaultTag defines the default tag used when performing images related actions and no tag string is specified
const DefaultTag = "latest"
var (
// ErrDoesNotExist is returned if a reference is not found in the
// store.
ErrDoesNotExist = errors.New("reference does not exist")
)
// An Association is a tuple associating a reference with an image ID.
type Association struct {
Ref reference.Named
ImageID image.ID
}
// Store provides the set of methods which can operate on a tag store.
type Store interface {
References(id image.ID) []reference.Named
ReferencesByName(ref reference.Named) []Association
Add(ref reference.Named, id image.ID, force bool) error
Delete(ref reference.Named) (bool, error)
Get(ref reference.Named) (image.ID, error)
}
type store struct {
mu sync.RWMutex
// jsonPath is the path to the file where the serialized tag data is
// stored.
jsonPath string
// Repositories is a map of repositories, indexed by name.
Repositories map[string]repository
// referencesByIDCache is a cache of references indexed by ID, to speed
// up References.
referencesByIDCache map[image.ID]map[string]reference.Named
}
// Repository maps tags to image IDs. The key is a a stringified Reference,
// including the repository name.
type repository map[string]image.ID
func defaultTagIfNameOnly(ref reference.Named) reference.Named {
switch ref.(type) {
case reference.Tagged:
return ref
case reference.Digested:
return ref
default:
// Should never fail
ref, _ = reference.WithTag(ref, DefaultTag)
return ref
}
}
// NewTagStore creates a new tag store, tied to a file path where the set of
// tags is serialized in JSON format.
func NewTagStore(jsonPath string) (Store, error) {
abspath, err := filepath.Abs(jsonPath)
if err != nil {
return nil, err
}
store := &store{
jsonPath: abspath,
Repositories: make(map[string]repository),
referencesByIDCache: make(map[image.ID]map[string]reference.Named),
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {
if err := store.save(); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return store, nil
}
// Add adds a tag or digest to the store. If force is set to true, existing
// references can be overwritten. This only works for tags, not digests.
func (store *store) Add(ref reference.Named, id image.ID, force bool) error {
ref = defaultTagIfNameOnly(ref)
store.mu.Lock()
defer store.mu.Unlock()
repository, exists := store.Repositories[ref.Name()]
if !exists || repository == nil {
repository = make(map[string]image.ID)
store.Repositories[ref.Name()] = repository
}
refStr := ref.String()
oldID, exists := repository[refStr]
if exists {
// force only works for tags
if digested, isDigest := ref.(reference.Digested); isDigest {
return fmt.Errorf("Cannot overwrite digest %s", digested.Digest().String())
}
if !force {
return fmt.Errorf("Conflict: Tag %s is already set to image %s, if you want to replace it, please use -f option", ref.String(), oldID.String())
}
if store.referencesByIDCache[oldID] != nil {
delete(store.referencesByIDCache[oldID], refStr)
if len(store.referencesByIDCache[oldID]) == 0 {
delete(store.referencesByIDCache, oldID)
}
}
}
repository[refStr] = id
if store.referencesByIDCache[id] == nil {
store.referencesByIDCache[id] = make(map[string]reference.Named)
}
store.referencesByIDCache[id][refStr] = ref
return store.save()
}
// Delete deletes a reference from the store. It returns true if a deletion
// happened, or false otherwise.
func (store *store) Delete(ref reference.Named) (bool, error) {
ref = defaultTagIfNameOnly(ref)
store.mu.Lock()
defer store.mu.Unlock()
repoName := ref.Name()
repository, exists := store.Repositories[repoName]
if !exists {
return false, ErrDoesNotExist
}
refStr := ref.String()
if id, exists := repository[refStr]; exists {
delete(repository, refStr)
if len(repository) == 0 {
delete(store.Repositories, repoName)
}
if store.referencesByIDCache[id] != nil {
delete(store.referencesByIDCache[id], refStr)
if len(store.referencesByIDCache[id]) == 0 {
delete(store.referencesByIDCache, id)
}
}
return true, store.save()
}
return false, ErrDoesNotExist
}
// Get retrieves an item from the store by reference.
func (store *store) Get(ref reference.Named) (image.ID, error) {
ref = defaultTagIfNameOnly(ref)
store.mu.RLock()
defer store.mu.RUnlock()
repository, exists := store.Repositories[ref.Name()]
if !exists || repository == nil {
return "", ErrDoesNotExist
}
id, exists := repository[ref.String()]
if !exists {
return "", ErrDoesNotExist
}
return id, nil
}
// References returns a slice of references to the given image ID. The slice
// will be nil if there are no references to this image ID.
func (store *store) References(id image.ID) []reference.Named {
store.mu.RLock()
defer store.mu.RUnlock()
// Convert the internal map to an array for two reasons:
// 1) We must not return a mutable reference.
// 2) It would be ugly to expose the extraneous map keys to callers.
var references []reference.Named
for _, ref := range store.referencesByIDCache[id] {
references = append(references, ref)
}
return references
}
// ReferencesByName returns the references for a given repository name.
// If there are no references known for this repository name,
// ReferencesByName returns nil.
func (store *store) ReferencesByName(ref reference.Named) []Association {
store.mu.RLock()
defer store.mu.RUnlock()
repository, exists := store.Repositories[ref.Name()]
if !exists {
return nil
}
var associations []Association
for refStr, refID := range repository {
ref, err := reference.ParseNamed(refStr)
if err != nil {
// Should never happen
return nil
}
associations = append(associations,
Association{
Ref: ref,
ImageID: refID,
})
}
return associations
}
func (store *store) save() error {
// Store the json
jsonData, err := json.Marshal(store)
if err != nil {
return err
}
tempFilePath := store.jsonPath + ".tmp"
if err := ioutil.WriteFile(tempFilePath, jsonData, 0600); err != nil {
return err
}
if err := os.Rename(tempFilePath, store.jsonPath); err != nil {
return err
}
return nil
}
func (store *store) reload() error {
f, err := os.Open(store.jsonPath)
if err != nil {
return err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&store); err != nil {
return err
}
for _, repository := range store.Repositories {
for refStr, refID := range repository {
ref, err := reference.ParseNamed(refStr)
if err != nil {
// Should never happen
continue
}
if store.referencesByIDCache[refID] == nil {
store.referencesByIDCache[refID] = make(map[string]reference.Named)
}
store.referencesByIDCache[refID][refStr] = ref
}
}
return nil
}

328
tag/store_test.go Normal file
View File

@ -0,0 +1,328 @@
package tag
import (
"bytes"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
)
var (
saveLoadTestCases = map[string]image.ID{
"registry:5000/foobar:HEAD": "sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6",
"registry:5000/foobar:alternate": "sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793",
"registry:5000/foobar:latest": "sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b",
"registry:5000/foobar:master": "sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc",
"jess/hollywood:latest": "sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe",
"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6": "sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c",
"busybox:latest": "sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c",
}
marshalledSaveLoadTestCases = []byte(`{"Repositories":{"busybox":{"busybox:latest":"sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c"},"jess/hollywood":{"jess/hollywood:latest":"sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe"},"registry":{"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6":"sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c"},"registry:5000/foobar":{"registry:5000/foobar:HEAD":"sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6","registry:5000/foobar:alternate":"sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793","registry:5000/foobar:latest":"sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b","registry:5000/foobar:master":"sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc"}}}`)
)
func TestLoad(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
defer os.RemoveAll(jsonFile.Name())
// Write canned json to the temp file
_, err = jsonFile.Write(marshalledSaveLoadTestCases)
if err != nil {
t.Fatalf("error writing to temp file: %v", err)
}
jsonFile.Close()
store, err := NewTagStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
for refStr, expectedID := range saveLoadTestCases {
ref, err := reference.ParseNamed(refStr)
if err != nil {
t.Fatalf("failed to parse reference: %v", err)
}
id, err := store.Get(ref)
if err != nil {
t.Fatalf("could not find reference %s: %v", refStr, err)
}
if id != expectedID {
t.Fatalf("expected %s - got %s", expectedID, id)
}
}
}
func TestSave(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
_, err = jsonFile.Write([]byte(`{}`))
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewTagStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
for refStr, id := range saveLoadTestCases {
ref, err := reference.ParseNamed(refStr)
if err != nil {
t.Fatalf("failed to parse reference: %v", err)
}
err = store.Add(ref, id, false)
if err != nil {
t.Fatalf("could not add reference %s: %v", refStr, err)
}
}
jsonBytes, err := ioutil.ReadFile(jsonFile.Name())
if err != nil {
t.Fatalf("could not read json file: %v", err)
}
if !bytes.Equal(jsonBytes, marshalledSaveLoadTestCases) {
t.Fatalf("save output did not match expectations\nexpected:\n%s\ngot:\n%s", marshalledSaveLoadTestCases, jsonBytes)
}
}
type LexicalRefs []reference.Named
func (a LexicalRefs) Len() int { return len(a) }
func (a LexicalRefs) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a LexicalRefs) Less(i, j int) bool { return a[i].String() < a[j].String() }
type LexicalAssociations []Association
func (a LexicalAssociations) Len() int { return len(a) }
func (a LexicalAssociations) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a LexicalAssociations) Less(i, j int) bool { return a[i].Ref.String() < a[j].Ref.String() }
func TestAddDeleteGet(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
_, err = jsonFile.Write([]byte(`{}`))
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewTagStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
testImageID1 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9c")
testImageID2 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9d")
testImageID3 := image.ID("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9e")
// Try adding a reference with no tag or digest
nameOnly, err := reference.WithName("username/repo")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(nameOnly, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
// Add a few references
ref1, err := reference.ParseNamed("username/repo1:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(ref1, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref2, err := reference.ParseNamed("username/repo1:old")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(ref2, testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref3, err := reference.ParseNamed("username/repo1:alias")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(ref3, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref4, err := reference.ParseNamed("username/repo2:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(ref4, testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref5, err := reference.ParseNamed("username/repo3@sha256:58153dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.Add(ref5, testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
// Attempt to overwrite with force == false
if err = store.Add(ref4, testImageID3, false); err == nil || !strings.HasPrefix(err.Error(), "Conflict:") {
t.Fatalf("did not get expected error on overwrite attempt - got %v", err)
}
// Repeat to overwrite with force == true
if err = store.Add(ref4, testImageID3, true); err != nil {
t.Fatalf("failed to force tag overwrite: %v", err)
}
// Check references so far
id, err := store.Get(nameOnly)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref1)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref2)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID2 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID2.String())
}
id, err = store.Get(ref3)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref4)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID3 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
}
id, err = store.Get(ref5)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID2 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
}
// Get should return ErrDoesNotExist for a nonexistent repo
nonExistRepo, err := reference.ParseNamed("username/nonexistrepo:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if _, err = store.Get(nonExistRepo); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
// Get should return ErrDoesNotExist for a nonexistent tag
nonExistTag, err := reference.ParseNamed("username/repo1:nonexist")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if _, err = store.Get(nonExistTag); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
// Check References
refs := store.References(testImageID1)
sort.Sort(LexicalRefs(refs))
if len(refs) != 3 {
t.Fatal("unexpected number of references")
}
if refs[0].String() != ref3.String() {
t.Fatalf("unexpected reference: %v", refs[0].String())
}
if refs[1].String() != ref1.String() {
t.Fatalf("unexpected reference: %v", refs[1].String())
}
if refs[2].String() != nameOnly.String()+":latest" {
t.Fatalf("unexpected reference: %v", refs[2].String())
}
// Check ReferencesByName
repoName, err := reference.WithName("username/repo1")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
associations := store.ReferencesByName(repoName)
sort.Sort(LexicalAssociations(associations))
if len(associations) != 3 {
t.Fatal("unexpected number of associations")
}
if associations[0].Ref.String() != ref3.String() {
t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
}
if associations[0].ImageID != testImageID1 {
t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
}
if associations[1].Ref.String() != ref1.String() {
t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
}
if associations[1].ImageID != testImageID1 {
t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
}
if associations[2].Ref.String() != ref2.String() {
t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
}
if associations[2].ImageID != testImageID2 {
t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
}
// Delete should return ErrDoesNotExist for a nonexistent repo
if _, err = store.Delete(nonExistRepo); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Delete")
}
// Delete should return ErrDoesNotExist for a nonexistent tag
if _, err = store.Delete(nonExistTag); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Delete")
}
// Delete a few references
if deleted, err := store.Delete(ref1); err != nil || deleted != true {
t.Fatal("Delete failed")
}
if _, err := store.Get(ref1); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
if deleted, err := store.Delete(ref5); err != nil || deleted != true {
t.Fatal("Delete failed")
}
if _, err := store.Get(ref5); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
if deleted, err := store.Delete(nameOnly); err != nil || deleted != true {
t.Fatal("Delete failed")
}
if _, err := store.Get(nameOnly); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
}