diff --git a/image/fixtures/post1.9/expected_computed_id b/image/fixtures/post1.9/expected_computed_id deleted file mode 100644 index cba6d81f4e..0000000000 --- a/image/fixtures/post1.9/expected_computed_id +++ /dev/null @@ -1 +0,0 @@ -sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105 diff --git a/image/fixtures/post1.9/expected_config b/image/fixtures/post1.9/expected_config deleted file mode 100644 index ae27bdd429..0000000000 --- a/image/fixtures/post1.9/expected_config +++ /dev/null @@ -1 +0,0 @@ -{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} diff --git a/image/fixtures/post1.9/layer_id b/image/fixtures/post1.9/layer_id deleted file mode 100644 index ded2db28e7..0000000000 --- a/image/fixtures/post1.9/layer_id +++ /dev/null @@ -1 +0,0 @@ -sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/post1.9/parent_id b/image/fixtures/post1.9/parent_id deleted file mode 100644 index 7d524f80c2..0000000000 --- a/image/fixtures/post1.9/parent_id +++ /dev/null @@ -1 +0,0 @@ -sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/post1.9/v1compatibility b/image/fixtures/post1.9/v1compatibility deleted file mode 100644 index d6697c2b68..0000000000 --- a/image/fixtures/post1.9/v1compatibility +++ /dev/null @@ -1 +0,0 @@ -{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"} diff --git a/image/fixtures/pre1.9/expected_computed_id b/image/fixtures/pre1.9/expected_computed_id deleted file mode 100644 index c27b0b6a20..0000000000 --- a/image/fixtures/pre1.9/expected_computed_id +++ /dev/null @@ -1 +0,0 @@ -sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89 diff --git a/image/fixtures/pre1.9/expected_config b/image/fixtures/pre1.9/expected_config deleted file mode 100644 index 121efe1fe6..0000000000 --- a/image/fixtures/pre1.9/expected_config +++ /dev/null @@ -1,2 +0,0 @@ -{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} - diff --git a/image/fixtures/pre1.9/layer_id b/image/fixtures/pre1.9/layer_id deleted file mode 100644 index ded2db28e7..0000000000 --- a/image/fixtures/pre1.9/layer_id +++ /dev/null @@ -1 +0,0 @@ -sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/pre1.9/parent_id b/image/fixtures/pre1.9/parent_id deleted file mode 100644 index 7d524f80c2..0000000000 --- a/image/fixtures/pre1.9/parent_id +++ /dev/null @@ -1 +0,0 @@ -sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/pre1.9/v1compatibility b/image/fixtures/pre1.9/v1compatibility deleted file mode 100644 index af96e82506..0000000000 --- a/image/fixtures/pre1.9/v1compatibility +++ /dev/null @@ -1 +0,0 @@ -{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0} diff --git a/image/fs.go b/image/fs.go new file mode 100644 index 0000000000..7c1c4c22d6 --- /dev/null +++ b/image/fs.go @@ -0,0 +1,192 @@ +package image + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" +) + +// IDWalkFunc is function called by StoreBackend.Walk +type IDWalkFunc func(id ID) error + +// StoreBackend provides interface for image.Store persistence +type StoreBackend interface { + Walk(f IDWalkFunc) error + Get(id ID) ([]byte, error) + Set(data []byte) (ID, error) + Delete(id ID) error + SetMetadata(id ID, key string, data []byte) error + GetMetadata(id ID, key string) ([]byte, error) + DeleteMetadata(id ID, key string) error +} + +// fs implements StoreBackend using the filesystem. +type fs struct { + sync.RWMutex + root string +} + +const ( + contentDirName = "content" + metadataDirName = "metadata" +) + +// NewFSStoreBackend returns new filesystem based backend for image.Store +func NewFSStoreBackend(root string) (StoreBackend, error) { + return newFSStore(root) +} + +func newFSStore(root string) (*fs, error) { + s := &fs{ + root: root, + } + if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil { + return nil, err + } + return s, nil +} + +func (s *fs) contentFile(id ID) string { + dgst := digest.Digest(id) + return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex()) +} + +func (s *fs) metadataDir(id ID) string { + dgst := digest.Digest(id) + return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex()) +} + +// Walk calls the supplied callback for each image ID in the storage backend. +func (s *fs) Walk(f IDWalkFunc) error { + // Only Canonical digest (sha256) is currently supported + s.RLock() + dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical))) + s.RUnlock() + if err != nil { + return err + } + for _, v := range dir { + dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name()) + if err := dgst.Validate(); err != nil { + logrus.Debugf("Skipping invalid digest %s: %s", dgst, err) + continue + } + if err := f(ID(dgst)); err != nil { + return err + } + } + return nil +} + +// Get returns the content stored under a given ID. +func (s *fs) Get(id ID) ([]byte, error) { + s.RLock() + defer s.RUnlock() + + return s.get(id) +} + +func (s *fs) get(id ID) ([]byte, error) { + content, err := ioutil.ReadFile(s.contentFile(id)) + if err != nil { + return nil, err + } + + // todo: maybe optional + validated, err := digest.FromBytes(content) + if err != nil { + return nil, err + } + if ID(validated) != id { + return nil, fmt.Errorf("failed to verify image: %v", id) + } + + return content, nil +} + +// Set stores content under a given ID. +func (s *fs) Set(data []byte) (ID, error) { + s.Lock() + defer s.Unlock() + + if len(data) == 0 { + return "", fmt.Errorf("Invalid empty data") + } + + dgst, err := digest.FromBytes(data) + if err != nil { + return "", err + } + id := ID(dgst) + filePath := s.contentFile(id) + tempFilePath := s.contentFile(id) + ".tmp" + if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil { + return "", err + } + if err := os.Rename(tempFilePath, filePath); err != nil { + return "", err + } + + return id, nil +} + +// Delete removes content and metadata files associated with the ID. +func (s *fs) Delete(id ID) error { + s.Lock() + defer s.Unlock() + + if err := os.RemoveAll(s.metadataDir(id)); err != nil { + return err + } + if err := os.Remove(s.contentFile(id)); err != nil { + return err + } + return nil +} + +// SetMetadata sets metadata for a given ID. It fails if there's no base file. +func (s *fs) SetMetadata(id ID, key string, data []byte) error { + s.Lock() + defer s.Unlock() + if _, err := s.get(id); err != nil { + return err + } + + baseDir := filepath.Join(s.metadataDir(id)) + if err := os.MkdirAll(baseDir, 0700); err != nil { + return err + } + filePath := filepath.Join(s.metadataDir(id), key) + tempFilePath := filePath + ".tmp" + if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil { + return err + } + return os.Rename(tempFilePath, filePath) +} + +// GetMetadata returns metadata for a given ID. +func (s *fs) GetMetadata(id ID, key string) ([]byte, error) { + s.RLock() + defer s.RUnlock() + + if _, err := s.get(id); err != nil { + return nil, err + } + return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key)) +} + +// DeleteMetadata removes the metadata associated with an ID. +func (s *fs) DeleteMetadata(id ID, key string) error { + s.Lock() + defer s.Unlock() + + return os.RemoveAll(filepath.Join(s.metadataDir(id), key)) +} diff --git a/image/fs_test.go b/image/fs_test.go new file mode 100644 index 0000000000..0790b78e45 --- /dev/null +++ b/image/fs_test.go @@ -0,0 +1,391 @@ +package image + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/distribution/digest" +) + +func TestFSGetSet(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + testGetSet(t, fs) +} + +func TestFSGetInvalidData(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + id, err := fs.Set([]byte("foobar")) + if err != nil { + t.Fatal(err) + } + + dgst := digest.Digest(id) + + if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, string(dgst.Algorithm()), dgst.Hex()), []byte("foobar2"), 0600); err != nil { + t.Fatal(err) + } + + _, err = fs.Get(id) + if err == nil { + t.Fatal("Expected get to fail after data modification.") + } +} + +func TestFSInvalidSet(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + id, err := digest.FromBytes([]byte("foobar")) + if err != nil { + t.Fatal(err) + } + err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700) + if err != nil { + t.Fatal(err) + } + + _, err = fs.Set([]byte("foobar")) + if err == nil { + t.Fatal("Expecting error from invalid filesystem data.") + } +} + +func TestFSInvalidRoot(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + tcases := []struct { + root, invalidFile string + }{ + {"root", "root"}, + {"root", "root/content"}, + {"root", "root/metadata"}, + } + + for _, tc := range tcases { + root := filepath.Join(tmpdir, tc.root) + filePath := filepath.Join(tmpdir, tc.invalidFile) + err := os.MkdirAll(filepath.Dir(filePath), 0700) + if err != nil { + t.Fatal(err) + } + f, err := os.Create(filePath) + if err != nil { + t.Fatal(err) + } + f.Close() + + _, err = NewFSStoreBackend(root) + if err == nil { + t.Fatalf("Expected error from root %q and invlid file %q", tc.root, tc.invalidFile) + } + + os.RemoveAll(root) + } + +} + +func testMetadataGetSet(t *testing.T, store StoreBackend) { + id, err := store.Set([]byte("foo")) + if err != nil { + t.Fatal(err) + } + id2, err := store.Set([]byte("bar")) + if err != nil { + t.Fatal(err) + } + + tcases := []struct { + id ID + key string + value []byte + }{ + {id, "tkey", []byte("tval1")}, + {id, "tkey2", []byte("tval2")}, + {id2, "tkey", []byte("tval3")}, + } + + for _, tc := range tcases { + err = store.SetMetadata(tc.id, tc.key, tc.value) + if err != nil { + t.Fatal(err) + } + + actual, err := store.GetMetadata(tc.id, tc.key) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(actual, tc.value) != 0 { + t.Fatalf("Metadata expected %q, got %q", tc.value, actual) + } + } + + _, err = store.GetMetadata(id2, "tkey2") + if err == nil { + t.Fatal("Expected error for getting metadata for unknown key") + } + + id3, err := digest.FromBytes([]byte("baz")) + if err != nil { + t.Fatal(err) + } + + err = store.SetMetadata(ID(id3), "tkey", []byte("tval")) + if err == nil { + t.Fatal("Expected error for setting metadata for unknown ID.") + } + + _, err = store.GetMetadata(ID(id3), "tkey") + if err == nil { + t.Fatal("Expected error for getting metadata for unknown ID.") + } +} + +func TestFSMetadataGetSet(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + testMetadataGetSet(t, fs) +} + +func TestFSDelete(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + testDelete(t, fs) +} + +func TestFSWalker(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + testWalker(t, fs) +} + +func TestFSInvalidWalker(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + fooID, err := fs.Set([]byte("foo")) + if err != nil { + t.Fatal(err) + } + + if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, "sha256/foobar"), []byte("foobar"), 0600); err != nil { + t.Fatal(err) + } + + n := 0 + err = fs.Walk(func(id ID) error { + if id != fooID { + t.Fatalf("Invalid walker ID %q, expected %q", id, fooID) + } + n++ + return nil + }) + if err != nil { + t.Fatalf("Invalid data should not have caused walker error, got %v", err) + } + if n != 1 { + t.Fatalf("Expected 1 walk initialization, got %d", n) + } +} + +func testGetSet(t *testing.T, store StoreBackend) { + type tcase struct { + input []byte + expected ID + } + tcases := []tcase{ + {[]byte("foobar"), ID("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")}, + } + + randomInput := make([]byte, 8*1024) + _, err := rand.Read(randomInput) + if err != nil { + t.Fatal(err) + } + // skipping use of digest pkg because its used by the imlementation + h := sha256.New() + _, err = h.Write(randomInput) + if err != nil { + t.Fatal(err) + } + tcases = append(tcases, tcase{ + input: randomInput, + expected: ID("sha256:" + hex.EncodeToString(h.Sum(nil))), + }) + + for _, tc := range tcases { + id, err := store.Set([]byte(tc.input)) + if err != nil { + t.Fatal(err) + } + if id != tc.expected { + t.Fatalf("Expected ID %q, got %q", tc.expected, id) + } + } + + for _, emptyData := range [][]byte{nil, {}} { + _, err := store.Set(emptyData) + if err == nil { + t.Fatal("Expected error for nil input.") + } + } + + for _, tc := range tcases { + data, err := store.Get(tc.expected) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(data, tc.input) != 0 { + t.Fatalf("Expected data %q, got %q", tc.input, data) + } + } + + for _, key := range []ID{"foobar:abc", "sha256:abc", "sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2a"} { + _, err := store.Get(key) + if err == nil { + t.Fatalf("Expected error for ID %q.", key) + } + } + +} + +func testDelete(t *testing.T, store StoreBackend) { + id, err := store.Set([]byte("foo")) + if err != nil { + t.Fatal(err) + } + id2, err := store.Set([]byte("bar")) + if err != nil { + t.Fatal(err) + } + + err = store.Delete(id) + if err != nil { + t.Fatal(err) + } + + _, err = store.Get(id) + if err == nil { + t.Fatalf("Expected getting deleted item %q to fail", id) + } + _, err = store.Get(id2) + if err != nil { + t.Fatal(err) + } + + err = store.Delete(id2) + if err != nil { + t.Fatal(err) + } + _, err = store.Get(id2) + if err == nil { + t.Fatalf("Expected getting deleted item %q to fail", id2) + } +} + +func testWalker(t *testing.T, store StoreBackend) { + id, err := store.Set([]byte("foo")) + if err != nil { + t.Fatal(err) + } + id2, err := store.Set([]byte("bar")) + if err != nil { + t.Fatal(err) + } + + tcases := make(map[ID]struct{}) + tcases[id] = struct{}{} + tcases[id2] = struct{}{} + n := 0 + err = store.Walk(func(id ID) error { + delete(tcases, id) + n++ + return nil + }) + if err != nil { + t.Fatal(err) + } + + if n != 2 { + t.Fatalf("Expected 2 walk initializations, got %d", n) + } + if len(tcases) != 0 { + t.Fatalf("Expected empty unwalked set, got %+v", tcases) + } + + // stop on error + tcases = make(map[ID]struct{}) + tcases[id] = struct{}{} + err = store.Walk(func(id ID) error { + return errors.New("") + }) + if err == nil { + t.Fatalf("Exected error from walker.") + } +} diff --git a/image/image.go b/image/image.go index 89799160da..ed44ef377f 100644 --- a/image/image.go +++ b/image/image.go @@ -2,36 +2,23 @@ package image import ( "encoding/json" - "fmt" - "regexp" + "errors" + "io" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" - derr "github.com/docker/docker/errors" - "github.com/docker/docker/pkg/version" "github.com/docker/docker/runconfig" ) -var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) +// ID is the content-addressable ID of an image. +type ID digest.Digest -// noFallbackMinVersion is the minimum version for which v1compatibility -// information will not be marshaled through the Image struct to remove -// blank fields. -var noFallbackMinVersion = version.Version("1.8.3") - -// Descriptor provides the information necessary to register an image in -// the graph. -type Descriptor interface { - ID() string - Parent() string - MarshalConfig() ([]byte, error) +func (id ID) String() string { + return digest.Digest(id).String() } -// Image stores the image configuration. -// All fields in this struct must be marked `omitempty` to keep getting -// predictable hashes from the old `v1Compatibility` configuration. -type Image struct { +// V1Image stores the V1 image configuration. +type V1Image struct { // ID a unique 64 character identifier of the image ID string `json:"id,omitempty"` // Parent id of the image @@ -55,95 +42,87 @@ type Image struct { // OS is the operating system used to build and run the image OS string `json:"os,omitempty"` // Size is the total size of the image including all layers it is composed of - Size int64 `json:",omitempty"` // capitalized for backwards compatibility - // ParentID specifies the strong, content address of the parent configuration. - ParentID digest.Digest `json:"parent_id,omitempty"` - // LayerID provides the content address of the associated layer. - LayerID digest.Digest `json:"layer_id,omitempty"` + Size int64 `json:",omitempty"` } -// NewImgJSON creates an Image configuration from json. -func NewImgJSON(src []byte) (*Image, error) { - ret := &Image{} +// Image stores the image configuration +type Image struct { + V1Image + Parent ID `json:"parent,omitempty"` + RootFS *RootFS `json:"rootfs,omitempty"` + History []History `json:"history,omitempty"` - // FIXME: Is there a cleaner way to "purify" the input json? - if err := json.Unmarshal(src, ret); err != nil { - return nil, err - } - return ret, nil + // rawJSON caches the immutable JSON associated with this image. + rawJSON []byte + + // computedID is the ID computed from the hash of the image config. + // Not to be confused with the legacy V1 ID in V1Image. + computedID ID } -// ValidateID checks whether an ID string is a valid image ID. -func ValidateID(id string) error { - if ok := validHex.MatchString(id); !ok { - return derr.ErrorCodeInvalidImageID.WithArgs(id) - } - return nil +// RawJSON returns the immutable JSON associated with the image. +func (img *Image) RawJSON() []byte { + return img.rawJSON } -// MakeImageConfig returns immutable configuration JSON for image based on the -// v1Compatibility object, layer digest and parent StrongID. SHA256() of this -// config is the new image ID (strongID). -func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) { +// ID returns the image's content-addressable ID. +func (img *Image) ID() ID { + return img.computedID +} - // Detect images created after 1.8.3 - img, err := NewImgJSON(v1Compatibility) +// MarshalJSON serializes the image to JSON. It sorts the top-level keys so +// that JSON that's been manipulated by a push/pull cycle with a legacy +// registry won't end up with a different key order. +func (img *Image) MarshalJSON() ([]byte, error) { + type MarshalImage Image + + pass1, err := json.Marshal(MarshalImage(*img)) if err != nil { return nil, err } - useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion) - - if useFallback { - // Fallback for pre-1.8.3. Calculate base config based on Image struct - // so that fields with default values added by Docker will use same ID - logrus.Debugf("Using fallback hash for %v", layerID) - - v1Compatibility, err = json.Marshal(img) - if err != nil { - return nil, err - } - } var c map[string]*json.RawMessage - if err := json.Unmarshal(v1Compatibility, &c); err != nil { + if err := json.Unmarshal(pass1, &c); err != nil { return nil, err } - - if err := layerID.Validate(); err != nil { - return nil, fmt.Errorf("invalid layerID: %v", err) - } - - c["layer_id"] = rawJSON(layerID) - - if parentID != "" { - if err := parentID.Validate(); err != nil { - return nil, fmt.Errorf("invalid parentID %v", err) - } - c["parent_id"] = rawJSON(parentID) - } - - delete(c, "id") - delete(c, "parent") - delete(c, "Size") // Size is calculated from data on disk and is inconsitent - return json.Marshal(c) } -// StrongID returns image ID for the config JSON. -func StrongID(configJSON []byte) (digest.Digest, error) { - digester := digest.Canonical.New() - if _, err := digester.Hash().Write(configJSON); err != nil { - return "", err - } - dgst := digester.Digest() - logrus.Debugf("H(%v) = %v", string(configJSON), dgst) - return dgst, nil +// History stores build commands that were used to create an image +type History struct { + // Created timestamp for build point + Created time.Time `json:"created"` + // Author of the build point + Author string `json:"author,omitempty"` + // CreatedBy keeps the Dockerfile command used while building image. + CreatedBy string `json:"created_by,omitempty"` + // Comment is custom mesage set by the user when creating the image. + Comment string `json:"comment,omitempty"` + // EmptyLayer is set to true if this history item did not generate a + // layer. Otherwise, the history item is associated with the next + // layer in the RootFS section. + EmptyLayer bool `json:"empty_layer,omitempty"` } -func rawJSON(value interface{}) *json.RawMessage { - jsonval, err := json.Marshal(value) - if err != nil { - return nil - } - return (*json.RawMessage)(&jsonval) +// Exporter provides interface for exporting and importing images +type Exporter interface { + Load(io.ReadCloser, io.Writer) error + // TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error + Save([]string, io.Writer) error +} + +// NewFromJSON creates an Image configuration from json. +func NewFromJSON(src []byte) (*Image, error) { + img := &Image{} + + if err := json.Unmarshal(src, img); err != nil { + return nil, err + } + if img.RootFS == nil { + return nil, errors.New("Invalid image JSON, no RootFS key.") + } + + img.rawJSON = src + + return img, nil } diff --git a/image/image_test.go b/image/image_test.go index 77d92c4490..525023b813 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -1,55 +1,59 @@ package image import ( - "bytes" - "io/ioutil" + "encoding/json" + "sort" + "strings" "testing" - - "github.com/docker/distribution/digest" ) -var fixtures = []string{ - "fixtures/pre1.9", - "fixtures/post1.9", -} +const sampleImageJSON = `{ + "architecture": "amd64", + "os": "linux", + "config": {}, + "rootfs": { + "type": "layers", + "diff_ids": [] + } +}` -func loadFixtureFile(t *testing.T, path string) []byte { - fileData, err := ioutil.ReadFile(path) +func TestJSON(t *testing.T) { + img, err := NewFromJSON([]byte(sampleImageJSON)) if err != nil { - t.Fatalf("error opening %s: %v", path, err) + t.Fatal(err) } - - return bytes.TrimSpace(fileData) -} - -// TestMakeImageConfig makes sure that MakeImageConfig returns the expected -// canonical JSON for a reference Image. -func TestMakeImageConfig(t *testing.T) { - for _, fixture := range fixtures { - v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility") - expectedConfig := loadFixtureFile(t, fixture+"/expected_config") - layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id")) - parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id")) - - json, err := MakeImageConfig(v1Compatibility, layerID, parentID) - if err != nil { - t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err) - } - if !bytes.Equal(json, expectedConfig) { - t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json) - } + rawJSON := img.RawJSON() + if string(rawJSON) != sampleImageJSON { + t.Fatalf("Raw JSON of config didn't match: expected %+v, got %v", sampleImageJSON, rawJSON) } } -// TestGetStrongID makes sure that GetConfigJSON returns the expected -// hash for a reference Image. -func TestGetStrongID(t *testing.T) { - for _, fixture := range fixtures { - expectedConfig := loadFixtureFile(t, fixture+"/expected_config") - expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id")) - - if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID { - t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err) - } +func TestInvalidJSON(t *testing.T) { + _, err := NewFromJSON([]byte("{}")) + if err == nil { + t.Fatal("Expected JSON parse error") + } +} + +func TestMarshalKeyOrder(t *testing.T) { + b, err := json.Marshal(&Image{ + V1Image: V1Image{ + Comment: "a", + Author: "b", + Architecture: "c", + }, + }) + if err != nil { + t.Fatal(err) + } + + expectedOrder := []string{"architecture", "author", "comment"} + var indexes []int + for _, k := range expectedOrder { + indexes = append(indexes, strings.Index(string(b), k)) + } + + if !sort.IntsAreSorted(indexes) { + t.Fatal("invalid key order in JSON: ", string(b)) } } diff --git a/image/rootfs.go b/image/rootfs.go new file mode 100644 index 0000000000..b546696d6a --- /dev/null +++ b/image/rootfs.go @@ -0,0 +1,8 @@ +package image + +import "github.com/docker/docker/layer" + +// Append appends a new diffID to rootfs +func (r *RootFS) Append(id layer.DiffID) { + r.DiffIDs = append(r.DiffIDs, id) +} diff --git a/image/rootfs_unix.go b/image/rootfs_unix.go new file mode 100644 index 0000000000..e817db5184 --- /dev/null +++ b/image/rootfs_unix.go @@ -0,0 +1,23 @@ +// +build !windows + +package image + +import "github.com/docker/docker/layer" + +// RootFS describes images root filesystem +// This is currently a placeholder that only supports layers. In the future +// this can be made into a interface that supports different implementaions. +type RootFS struct { + Type string `json:"type"` + DiffIDs []layer.DiffID `json:"diff_ids,omitempty"` +} + +// ChainID returns the ChainID for the top layer in RootFS. +func (r *RootFS) ChainID() layer.ChainID { + return layer.CreateChainID(r.DiffIDs) +} + +// NewRootFS returns empty RootFS struct +func NewRootFS() *RootFS { + return &RootFS{Type: "layers"} +} diff --git a/image/rootfs_windows.go b/image/rootfs_windows.go new file mode 100644 index 0000000000..6a2b179d45 --- /dev/null +++ b/image/rootfs_windows.go @@ -0,0 +1,37 @@ +// +build windows + +package image + +import ( + "crypto/sha512" + "fmt" + + "github.com/docker/distribution/digest" + "github.com/docker/docker/layer" +) + +// RootFS describes images root filesystem +// This is currently a placeholder that only supports layers. In the future +// this can be made into a interface that supports different implementaions. +type RootFS struct { + Type string `json:"type"` + DiffIDs []layer.DiffID `json:"diff_ids,omitempty"` + BaseLayer string `json:"base_layer,omitempty"` +} + +// BaseLayerID returns the 64 byte hex ID for the baselayer name. +func (r *RootFS) BaseLayerID() string { + baseID := sha512.Sum384([]byte(r.BaseLayer)) + return fmt.Sprintf("%x", baseID[:32]) +} + +// ChainID returns the ChainID for the top layer in RootFS. +func (r *RootFS) ChainID() layer.ChainID { + baseDiffID, _ := digest.FromBytes([]byte(r.BaseLayerID())) // can never error + return layer.CreateChainID(append([]layer.DiffID{layer.DiffID(baseDiffID)}, r.DiffIDs...)) +} + +// NewRootFS returns empty RootFS struct +func NewRootFS() *RootFS { + return &RootFS{Type: "layers+base"} +} diff --git a/image/store.go b/image/store.go new file mode 100644 index 0000000000..a9c02d6ed3 --- /dev/null +++ b/image/store.go @@ -0,0 +1,286 @@ +package image + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/docker/layer" +) + +// Store is an interface for creating and accessing images +type Store interface { + Create(config []byte) (ID, error) + Get(id ID) (*Image, error) + Delete(id ID) ([]layer.Metadata, error) + Search(partialID string) (ID, error) + SetParent(id ID, parent ID) error + GetParent(id ID) (ID, error) + Children(id ID) []ID + Map() map[ID]*Image + Heads() map[ID]*Image +} + +// LayerGetReleaser is a minimal interface for getting and releasing images. +type LayerGetReleaser interface { + Get(layer.ChainID) (layer.Layer, error) + Release(layer.Layer) ([]layer.Metadata, error) +} + +type imageMeta struct { + layer layer.Layer + children map[ID]struct{} +} + +type store struct { + sync.Mutex + ls LayerGetReleaser + images map[ID]*imageMeta + fs StoreBackend + digestSet *digest.Set +} + +// NewImageStore returns new store object for given layer store +func NewImageStore(fs StoreBackend, ls LayerGetReleaser) (Store, error) { + is := &store{ + ls: ls, + images: make(map[ID]*imageMeta), + fs: fs, + digestSet: digest.NewSet(), + } + + // load all current images and retain layers + if err := is.restore(); err != nil { + return nil, err + } + + return is, nil +} + +func (is *store) restore() error { + err := is.fs.Walk(func(id ID) error { + img, err := is.Get(id) + if err != nil { + logrus.Errorf("invalid image %v, %v", id, err) + return nil + } + var l layer.Layer + if chainID := img.RootFS.ChainID(); chainID != "" { + l, err = is.ls.Get(chainID) + if err != nil { + return err + } + } + if err := is.digestSet.Add(digest.Digest(id)); err != nil { + return err + } + + imageMeta := &imageMeta{ + layer: l, + children: make(map[ID]struct{}), + } + + is.images[ID(id)] = imageMeta + + return nil + }) + if err != nil { + return err + } + + // Second pass to fill in children maps + for id := range is.images { + if parent, err := is.GetParent(id); err == nil { + if parentMeta := is.images[parent]; parentMeta != nil { + parentMeta.children[id] = struct{}{} + } + } + } + + return nil +} + +func (is *store) Create(config []byte) (ID, error) { + var img Image + err := json.Unmarshal(config, &img) + if err != nil { + return "", err + } + + // Must reject any config that references diffIDs from the history + // which aren't among the rootfs layers. + rootFSLayers := make(map[layer.DiffID]struct{}) + for _, diffID := range img.RootFS.DiffIDs { + rootFSLayers[diffID] = struct{}{} + } + + layerCounter := 0 + for _, h := range img.History { + if !h.EmptyLayer { + layerCounter++ + } + } + if layerCounter > len(img.RootFS.DiffIDs) { + return "", errors.New("too many non-empty layers in History section") + } + + dgst, err := is.fs.Set(config) + if err != nil { + return "", err + } + imageID := ID(dgst) + + is.Lock() + defer is.Unlock() + + if _, exists := is.images[imageID]; exists { + return imageID, nil + } + + layerID := img.RootFS.ChainID() + + var l layer.Layer + if layerID != "" { + l, err = is.ls.Get(layerID) + if err != nil { + return "", err + } + } + + imageMeta := &imageMeta{ + layer: l, + children: make(map[ID]struct{}), + } + + is.images[imageID] = imageMeta + if err := is.digestSet.Add(digest.Digest(imageID)); err != nil { + delete(is.images, imageID) + return "", err + } + + return imageID, nil +} + +func (is *store) Search(term string) (ID, error) { + is.Lock() + defer is.Unlock() + + dgst, err := is.digestSet.Lookup(term) + if err != nil { + return "", err + } + return ID(dgst), nil +} + +func (is *store) Get(id ID) (*Image, error) { + // todo: Check if image is in images + // todo: Detect manual insertions and start using them + config, err := is.fs.Get(id) + if err != nil { + return nil, err + } + + img, err := NewFromJSON(config) + if err != nil { + return nil, err + } + img.computedID = id + + img.Parent, err = is.GetParent(id) + if err != nil { + img.Parent = "" + } + + return img, nil +} + +func (is *store) Delete(id ID) ([]layer.Metadata, error) { + is.Lock() + defer is.Unlock() + + imageMeta := is.images[id] + if imageMeta == nil { + return nil, fmt.Errorf("unrecognized image ID %s", id.String()) + } + for id := range imageMeta.children { + is.fs.DeleteMetadata(id, "parent") + } + if parent, err := is.GetParent(id); err == nil && is.images[parent] != nil { + delete(is.images[parent].children, id) + } + + delete(is.images, id) + is.fs.Delete(id) + + if imageMeta.layer != nil { + return is.ls.Release(imageMeta.layer) + } + return nil, nil +} + +func (is *store) SetParent(id, parent ID) error { + is.Lock() + defer is.Unlock() + parentMeta := is.images[parent] + if parentMeta == nil { + return fmt.Errorf("unknown parent image ID %s", parent.String()) + } + parentMeta.children[id] = struct{}{} + return is.fs.SetMetadata(id, "parent", []byte(parent)) +} + +func (is *store) GetParent(id ID) (ID, error) { + d, err := is.fs.GetMetadata(id, "parent") + if err != nil { + return "", err + } + return ID(d), nil // todo: validate? +} + +func (is *store) Children(id ID) []ID { + is.Lock() + defer is.Unlock() + + return is.children(id) +} + +func (is *store) children(id ID) []ID { + var ids []ID + if is.images[id] != nil { + for id := range is.images[id].children { + ids = append(ids, id) + } + } + return ids +} + +func (is *store) Heads() map[ID]*Image { + return is.imagesMap(false) +} + +func (is *store) Map() map[ID]*Image { + return is.imagesMap(true) +} + +func (is *store) imagesMap(all bool) map[ID]*Image { + is.Lock() + defer is.Unlock() + + images := make(map[ID]*Image) + + for id := range is.images { + if !all && len(is.children(id)) > 0 { + continue + } + img, err := is.Get(id) + if err != nil { + logrus.Errorf("invalid image access: %q, error: %q", id, err) + continue + } + images[id] = img + } + return images +} diff --git a/image/store_test.go b/image/store_test.go new file mode 100644 index 0000000000..756b3922d6 --- /dev/null +++ b/image/store_test.go @@ -0,0 +1,205 @@ +package image + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/docker/distribution/digest" + "github.com/docker/docker/layer" +) + +func TestRestore(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + id1, err := fs.Set([]byte(`{"comment": "abc", "rootfs": {"type": "layers"}}`)) + if err != nil { + t.Fatal(err) + } + _, err = fs.Set([]byte(`invalid`)) + if err != nil { + t.Fatal(err) + } + id2, err := fs.Set([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`)) + if err != nil { + t.Fatal(err) + } + err = fs.SetMetadata(id2, "parent", []byte(id1)) + if err != nil { + t.Fatal(err) + } + + is, err := NewImageStore(fs, &mockLayerGetReleaser{}) + if err != nil { + t.Fatal(err) + } + + imgs := is.Map() + if actual, expected := len(imgs), 2; actual != expected { + t.Fatalf("invalid images length, expected 2, got %q", len(imgs)) + } + + img1, err := is.Get(ID(id1)) + if err != nil { + t.Fatal(err) + } + + if actual, expected := img1.computedID, ID(id1); actual != expected { + t.Fatalf("invalid image ID: expected %q, got %q", expected, actual) + } + + if actual, expected := img1.computedID.String(), string(id1); actual != expected { + t.Fatalf("invalid image ID string: expected %q, got %q", expected, actual) + } + + img2, err := is.Get(ID(id2)) + if err != nil { + t.Fatal(err) + } + + if actual, expected := img1.Comment, "abc"; actual != expected { + t.Fatalf("invalid comment for image1: expected %q, got %q", expected, actual) + } + + if actual, expected := img2.Comment, "def"; actual != expected { + t.Fatalf("invalid comment for image2: expected %q, got %q", expected, actual) + } + + p, err := is.GetParent(ID(id1)) + if err == nil { + t.Fatal("expected error for getting parent") + } + + p, err = is.GetParent(ID(id2)) + if err != nil { + t.Fatal(err) + } + if actual, expected := p, ID(id1); actual != expected { + t.Fatalf("invalid parent: expected %q, got %q", expected, actual) + } + + children := is.Children(ID(id1)) + if len(children) != 1 { + t.Fatalf("invalid children length: %q", len(children)) + } + if actual, expected := children[0], ID(id2); actual != expected { + t.Fatalf("invalid child for id1: expected %q, got %q", expected, actual) + } + + heads := is.Heads() + if actual, expected := len(heads), 1; actual != expected { + t.Fatalf("invalid images length: expected %q, got %q", expected, actual) + } + + sid1, err := is.Search(string(id1)[:10]) + if err != nil { + t.Fatal(err) + } + if actual, expected := sid1, ID(id1); actual != expected { + t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual) + } + + sid1, err = is.Search(digest.Digest(id1).Hex()[:6]) + if err != nil { + t.Fatal(err) + } + if actual, expected := sid1, ID(id1); actual != expected { + t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual) + } + + invalidPattern := digest.Digest(id1).Hex()[1:6] + _, err = is.Search(invalidPattern) + if err == nil { + t.Fatalf("expected search for %q to fail", invalidPattern) + } + +} + +func TestAddDelete(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "images-fs-store") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + fs, err := NewFSStoreBackend(tmpdir) + if err != nil { + t.Fatal(err) + } + + is, err := NewImageStore(fs, &mockLayerGetReleaser{}) + if err != nil { + t.Fatal(err) + } + + id1, err := is.Create([]byte(`{"comment": "abc", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`)) + if err != nil { + t.Fatal(err) + } + + if actual, expected := id1, ID("sha256:8d25a9c45df515f9d0fe8e4a6b1c64dd3b965a84790ddbcc7954bb9bc89eb993"); actual != expected { + t.Fatalf("create ID mismatch: expected %q, got %q", expected, actual) + } + + img, err := is.Get(id1) + if err != nil { + t.Fatal(err) + } + + if actual, expected := img.Comment, "abc"; actual != expected { + t.Fatalf("invalid comment in image: expected %q, got %q", expected, actual) + } + + id2, err := is.Create([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`)) + if err != nil { + t.Fatal(err) + } + + err = is.SetParent(id2, id1) + if err != nil { + t.Fatal(err) + } + + pid1, err := is.GetParent(id2) + if err != nil { + t.Fatal(err) + } + if actual, expected := pid1, id1; actual != expected { + t.Fatalf("invalid parent for image: expected %q, got %q", expected, actual) + } + + _, err = is.Delete(id1) + if err != nil { + t.Fatal(err) + } + _, err = is.Get(id1) + if err == nil { + t.Fatalf("expected get for deleted image %q to fail", id1) + } + _, err = is.Get(id2) + if err != nil { + t.Fatal(err) + } + pid1, err = is.GetParent(id2) + if err == nil { + t.Fatalf("expected parent check for image %q to fail, got %q", id2, pid1) + } + +} + +type mockLayerGetReleaser struct{} + +func (ls *mockLayerGetReleaser) Get(layer.ChainID) (layer.Layer, error) { + return nil, nil +} + +func (ls *mockLayerGetReleaser) Release(layer.Layer) ([]layer.Metadata, error) { + return nil, nil +} diff --git a/image/tarexport/load.go b/image/tarexport/load.go new file mode 100644 index 0000000000..5d5507e0f3 --- /dev/null +++ b/image/tarexport/load.go @@ -0,0 +1,284 @@ +package tarexport + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" + "github.com/docker/docker/image" + "github.com/docker/docker/image/v1" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" + "github.com/docker/docker/pkg/symlink" +) + +func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error { + tmpDir, err := ioutil.TempDir("", "docker-import-") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil { + return err + } + // read manifest, if no file then load in legacy mode + manifestPath, err := safePath(tmpDir, manifestFileName) + if err != nil { + return err + } + manifestFile, err := os.Open(manifestPath) + if err != nil { + if os.IsNotExist(err) { + return l.legacyLoad(tmpDir, outStream) + } + return manifestFile.Close() + } + defer manifestFile.Close() + + var manifest []manifestItem + if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil { + return err + } + + for _, m := range manifest { + configPath, err := safePath(tmpDir, m.Config) + if err != nil { + return err + } + config, err := ioutil.ReadFile(configPath) + if err != nil { + return err + } + img, err := image.NewFromJSON(config) + if err != nil { + return err + } + var rootFS image.RootFS + rootFS = *img.RootFS + rootFS.DiffIDs = nil + + if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual { + return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual) + } + + for i, diffID := range img.RootFS.DiffIDs { + layerPath, err := safePath(tmpDir, m.Layers[i]) + if err != nil { + return err + } + newLayer, err := l.loadLayer(layerPath, rootFS) + if err != nil { + return err + } + defer layer.ReleaseAndLog(l.ls, newLayer) + if expected, actual := diffID, newLayer.DiffID(); expected != actual { + return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual) + } + rootFS.Append(diffID) + } + + imgID, err := l.is.Create(config) + if err != nil { + return err + } + + for _, repoTag := range m.RepoTags { + named, err := reference.ParseNamed(repoTag) + if err != nil { + return err + } + ref, ok := named.(reference.NamedTagged) + if !ok { + return fmt.Errorf("invalid tag %q", repoTag) + } + l.setLoadedTag(ref, imgID, outStream) + } + + } + + return nil +} + +func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) { + rawTar, err := os.Open(filename) + if err != nil { + logrus.Debugf("Error reading embedded tar: %v", err) + return nil, err + } + inflatedLayerData, err := archive.DecompressStream(rawTar) + if err != nil { + return nil, err + } + + defer rawTar.Close() + defer inflatedLayerData.Close() + + return l.ls.Register(inflatedLayerData, rootFS.ChainID()) +} + +func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error { + if prevID, err := l.ts.Get(ref); err == nil && prevID != imgID { + fmt.Fprintf(outStream, "The image %s already exists, renaming the old one with ID %s to empty string\n", ref.String(), string(prevID)) // todo: this message is wrong in case of multiple tags + } + + if err := l.ts.Add(ref, imgID, true); err != nil { + return err + } + return nil +} + +func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer) error { + legacyLoadedMap := make(map[string]image.ID) + + dirs, err := ioutil.ReadDir(tmpDir) + if err != nil { + return err + } + + // every dir represents an image + for _, d := range dirs { + if d.IsDir() { + if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap); err != nil { + return err + } + } + } + + // load tags from repositories file + repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName) + if err != nil { + return err + } + repositoriesFile, err := os.Open(repositoriesPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + return repositoriesFile.Close() + } + defer repositoriesFile.Close() + + repositories := make(map[string]map[string]string) + if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil { + return err + } + + for name, tagMap := range repositories { + for tag, oldID := range tagMap { + imgID, ok := legacyLoadedMap[oldID] + if !ok { + return fmt.Errorf("invalid target ID: %v", oldID) + } + named, err := reference.WithName(name) + if err != nil { + return err + } + ref, err := reference.WithTag(named, tag) + if err != nil { + return err + } + l.setLoadedTag(ref, imgID, outStream) + } + } + + return nil +} + +func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[string]image.ID) error { + if _, loaded := loadedMap[oldID]; loaded { + return nil + } + configPath, err := safePath(sourceDir, filepath.Join(oldID, legacyConfigFileName)) + if err != nil { + return err + } + imageJSON, err := ioutil.ReadFile(configPath) + if err != nil { + logrus.Debugf("Error reading json: %v", err) + return err + } + + var img struct{ Parent string } + if err := json.Unmarshal(imageJSON, &img); err != nil { + return err + } + + var parentID image.ID + if img.Parent != "" { + for { + var loaded bool + if parentID, loaded = loadedMap[img.Parent]; !loaded { + if err := l.legacyLoadImage(img.Parent, sourceDir, loadedMap); err != nil { + return err + } + } else { + break + } + } + } + + // todo: try to connect with migrate code + rootFS := image.NewRootFS() + var history []image.History + + if parentID != "" { + parentImg, err := l.is.Get(parentID) + if err != nil { + return err + } + + rootFS = parentImg.RootFS + history = parentImg.History + } + + layerPath, err := safePath(sourceDir, filepath.Join(oldID, legacyLayerFileName)) + if err != nil { + return err + } + newLayer, err := l.loadLayer(layerPath, *rootFS) + if err != nil { + return err + } + rootFS.Append(newLayer.DiffID()) + + h, err := v1.HistoryFromConfig(imageJSON, false) + if err != nil { + return err + } + history = append(history, h) + + config, err := v1.MakeConfigFromV1Config(imageJSON, rootFS, history) + if err != nil { + return err + } + imgID, err := l.is.Create(config) + if err != nil { + return err + } + + metadata, err := l.ls.Release(newLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + return err + } + + if parentID != "" { + if err := l.is.SetParent(imgID, parentID); err != nil { + return err + } + } + + loadedMap[oldID] = imgID + return nil +} + +func safePath(base, path string) (string, error) { + return symlink.FollowSymlinkInScope(filepath.Join(base, path), base) +} diff --git a/image/tarexport/save.go b/image/tarexport/save.go new file mode 100644 index 0000000000..1019e29244 --- /dev/null +++ b/image/tarexport/save.go @@ -0,0 +1,303 @@ +package tarexport + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" + "github.com/docker/docker/image" + "github.com/docker/docker/image/v1" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/registry" + "github.com/docker/docker/tag" +) + +type imageDescriptor struct { + refs []reference.NamedTagged + layers []string +} + +type saveSession struct { + *tarexporter + outDir string + images map[image.ID]*imageDescriptor + savedLayers map[string]struct{} +} + +func (l *tarexporter) Save(names []string, outStream io.Writer) error { + images, err := l.parseNames(names) + if err != nil { + return err + } + + return (&saveSession{tarexporter: l, images: images}).save(outStream) +} + +func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor, error) { + imgDescr := make(map[image.ID]*imageDescriptor) + + addAssoc := func(id image.ID, ref reference.Named) { + if _, ok := imgDescr[id]; !ok { + imgDescr[id] = &imageDescriptor{} + } + + if ref != nil { + var tagged reference.NamedTagged + if _, ok := ref.(reference.Digested); ok { + return + } + var ok bool + if tagged, ok = ref.(reference.NamedTagged); !ok { + var err error + if tagged, err = reference.WithTag(ref, tag.DefaultTag); err != nil { + return + } + } + + for _, t := range imgDescr[id].refs { + if tagged.String() == t.String() { + return + } + } + imgDescr[id].refs = append(imgDescr[id].refs, tagged) + } + } + + for _, name := range names { + ref, err := reference.ParseNamed(name) + if err != nil { + return nil, err + } + ref = registry.NormalizeLocalReference(ref) + if ref.Name() == string(digest.Canonical) { + imgID, err := l.is.Search(name) + if err != nil { + return nil, err + } + addAssoc(imgID, nil) + continue + } + if _, ok := ref.(reference.Digested); !ok { + if _, ok := ref.(reference.NamedTagged); !ok { + assocs := l.ts.ReferencesByName(ref) + for _, assoc := range assocs { + addAssoc(assoc.ImageID, assoc.Ref) + } + if len(assocs) == 0 { + imgID, err := l.is.Search(name) + if err != nil { + return nil, err + } + addAssoc(imgID, nil) + } + continue + } + } + var imgID image.ID + if imgID, err = l.ts.Get(ref); err != nil { + return nil, err + } + addAssoc(imgID, ref) + + } + return imgDescr, nil +} + +func (s *saveSession) save(outStream io.Writer) error { + s.savedLayers = make(map[string]struct{}) + + // get image json + tempDir, err := ioutil.TempDir("", "docker-export-") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + s.outDir = tempDir + reposLegacy := make(map[string]map[string]string) + + var manifest []manifestItem + + for id, imageDescr := range s.images { + if err = s.saveImage(id); err != nil { + return err + } + + var repoTags []string + var layers []string + + for _, ref := range imageDescr.refs { + if _, ok := reposLegacy[ref.Name()]; !ok { + reposLegacy[ref.Name()] = make(map[string]string) + } + reposLegacy[ref.Name()][ref.Tag()] = imageDescr.layers[len(imageDescr.layers)-1] + repoTags = append(repoTags, ref.String()) + } + + for _, l := range imageDescr.layers { + layers = append(layers, filepath.Join(l, legacyLayerFileName)) + } + + manifest = append(manifest, manifestItem{ + Config: digest.Digest(id).Hex() + ".json", + RepoTags: repoTags, + Layers: layers, + }) + } + + if len(reposLegacy) > 0 { + reposFile := filepath.Join(tempDir, legacyRepositoriesFileName) + f, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + f.Close() + return err + } + if err := json.NewEncoder(f).Encode(reposLegacy); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil { + return err + } + } + + manifestFileName := filepath.Join(tempDir, manifestFileName) + f, err := os.OpenFile(manifestFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + f.Close() + return err + } + if err := json.NewEncoder(f).Encode(manifest); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Chtimes(manifestFileName, time.Unix(0, 0), time.Unix(0, 0)); err != nil { + return err + } + + fs, err := archive.Tar(tempDir, archive.Uncompressed) + if err != nil { + return err + } + defer fs.Close() + + if _, err := io.Copy(outStream, fs); err != nil { + return err + } + return nil +} + +func (s *saveSession) saveImage(id image.ID) error { + img, err := s.is.Get(id) + if err != nil { + return err + } + + if len(img.RootFS.DiffIDs) == 0 { + return fmt.Errorf("empty export - not implemented") + } + + var parent digest.Digest + var layers []string + for i := range img.RootFS.DiffIDs { + v1Img := image.V1Image{} + if i == len(img.RootFS.DiffIDs)-1 { + v1Img = img.V1Image + } + rootFS := *img.RootFS + rootFS.DiffIDs = rootFS.DiffIDs[:i+1] + v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent) + if err != nil { + return err + } + + v1Img.ID = v1ID.Hex() + if parent != "" { + v1Img.Parent = parent.Hex() + } + + if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil { + return err + } + layers = append(layers, v1Img.ID) + parent = v1ID + } + + configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json") + if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil { + return err + } + if err := os.Chtimes(configFile, img.Created, img.Created); err != nil { + return err + } + + s.images[id].layers = layers + return nil +} + +func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error { + if _, exists := s.savedLayers[legacyImg.ID]; exists { + return nil + } + + outDir := filepath.Join(s.outDir, legacyImg.ID) + if err := os.Mkdir(outDir, 0755); err != nil { + return err + } + + // todo: why is this version file here? + if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil { + return err + } + + imageConfig, err := json.Marshal(legacyImg) + if err != nil { + return err + } + + if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil { + return err + } + + // serialize filesystem + tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName)) + if err != nil { + return err + } + defer tarFile.Close() + + l, err := s.ls.Get(id) + if err != nil { + return err + } + defer layer.ReleaseAndLog(s.ls, l) + + arch, err := l.TarStream() + if err != nil { + return err + } + if _, err := io.Copy(tarFile, arch); err != nil { + return err + } + + for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} { + // todo: maybe save layer created timestamp? + if err := os.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil { + return err + } + } + + s.savedLayers[legacyImg.ID] = struct{}{} + return nil +} diff --git a/image/tarexport/tarexport.go b/image/tarexport/tarexport.go new file mode 100644 index 0000000000..3369d665c9 --- /dev/null +++ b/image/tarexport/tarexport.go @@ -0,0 +1,36 @@ +package tarexport + +import ( + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/tag" +) + +const ( + manifestFileName = "manifest.json" + legacyLayerFileName = "layer.tar" + legacyConfigFileName = "json" + legacyVersionFileName = "VERSION" + legacyRepositoriesFileName = "repositories" +) + +type manifestItem struct { + Config string + RepoTags []string + Layers []string +} + +type tarexporter struct { + is image.Store + ls layer.Store + ts tag.Store +} + +// NewTarExporter returns new ImageExporter for tar packages +func NewTarExporter(is image.Store, ls layer.Store, ts tag.Store) image.Exporter { + return &tarexporter{ + is: is, + ls: ls, + ts: ts, + } +} diff --git a/image/v1/imagev1.go b/image/v1/imagev1.go new file mode 100644 index 0000000000..4a67c017b3 --- /dev/null +++ b/image/v1/imagev1.go @@ -0,0 +1,148 @@ +package v1 + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/version" +) + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +// noFallbackMinVersion is the minimum version for which v1compatibility +// information will not be marshaled through the Image struct to remove +// blank fields. +var noFallbackMinVersion = version.Version("1.8.3") + +// HistoryFromConfig creates a History struct from v1 configuration JSON +func HistoryFromConfig(imageJSON []byte, emptyLayer bool) (image.History, error) { + h := image.History{} + var v1Image image.V1Image + if err := json.Unmarshal(imageJSON, &v1Image); err != nil { + return h, err + } + + return image.History{ + Author: v1Image.Author, + Created: v1Image.Created, + CreatedBy: strings.Join(v1Image.ContainerConfig.Cmd.Slice(), " "), + Comment: v1Image.Comment, + EmptyLayer: emptyLayer, + }, nil +} + +// CreateID creates an ID from v1 image, layerID and parent ID. +// Used for backwards compatibility with old clients. +func CreateID(v1Image image.V1Image, layerID layer.ChainID, parent digest.Digest) (digest.Digest, error) { + v1Image.ID = "" + v1JSON, err := json.Marshal(v1Image) + if err != nil { + return "", err + } + + var config map[string]*json.RawMessage + if err := json.Unmarshal(v1JSON, &config); err != nil { + return "", err + } + + // FIXME: note that this is slightly incompatible with RootFS logic + config["layer_id"] = rawJSON(layerID) + if parent != "" { + config["parent"] = rawJSON(parent) + } + + configJSON, err := json.Marshal(config) + if err != nil { + return "", err + } + logrus.Debugf("CreateV1ID %s", configJSON) + + return digest.FromBytes(configJSON) +} + +// MakeConfigFromV1Config creates an image config from the legacy V1 config format. +func MakeConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []image.History) ([]byte, error) { + var dver struct { + DockerVersion string `json:"docker_version"` + } + + if err := json.Unmarshal(imageJSON, &dver); err != nil { + return nil, err + } + + useFallback := version.Version(dver.DockerVersion).LessThan(noFallbackMinVersion) + + if useFallback { + var v1Image image.V1Image + err := json.Unmarshal(imageJSON, &v1Image) + if err != nil { + return nil, err + } + imageJSON, err = json.Marshal(v1Image) + if err != nil { + return nil, err + } + } + + var c map[string]*json.RawMessage + if err := json.Unmarshal(imageJSON, &c); err != nil { + return nil, err + } + + delete(c, "id") + delete(c, "parent") + delete(c, "Size") // Size is calculated from data on disk and is inconsitent + delete(c, "parent_id") + delete(c, "layer_id") + delete(c, "throwaway") + + c["rootfs"] = rawJSON(rootfs) + c["history"] = rawJSON(history) + + return json.Marshal(c) +} + +// MakeV1ConfigFromConfig creates an legacy V1 image config from an Image struct +func MakeV1ConfigFromConfig(img *image.Image, v1ID, parentV1ID string, throwaway bool) ([]byte, error) { + // Top-level v1compatibility string should be a modified version of the + // image config. + var configAsMap map[string]*json.RawMessage + if err := json.Unmarshal(img.RawJSON(), &configAsMap); err != nil { + return nil, err + } + + // Delete fields that didn't exist in old manifest + delete(configAsMap, "rootfs") + delete(configAsMap, "history") + configAsMap["id"] = rawJSON(v1ID) + if parentV1ID != "" { + configAsMap["parent"] = rawJSON(parentV1ID) + } + if throwaway { + configAsMap["throwaway"] = rawJSON(true) + } + + return json.Marshal(configAsMap) +} + +func rawJSON(value interface{}) *json.RawMessage { + jsonval, err := json.Marshal(value) + if err != nil { + return nil + } + return (*json.RawMessage)(&jsonval) +} + +// ValidateID checks whether an ID string is a valid image ID. +func ValidateID(id string) error { + if ok := validHex.MatchString(id); !ok { + return fmt.Errorf("image ID '%s' is invalid ", id) + } + return nil +}