mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
50a498ea1c
This adds verification for getting layer data out
of layerstore. These failures should only be possible
if layer metadata files have been manually changed
of if something is wrong with tar-split algorithm.
Failing early makes sure we don’t upload invalid data
to the registries where it would fail after someone
tries to pull it.
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit e29e580f7f)
435 lines
10 KiB
Go
435 lines
10 KiB
Go
package layer
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/docker/docker/daemon/graphdriver"
|
|
"github.com/docker/docker/pkg/archive"
|
|
"github.com/docker/docker/pkg/stringid"
|
|
"github.com/vbatts/tar-split/tar/asm"
|
|
"github.com/vbatts/tar-split/tar/storage"
|
|
)
|
|
|
|
func writeTarSplitFile(name string, tarContent []byte) error {
|
|
f, err := os.OpenFile(name, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
fz := gzip.NewWriter(f)
|
|
|
|
metaPacker := storage.NewJSONPacker(fz)
|
|
defer fz.Close()
|
|
|
|
rdr, err := asm.NewInputTarStream(bytes.NewReader(tarContent), metaPacker, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := io.Copy(ioutil.Discard, rdr); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func TestLayerMigration(t *testing.T) {
|
|
td, err := ioutil.TempDir("", "migration-test-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(td)
|
|
|
|
layer1Files := []FileApplier{
|
|
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
|
|
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
|
|
}
|
|
|
|
layer2Files := []FileApplier{
|
|
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
|
|
}
|
|
|
|
tar1, err := tarFromFiles(layer1Files...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tar2, err := tarFromFiles(layer2Files...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
graphID1 := stringid.GenerateRandomID()
|
|
if err := graph.Create(graphID1, "", ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := graph.ApplyDiff(graphID1, "", archive.Reader(bytes.NewReader(tar1))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tf1 := filepath.Join(td, "tar1.json.gz")
|
|
if err := writeTarSplitFile(tf1, tar1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ls, err := NewStoreFromGraphDriver(fms, graph)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
newTarDataPath := filepath.Join(td, ".migration-tardata")
|
|
diffID, size, err := ls.(*layerStore).ChecksumForGraphID(graphID1, "", tf1, newTarDataPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", diffID, newTarDataPath, size)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertReferences(t, layer1a, layer1b)
|
|
// Attempt register, should be same
|
|
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
graphID2 := stringid.GenerateRandomID()
|
|
if err := graph.Create(graphID2, graphID1, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := graph.ApplyDiff(graphID2, graphID1, archive.Reader(bytes.NewReader(tar2))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tf2 := filepath.Join(td, "tar2.json.gz")
|
|
if err := writeTarSplitFile(tf2, tar2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
diffID, size, err = ls.(*layerStore).ChecksumForGraphID(graphID2, graphID1, tf2, newTarDataPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), diffID, tf2, size)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertReferences(t, layer2a, layer2b)
|
|
|
|
if metadata, err := ls.Release(layer2a); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(metadata) > 0 {
|
|
t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
|
|
}
|
|
|
|
metadata, err := ls.Release(layer2b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertMetadata(t, metadata, createMetadata(layer2a))
|
|
}
|
|
|
|
func tarFromFilesInGraph(graph graphdriver.Driver, graphID, parentID string, files ...FileApplier) ([]byte, error) {
|
|
t, err := tarFromFiles(files...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := graph.Create(graphID, parentID, ""); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := graph.ApplyDiff(graphID, parentID, archive.Reader(bytes.NewReader(t))); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ar, err := graph.Diff(graphID, parentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer ar.Close()
|
|
|
|
return ioutil.ReadAll(ar)
|
|
}
|
|
|
|
func TestLayerMigrationNoTarsplit(t *testing.T) {
|
|
td, err := ioutil.TempDir("", "migration-test-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(td)
|
|
|
|
layer1Files := []FileApplier{
|
|
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
|
|
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
|
|
}
|
|
|
|
layer2Files := []FileApplier{
|
|
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
|
|
}
|
|
|
|
graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
graphID1 := stringid.GenerateRandomID()
|
|
graphID2 := stringid.GenerateRandomID()
|
|
|
|
tar1, err := tarFromFilesInGraph(graph, graphID1, "", layer1Files...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tar2, err := tarFromFilesInGraph(graph, graphID2, graphID1, layer2Files...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ls, err := NewStoreFromGraphDriver(fms, graph)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
newTarDataPath := filepath.Join(td, ".migration-tardata")
|
|
diffID, size, err := ls.(*layerStore).ChecksumForGraphID(graphID1, "", "", newTarDataPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", diffID, newTarDataPath, size)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertReferences(t, layer1a, layer1b)
|
|
|
|
// Attempt register, should be same
|
|
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
diffID, size, err = ls.(*layerStore).ChecksumForGraphID(graphID2, graphID1, "", newTarDataPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), diffID, newTarDataPath, size)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertReferences(t, layer2a, layer2b)
|
|
|
|
if metadata, err := ls.Release(layer2a); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(metadata) > 0 {
|
|
t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
|
|
}
|
|
|
|
metadata, err := ls.Release(layer2b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertMetadata(t, metadata, createMetadata(layer2a))
|
|
}
|
|
|
|
func TestMountMigration(t *testing.T) {
|
|
ls, _, cleanup := newTestStore(t)
|
|
defer cleanup()
|
|
|
|
baseFiles := []FileApplier{
|
|
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
|
|
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
|
|
}
|
|
initFiles := []FileApplier{
|
|
newTestFile("/etc/hosts", []byte{}, 0644),
|
|
newTestFile("/etc/resolv.conf", []byte{}, 0644),
|
|
}
|
|
mountFiles := []FileApplier{
|
|
newTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644),
|
|
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
|
|
newTestFile("/root/testfile1.txt", []byte("nothing valuable"), 0644),
|
|
}
|
|
|
|
initTar, err := tarFromFiles(initFiles...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
mountTar, err := tarFromFiles(mountFiles...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
graph := ls.(*layerStore).driver
|
|
|
|
layer1, err := createLayer(ls, "", initWithFiles(baseFiles...))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
graphID1 := layer1.(*referencedCacheLayer).cacheID
|
|
|
|
containerID := stringid.GenerateRandomID()
|
|
containerInit := fmt.Sprintf("%s-init", containerID)
|
|
|
|
if err := graph.Create(containerInit, graphID1, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := graph.ApplyDiff(containerInit, graphID1, archive.Reader(bytes.NewReader(initTar))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := graph.Create(containerID, containerInit, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := graph.ApplyDiff(containerID, containerInit, archive.Reader(bytes.NewReader(mountTar))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := ls.(*layerStore).CreateRWLayerByGraphID("migration-mount", containerID, layer1.ChainID()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rwLayer1, err := ls.GetRWLayer("migration-mount")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if _, err := rwLayer1.Mount(""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
changes, err := rwLayer1.Changes()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if expected := 5; len(changes) != expected {
|
|
t.Logf("Changes %#v", changes)
|
|
t.Fatalf("Wrong number of changes %d, expected %d", len(changes), expected)
|
|
}
|
|
|
|
sortChanges(changes)
|
|
|
|
assertChange(t, changes[0], archive.Change{
|
|
Path: "/etc",
|
|
Kind: archive.ChangeModify,
|
|
})
|
|
assertChange(t, changes[1], archive.Change{
|
|
Path: "/etc/hosts",
|
|
Kind: archive.ChangeModify,
|
|
})
|
|
assertChange(t, changes[2], archive.Change{
|
|
Path: "/root",
|
|
Kind: archive.ChangeModify,
|
|
})
|
|
assertChange(t, changes[3], archive.Change{
|
|
Path: "/root/.bashrc",
|
|
Kind: archive.ChangeModify,
|
|
})
|
|
assertChange(t, changes[4], archive.Change{
|
|
Path: "/root/testfile1.txt",
|
|
Kind: archive.ChangeAdd,
|
|
})
|
|
|
|
assertActivityCount(t, rwLayer1, 1)
|
|
|
|
if _, err := ls.CreateRWLayer("migration-mount", layer1.ChainID(), "", nil); err == nil {
|
|
t.Fatal("Expected error creating mount with same name")
|
|
} else if err != ErrMountNameConflict {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rwLayer2, err := ls.GetRWLayer("migration-mount")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if getMountLayer(rwLayer1) != getMountLayer(rwLayer2) {
|
|
t.Fatal("Expected same layer from get with same name as from migrate")
|
|
}
|
|
|
|
if _, err := rwLayer2.Mount(""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertActivityCount(t, rwLayer2, 1)
|
|
assertActivityCount(t, rwLayer1, 1)
|
|
|
|
if _, err := rwLayer2.Mount(""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertActivityCount(t, rwLayer2, 2)
|
|
assertActivityCount(t, rwLayer1, 1)
|
|
|
|
if metadata, err := ls.Release(layer1); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(metadata) > 0 {
|
|
t.Fatalf("Expected no layers to be deleted, deleted %#v", metadata)
|
|
}
|
|
|
|
if err := rwLayer1.Unmount(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertActivityCount(t, rwLayer2, 2)
|
|
assertActivityCount(t, rwLayer1, 0)
|
|
|
|
if _, err := ls.ReleaseRWLayer(rwLayer1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := rwLayer2.Unmount(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := ls.ReleaseRWLayer(rwLayer2); err == nil {
|
|
t.Fatal("Expected error deleting active mount")
|
|
}
|
|
if err := rwLayer2.Unmount(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
metadata, err := ls.ReleaseRWLayer(rwLayer2)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(metadata) == 0 {
|
|
t.Fatal("Expected base layer to be deleted when deleting mount")
|
|
}
|
|
|
|
assertMetadata(t, metadata, createMetadata(layer1))
|
|
}
|