From 21ae66c664b439052eb8d90d3a0e2582564bbf89 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 17 Jun 2019 21:50:31 +0000 Subject: [PATCH] Revert "Remove Schema1 integration test suite" This reverts commit 13b7d11be1a97ebe82b3a0f0b5db9b873a481637. Signed-off-by: Tibor Vass (cherry picked from commit f23a51a8603a92b124e5037106958baf8c70c5e1) Signed-off-by: Tibor Vass --- Dockerfile | 12 ++ integration-cli/check_test.go | 33 ++++ integration-cli/docker_cli_by_digest_test.go | 143 ++++++++++++++++++ integration-cli/docker_cli_daemon_test.go | 74 +++++++++ integration-cli/docker_cli_pull_local_test.go | 24 +++ integration-cli/docker_cli_push_test.go | 57 +++++++ integration/system/uuid_test.go | 23 --- 7 files changed, 343 insertions(+), 23 deletions(-) delete mode 100644 integration/system/uuid_test.go diff --git a/Dockerfile b/Dockerfile index 1dad41e2cc..d563fabdac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,11 @@ RUN apt-get update && apt-get install -y \ && make PREFIX=/build/ install-criu FROM base AS registry +# Install two versions of the registry. The first is an older version that +# only supports schema1 manifests. The second is a newer version that supports +# both. This allows integration-cli tests to cover push/pull with both schema1 +# and schema2 manifests. +ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd ENV REGISTRY_COMMIT 47a064d4195a9b56133891bbb13620c3ac83a827 RUN set -x \ && export GOPATH="$(mktemp -d)" \ @@ -58,6 +63,13 @@ RUN set -x \ && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \ && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ go build -buildmode=pie -o /build/registry-v2 github.com/docker/distribution/cmd/registry \ + && case $(dpkg --print-architecture) in \ + amd64|ppc64*|s390x) \ + (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1"); \ + GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH"; \ + go build -buildmode=pie -o /build/registry-v2-schema1 github.com/docker/distribution/cmd/registry; \ + ;; \ + esac \ && rm -rf "$GOPATH" diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index c344b63bd3..2a0b1a25d5 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -141,6 +141,39 @@ func (s *DockerRegistrySuite) TearDownTest(c *check.C) { s.ds.TearDownTest(c) } +func init() { + check.Suite(&DockerSchema1RegistrySuite{ + ds: &DockerSuite{}, + }) +} + +type DockerSchema1RegistrySuite struct { + ds *DockerSuite + reg *registry.V2 + d *daemon.Daemon +} + +func (s *DockerSchema1RegistrySuite) OnTimeout(c *check.C) { + s.d.DumpStackAndQuit() +} + +func (s *DockerSchema1RegistrySuite) SetUpTest(c *check.C) { + testRequires(c, DaemonIsLinux, RegistryHosting, NotArm64, testEnv.IsLocalDaemon) + s.reg = registry.NewV2(c, registry.Schema1) + s.reg.WaitReady(c) + s.d = daemon.New(c, dockerBinary, dockerdBinary, testdaemon.WithEnvironment(testEnv.Execution)) +} + +func (s *DockerSchema1RegistrySuite) TearDownTest(c *check.C) { + if s.reg != nil { + s.reg.Close() + } + if s.d != nil { + s.d.Stop(c) + } + s.ds.TearDownTest(c) +} + func init() { check.Suite(&DockerRegistryAuthHtpasswdSuite{ ds: &DockerSuite{}, diff --git a/integration-cli/docker_cli_by_digest_test.go b/integration-cli/docker_cli_by_digest_test.go index dd5549c2bd..c884d31751 100644 --- a/integration-cli/docker_cli_by_digest_test.go +++ b/integration-cli/docker_cli_by_digest_test.go @@ -3,9 +3,12 @@ package main import ( "encoding/json" "fmt" + "os" + "path/filepath" "regexp" "strings" + "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/docker/docker/api/types" "github.com/docker/docker/integration-cli/checker" @@ -77,6 +80,10 @@ func (s *DockerRegistrySuite) TestPullByTagDisplaysDigest(c *check.C) { testPullByTagDisplaysDigest(c) } +func (s *DockerSchema1RegistrySuite) TestPullByTagDisplaysDigest(c *check.C) { + testPullByTagDisplaysDigest(c) +} + func testPullByDigest(c *check.C) { testRequires(c, DaemonIsLinux) pushDigest, err := setupImage(c) @@ -99,6 +106,10 @@ func (s *DockerRegistrySuite) TestPullByDigest(c *check.C) { testPullByDigest(c) } +func (s *DockerSchema1RegistrySuite) TestPullByDigest(c *check.C) { + testPullByDigest(c) +} + func testPullByDigestNoFallback(c *check.C) { testRequires(c, DaemonIsLinux) // pull from the registry using the @ reference @@ -112,6 +123,10 @@ func (s *DockerRegistrySuite) TestPullByDigestNoFallback(c *check.C) { testPullByDigestNoFallback(c) } +func (s *DockerSchema1RegistrySuite) TestPullByDigestNoFallback(c *check.C) { + testPullByDigestNoFallback(c) +} + func (s *DockerRegistrySuite) TestCreateByDigest(c *check.C) { pushDigest, err := setupImage(c) assert.NilError(c, err, "error setting up image") @@ -546,3 +561,131 @@ func (s *DockerRegistrySuite) TestPullFailsWithAlteredManifest(c *check.C) { expectedErrorMsg := fmt.Sprintf("manifest verification failed for digest %s", manifestDigest) assert.Assert(c, is.Contains(out, expectedErrorMsg)) } + +// TestPullFailsWithAlteredManifest tests that a `docker pull` fails when +// we have modified a manifest blob and its digest cannot be verified. +// This is the schema1 version of the test. +func (s *DockerSchema1RegistrySuite) TestPullFailsWithAlteredManifest(c *check.C) { + testRequires(c, DaemonIsLinux) + manifestDigest, err := setupImage(c) + c.Assert(err, checker.IsNil, check.Commentf("error setting up image")) + + // Load the target manifest blob. + manifestBlob := s.reg.ReadBlobContents(c, manifestDigest) + + var imgManifest schema1.Manifest + err = json.Unmarshal(manifestBlob, &imgManifest) + c.Assert(err, checker.IsNil, check.Commentf("unable to decode image manifest from blob")) + + // Change a layer in the manifest. + imgManifest.FSLayers[0] = schema1.FSLayer{ + BlobSum: digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + } + + // Move the existing data file aside, so that we can replace it with a + // malicious blob of data. NOTE: we defer the returned undo func. + undo := s.reg.TempMoveBlobData(c, manifestDigest) + defer undo() + + alteredManifestBlob, err := json.MarshalIndent(imgManifest, "", " ") + c.Assert(err, checker.IsNil, check.Commentf("unable to encode altered image manifest to JSON")) + + s.reg.WriteBlobContents(c, manifestDigest, alteredManifestBlob) + + // Now try pulling that image by digest. We should get an error about + // digest verification for the manifest digest. + + // Pull from the registry using the @ reference. + imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest) + out, exitStatus, _ := dockerCmdWithError("pull", imageReference) + c.Assert(exitStatus, checker.Not(check.Equals), 0) + + expectedErrorMsg := fmt.Sprintf("image verification failed for digest %s", manifestDigest) + c.Assert(out, checker.Contains, expectedErrorMsg) +} + +// TestPullFailsWithAlteredLayer tests that a `docker pull` fails when +// we have modified a layer blob and its digest cannot be verified. +// This is the schema2 version of the test. +func (s *DockerRegistrySuite) TestPullFailsWithAlteredLayer(c *check.C) { + testRequires(c, DaemonIsLinux) + manifestDigest, err := setupImage(c) + c.Assert(err, checker.IsNil) + + // Load the target manifest blob. + manifestBlob := s.reg.ReadBlobContents(c, manifestDigest) + + var imgManifest schema2.Manifest + err = json.Unmarshal(manifestBlob, &imgManifest) + c.Assert(err, checker.IsNil) + + // Next, get the digest of one of the layers from the manifest. + targetLayerDigest := imgManifest.Layers[0].Digest + + // Move the existing data file aside, so that we can replace it with a + // malicious blob of data. NOTE: we defer the returned undo func. + undo := s.reg.TempMoveBlobData(c, targetLayerDigest) + defer undo() + + // Now make a fake data blob in this directory. + s.reg.WriteBlobContents(c, targetLayerDigest, []byte("This is not the data you are looking for.")) + + // Now try pulling that image by digest. We should get an error about + // digest verification for the target layer digest. + + // Remove distribution cache to force a re-pull of the blobs + if err := os.RemoveAll(filepath.Join(testEnv.DaemonInfo.DockerRootDir, "image", s.d.StorageDriver(), "distribution")); err != nil { + c.Fatalf("error clearing distribution cache: %v", err) + } + + // Pull from the registry using the @ reference. + imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest) + out, exitStatus, _ := dockerCmdWithError("pull", imageReference) + c.Assert(exitStatus, checker.Not(check.Equals), 0, check.Commentf("expected a non-zero exit status")) + + expectedErrorMsg := fmt.Sprintf("filesystem layer verification failed for digest %s", targetLayerDigest) + c.Assert(out, checker.Contains, expectedErrorMsg, check.Commentf("expected error message in output: %s", out)) +} + +// TestPullFailsWithAlteredLayer tests that a `docker pull` fails when +// we have modified a layer blob and its digest cannot be verified. +// This is the schema1 version of the test. +func (s *DockerSchema1RegistrySuite) TestPullFailsWithAlteredLayer(c *check.C) { + testRequires(c, DaemonIsLinux) + manifestDigest, err := setupImage(c) + c.Assert(err, checker.IsNil) + + // Load the target manifest blob. + manifestBlob := s.reg.ReadBlobContents(c, manifestDigest) + + var imgManifest schema1.Manifest + err = json.Unmarshal(manifestBlob, &imgManifest) + c.Assert(err, checker.IsNil) + + // Next, get the digest of one of the layers from the manifest. + targetLayerDigest := imgManifest.FSLayers[0].BlobSum + + // Move the existing data file aside, so that we can replace it with a + // malicious blob of data. NOTE: we defer the returned undo func. + undo := s.reg.TempMoveBlobData(c, targetLayerDigest) + defer undo() + + // Now make a fake data blob in this directory. + s.reg.WriteBlobContents(c, targetLayerDigest, []byte("This is not the data you are looking for.")) + + // Now try pulling that image by digest. We should get an error about + // digest verification for the target layer digest. + + // Remove distribution cache to force a re-pull of the blobs + if err := os.RemoveAll(filepath.Join(testEnv.DaemonInfo.DockerRootDir, "image", s.d.StorageDriver(), "distribution")); err != nil { + c.Fatalf("error clearing distribution cache: %v", err) + } + + // Pull from the registry using the @ reference. + imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest) + out, exitStatus, _ := dockerCmdWithError("pull", imageReference) + c.Assert(exitStatus, checker.Not(check.Equals), 0, check.Commentf("expected a non-zero exit status")) + + expectedErrorMsg := fmt.Sprintf("filesystem layer verification failed for digest %s", targetLayerDigest) + c.Assert(out, checker.Contains, expectedErrorMsg, check.Commentf("expected error message in output: %s", out)) +} diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go index c9e88fa58a..9d4938993d 100644 --- a/integration-cli/docker_cli_daemon_test.go +++ b/integration-cli/docker_cli_daemon_test.go @@ -18,6 +18,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -550,6 +551,26 @@ func (s *DockerDaemonSuite) TestDaemonAllocatesListeningPort(c *check.C) { } } +func (s *DockerDaemonSuite) TestDaemonUUIDGeneration(c *check.C) { + dir := "/var/lib/docker" + if runtime.GOOS == "windows" { + dir = filepath.Join(os.Getenv("programdata"), "docker") + } + file := filepath.Join(dir, "engine_uuid") + os.Remove(file) + s.d.Start(c) + s.d.Stop(c) + + fi, err := os.Stat(file) + if err != nil { + c.Fatalf("Error opening uuid file") + } + // Test for uuid length + if fi.Size() != 36 { + c.Fatalf("Bad UUID size %d", fi.Size()) + } +} + // GH#11320 - verify that the daemon exits on failure properly // Note that this explicitly tests the conflict of {-b,--bridge} and {--bip} options as the means // to get a daemon init failure; no other tests for -b/--bip conflict are therefore required @@ -1174,6 +1195,59 @@ func (s *DockerDaemonSuite) TestDaemonUnixSockCleanedUp(c *check.C) { } } +func (s *DockerDaemonSuite) TestDaemonWithWrongkey(c *check.C) { + type Config struct { + Crv string `json:"crv"` + D string `json:"d"` + Kid string `json:"kid"` + Kty string `json:"kty"` + X string `json:"x"` + Y string `json:"y"` + } + + os.Remove("/etc/docker/key.json") + s.d.Start(c) + s.d.Stop(c) + + config := &Config{} + bytes, err := ioutil.ReadFile("/etc/docker/key.json") + if err != nil { + c.Fatalf("Error reading key.json file: %s", err) + } + + // byte[] to Data-Struct + if err := json.Unmarshal(bytes, &config); err != nil { + c.Fatalf("Error Unmarshal: %s", err) + } + + //replace config.Kid with the fake value + config.Kid = "VSAJ:FUYR:X3H2:B2VZ:KZ6U:CJD5:K7BX:ZXHY:UZXT:P4FT:MJWG:HRJ4" + + // NEW Data-Struct to byte[] + newBytes, err := json.Marshal(&config) + if err != nil { + c.Fatalf("Error Marshal: %s", err) + } + + // write back + if err := ioutil.WriteFile("/etc/docker/key.json", newBytes, 0400); err != nil { + c.Fatalf("Error ioutil.WriteFile: %s", err) + } + + defer os.Remove("/etc/docker/key.json") + + if err := s.d.StartWithError(); err == nil { + c.Fatalf("It should not be successful to start daemon with wrong key: %v", err) + } + + content, err := s.d.ReadLogFile() + c.Assert(err, checker.IsNil) + + if !strings.Contains(string(content), "Public Key ID does not match") { + c.Fatalf("Missing KeyID message from daemon logs: %s", string(content)) + } +} + func (s *DockerDaemonSuite) TestDaemonRestartKillWait(c *check.C) { s.d.StartWithBusybox(c) diff --git a/integration-cli/docker_cli_pull_local_test.go b/integration-cli/docker_cli_pull_local_test.go index 0ddf7770d1..b6b774ea44 100644 --- a/integration-cli/docker_cli_pull_local_test.go +++ b/integration-cli/docker_cli_pull_local_test.go @@ -56,6 +56,10 @@ func (s *DockerRegistrySuite) TestPullImageWithAliases(c *check.C) { testPullImageWithAliases(c) } +func (s *DockerSchema1RegistrySuite) TestPullImageWithAliases(c *check.C) { + testPullImageWithAliases(c) +} + // testConcurrentPullWholeRepo pulls the same repo concurrently. func testConcurrentPullWholeRepo(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) @@ -108,6 +112,10 @@ func (s *DockerRegistrySuite) testConcurrentPullWholeRepo(c *check.C) { testConcurrentPullWholeRepo(c) } +func (s *DockerSchema1RegistrySuite) testConcurrentPullWholeRepo(c *check.C) { + testConcurrentPullWholeRepo(c) +} + // testConcurrentFailingPull tries a concurrent pull that doesn't succeed. func testConcurrentFailingPull(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) @@ -135,6 +143,10 @@ func (s *DockerRegistrySuite) testConcurrentFailingPull(c *check.C) { testConcurrentFailingPull(c) } +func (s *DockerSchema1RegistrySuite) testConcurrentFailingPull(c *check.C) { + testConcurrentFailingPull(c) +} + // testConcurrentPullMultipleTags pulls multiple tags from the same repo // concurrently. func testConcurrentPullMultipleTags(c *check.C) { @@ -187,6 +199,10 @@ func (s *DockerRegistrySuite) TestConcurrentPullMultipleTags(c *check.C) { testConcurrentPullMultipleTags(c) } +func (s *DockerSchema1RegistrySuite) TestConcurrentPullMultipleTags(c *check.C) { + testConcurrentPullMultipleTags(c) +} + // testPullIDStability verifies that pushing an image and pulling it back // preserves the image ID. func testPullIDStability(c *check.C) { @@ -244,6 +260,10 @@ func (s *DockerRegistrySuite) TestPullIDStability(c *check.C) { testPullIDStability(c) } +func (s *DockerSchema1RegistrySuite) TestPullIDStability(c *check.C) { + testPullIDStability(c) +} + // #21213 func testPullNoLayers(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/scratch", privateRegistryURL) @@ -260,6 +280,10 @@ func (s *DockerRegistrySuite) TestPullNoLayers(c *check.C) { testPullNoLayers(c) } +func (s *DockerSchema1RegistrySuite) TestPullNoLayers(c *check.C) { + testPullNoLayers(c) +} + func (s *DockerRegistrySuite) TestPullManifestList(c *check.C) { testRequires(c, NotArm) pushDigest, err := setupImage(c) diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 40fb2b8417..ebef8493ef 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -30,6 +30,10 @@ func (s *DockerRegistrySuite) TestPushBusyboxImage(c *check.C) { testPushBusyboxImage(c) } +func (s *DockerSchema1RegistrySuite) TestPushBusyboxImage(c *check.C) { + testPushBusyboxImage(c) +} + // pushing an image without a prefix should throw an error func (s *DockerSuite) TestPushUnprefixedRepo(c *check.C) { out, _, err := dockerCmdWithError("push", "busybox") @@ -49,6 +53,10 @@ func (s *DockerRegistrySuite) TestPushUntagged(c *check.C) { testPushUntagged(c) } +func (s *DockerSchema1RegistrySuite) TestPushUntagged(c *check.C) { + testPushUntagged(c) +} + func testPushBadTag(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/busybox:latest", privateRegistryURL) expected := "does not exist" @@ -62,6 +70,10 @@ func (s *DockerRegistrySuite) TestPushBadTag(c *check.C) { testPushBadTag(c) } +func (s *DockerSchema1RegistrySuite) TestPushBadTag(c *check.C) { + testPushBadTag(c) +} + func testPushMultipleTags(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) repoTag1 := fmt.Sprintf("%v/dockercli/busybox:t1", privateRegistryURL) @@ -103,6 +115,10 @@ func (s *DockerRegistrySuite) TestPushMultipleTags(c *check.C) { testPushMultipleTags(c) } +func (s *DockerSchema1RegistrySuite) TestPushMultipleTags(c *check.C) { + testPushMultipleTags(c) +} + func testPushEmptyLayer(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/emptylayer", privateRegistryURL) emptyTarball, err := ioutil.TempFile("", "empty_tarball") @@ -130,6 +146,10 @@ func (s *DockerRegistrySuite) TestPushEmptyLayer(c *check.C) { testPushEmptyLayer(c) } +func (s *DockerSchema1RegistrySuite) TestPushEmptyLayer(c *check.C) { + testPushEmptyLayer(c) +} + // testConcurrentPush pushes multiple tags to the same repo // concurrently. func testConcurrentPush(c *check.C) { @@ -180,6 +200,10 @@ func (s *DockerRegistrySuite) TestConcurrentPush(c *check.C) { testConcurrentPush(c) } +func (s *DockerSchema1RegistrySuite) TestConcurrentPush(c *check.C) { + testConcurrentPush(c) +} + func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) { sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) // tag the image to upload it to the private registry @@ -222,6 +246,39 @@ func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) { assert.Equal(c, out4, "hello world") } +func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c *check.C) { + sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it to the private registry + dockerCmd(c, "tag", "busybox", sourceRepoName) + // push the image to the registry + out1, _, err := dockerCmdWithError("push", sourceRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out1)) + // ensure that none of the layers were mounted from another repository during push + c.Assert(strings.Contains(out1, "Mounted from"), check.Equals, false) + + digest1 := reference.DigestRegexp.FindString(out1) + c.Assert(len(digest1), checker.GreaterThan, 0, check.Commentf("no digest found for pushed manifest")) + + destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL) + // retag the image to upload the same layers to another repo in the same registry + dockerCmd(c, "tag", "busybox", destRepoName) + // push the image to the registry + out2, _, err := dockerCmdWithError("push", destRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2)) + // schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen + c.Assert(strings.Contains(out2, "Mounted from"), check.Equals, false) + + digest2 := reference.DigestRegexp.FindString(out2) + c.Assert(len(digest2), checker.GreaterThan, 0, check.Commentf("no digest found for pushed manifest")) + c.Assert(digest1, check.Not(check.Equals), digest2) + + // ensure that we can pull and run the second pushed repository + dockerCmd(c, "rmi", destRepoName) + dockerCmd(c, "pull", destRepoName) + out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world") + c.Assert(out3, check.Equals, "hello world") +} + func (s *DockerRegistryAuthHtpasswdSuite) TestPushNoCredentialsNoRetry(c *check.C) { repoName := fmt.Sprintf("%s/busybox", privateRegistryURL) dockerCmd(c, "tag", "busybox", repoName) diff --git a/integration/system/uuid_test.go b/integration/system/uuid_test.go deleted file mode 100644 index d6df8fd4c8..0000000000 --- a/integration/system/uuid_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package system - -import ( - "context" - "testing" - - "github.com/docker/docker/api/types/versions" - "github.com/google/uuid" - "gotest.tools/assert" - "gotest.tools/skip" -) - -func TestUUIDGeneration(t *testing.T) { - skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "ID format changed") - defer setupTest(t)() - - c := testEnv.APIClient() - info, err := c.Info(context.Background()) - assert.NilError(t, err) - - _, err = uuid.Parse(info.ID) - assert.NilError(t, err, info.ID) -}