From ed13c3abfb242905ec012e8255dc6f26dcf122f6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 15 Jul 2015 13:42:45 -0700 Subject: [PATCH] Use notary library for trusted image fetch and signing Add a trusted flag to force the cli to resolve a tag into a digest via the notary trust library and pull by digest. On push the flag the trust flag will indicate the digest and size of a manifest should be signed and push to a notary server. If a tag is given, the cli will resolve the tag into a digest and pull by digest. After pulling, if a tag is given the cli makes a request to tag the image. Use certificate directory for notary requests Read certificates using same logic used by daemon for registry requests. Catch JSON syntax errors from Notary client When an uncaught error occurs in Notary it may show up in Docker as a JSON syntax error, causing a confusing error message to the user. Provide a generic error when a JSON syntax error occurs. Catch expiration errors and wrap in additional context. Signed-off-by: Derek McGowan (github: dmcgowan) --- api/client/create.go | 34 +- api/client/pull.go | 25 +- api/client/push.go | 5 + api/client/run.go | 1 + api/client/trust.go | 435 +++++++++++++++++++ graph/pull_v2.go | 2 +- graph/push_v2.go | 4 +- graph/registry.go | 6 +- integration-cli/docker_cli_by_digest_test.go | 8 +- pkg/ansiescape/split.go | 89 ++++ pkg/ansiescape/split_test.go | 53 +++ registry/config.go | 4 +- registry/reference.go | 68 +++ registry/registry.go | 52 +++ registry/service.go | 51 +-- 15 files changed, 759 insertions(+), 78 deletions(-) create mode 100644 api/client/trust.go create mode 100644 pkg/ansiescape/split.go create mode 100644 pkg/ansiescape/split_test.go create mode 100644 registry/reference.go diff --git a/api/client/create.go b/api/client/create.go index 8d573c2218..76e935eb15 100644 --- a/api/client/create.go +++ b/api/client/create.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) func (cli *DockerCli) pullImage(image string) error { @@ -95,20 +94,42 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc defer containerIDFile.Close() } + repo, tag := parsers.ParseRepositoryTag(config.Image) + if tag == "" { + tag = tags.DEFAULTTAG + } + + ref := registry.ParseReference(tag) + var trustedRef registry.Reference + + if isTrusted() && !ref.HasDigest() { + var err error + trustedRef, err = cli.trustedReference(repo, ref) + if err != nil { + return nil, err + } + config.Image = trustedRef.ImageName(repo) + } + //create the container serverResp, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, nil) //if image not found try to pull it if serverResp.statusCode == 404 && strings.Contains(err.Error(), config.Image) { - repo, tag := parsers.ParseRepositoryTag(config.Image) - if tag == "" { - tag = tags.DEFAULTTAG - } - fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", utils.ImageReference(repo, tag)) + fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.ImageName(repo)) // we don't want to write to stdout anything apart from container.ID if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil { return nil, err } + if trustedRef != nil && !ref.HasDigest() { + repoInfo, err := registry.ParseRepositoryInfo(repo) + if err != nil { + return nil, err + } + if err := cli.tagTrusted(repoInfo, trustedRef, ref); err != nil { + return nil, err + } + } // Retry if serverResp, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, nil); err != nil { return nil, err @@ -139,6 +160,7 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc // Usage: docker create [OPTIONS] IMAGE [COMMAND] [ARG...] func (cli *DockerCli) CmdCreate(args ...string) error { cmd := Cli.Subcmd("create", []string{"IMAGE [COMMAND] [ARG...]"}, "Create a new container", true) + addTrustedFlags(cmd, true) // These are flags not stored in Config/HostConfig var ( diff --git a/api/client/pull.go b/api/client/pull.go index 7773b975ea..d6b8554311 100644 --- a/api/client/pull.go +++ b/api/client/pull.go @@ -9,7 +9,6 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" - "github.com/docker/docker/utils" ) // CmdPull pulls an image or a repository from the registry. @@ -18,24 +17,21 @@ import ( func (cli *DockerCli) CmdPull(args ...string) error { cmd := Cli.Subcmd("pull", []string{"NAME[:TAG|@DIGEST]"}, "Pull an image or a repository from a registry", true) allTags := cmd.Bool([]string{"a", "-all-tags"}, false, "Download all tagged images in the repository") + addTrustedFlags(cmd, true) cmd.Require(flag.Exact, 1) cmd.ParseFlags(args, true) + remote := cmd.Arg(0) - var ( - v = url.Values{} - remote = cmd.Arg(0) - newRemote = remote - ) taglessRemote, tag := parsers.ParseRepositoryTag(remote) if tag == "" && !*allTags { - newRemote = utils.ImageReference(taglessRemote, tags.DEFAULTTAG) - } - if tag != "" && *allTags { + tag = tags.DEFAULTTAG + fmt.Fprintf(cli.out, "Using default tag: %s\n", tag) + } else if tag != "" && *allTags { return fmt.Errorf("tag can't be used with --all-tags/-a") } - v.Set("fromImage", newRemote) + ref := registry.ParseReference(tag) // Resolve the Repository name from fqn to RepositoryInfo repoInfo, err := registry.ParseRepositoryInfo(taglessRemote) @@ -43,6 +39,15 @@ func (cli *DockerCli) CmdPull(args ...string) error { return err } + if isTrusted() && !ref.HasDigest() { + // Check if tag is digest + authConfig := registry.ResolveAuthConfig(cli.configFile, repoInfo.Index) + return cli.trustedPull(repoInfo, ref, authConfig) + } + + v := url.Values{} + v.Set("fromImage", ref.ImageName(taglessRemote)) + _, _, err = cli.clientRequestAttemptLogin("POST", "/images/create?"+v.Encode(), nil, cli.out, repoInfo.Index, "pull") return err } diff --git a/api/client/push.go b/api/client/push.go index 4a31f9ba61..5d01511ca7 100644 --- a/api/client/push.go +++ b/api/client/push.go @@ -15,6 +15,7 @@ import ( // Usage: docker push NAME[:TAG] func (cli *DockerCli) CmdPush(args ...string) error { cmd := Cli.Subcmd("push", []string{"NAME[:TAG]"}, "Push an image or a repository to a registry", true) + addTrustedFlags(cmd, false) cmd.Require(flag.Exact, 1) cmd.ParseFlags(args, true) @@ -40,6 +41,10 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("You cannot push a \"root\" repository. Please rename your repository to / (ex: %s/%s)", username, repoInfo.LocalName) } + if isTrusted() { + return cli.trustedPush(repoInfo, tag, authConfig) + } + v := url.Values{} v.Set("tag", tag) diff --git a/api/client/run.go b/api/client/run.go index 7e28cb2219..182dc4b1f9 100644 --- a/api/client/run.go +++ b/api/client/run.go @@ -41,6 +41,7 @@ func (cid *cidFile) Write(id string) error { // Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...] func (cli *DockerCli) CmdRun(args ...string) error { cmd := Cli.Subcmd("run", []string{"IMAGE [COMMAND] [ARG...]"}, "Run a command in a new container", true) + addTrustedFlags(cmd, true) // These are flags not stored in Config/HostConfig var ( diff --git a/api/client/trust.go b/api/client/trust.go new file mode 100644 index 0000000000..8e3635da3c --- /dev/null +++ b/api/client/trust.go @@ -0,0 +1,435 @@ +package client + +import ( + "bufio" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/cliconfig" + "github.com/docker/docker/pkg/ansiescape" + "github.com/docker/docker/pkg/ioutils" + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/pkg/tlsconfig" + "github.com/docker/docker/registry" + "github.com/docker/notary/client" + "github.com/docker/notary/pkg/passphrase" + "github.com/docker/notary/trustmanager" + "github.com/endophage/gotuf/data" +) + +var untrusted bool + +func addTrustedFlags(fs *flag.FlagSet, verify bool) { + var trusted bool + if e := os.Getenv("DOCKER_TRUST"); e != "" { + if t, err := strconv.ParseBool(e); t || err != nil { + // treat any other value as true + trusted = true + } + } + message := "Skip image signing" + if verify { + message = "Skip image verification" + } + fs.BoolVar(&untrusted, []string{"-untrusted"}, !trusted, message) +} + +func isTrusted() bool { + return !untrusted +} + +var targetRegexp = regexp.MustCompile(`([\S]+): digest: ([\S]+) size: ([\d]+)`) + +type target struct { + reference registry.Reference + digest digest.Digest + size int64 +} + +func (cli *DockerCli) trustDirectory() string { + return filepath.Join(cliconfig.ConfigDir(), "trust") +} + +// certificateDirectory returns the directory containing +// TLS certificates for the given server. An error is +// returned if there was an error parsing the server string. +func (cli *DockerCli) certificateDirectory(server string) (string, error) { + u, err := url.Parse(server) + if err != nil { + return "", err + } + + return filepath.Join(cliconfig.ConfigDir(), "tls", u.Host), nil +} + +func trustServer(index *registry.IndexInfo) string { + if s := os.Getenv("DOCKER_TRUST_SERVER"); s != "" { + if !strings.HasPrefix(s, "https://") { + return "https://" + s + } + return s + } + if index.Official { + return registry.NotaryServer + } + return "https://" + index.Name +} + +type simpleCredentialStore struct { + auth cliconfig.AuthConfig +} + +func (scs simpleCredentialStore) Basic(u *url.URL) (string, string) { + return scs.auth.Username, scs.auth.Password +} + +func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, authConfig cliconfig.AuthConfig) (*client.NotaryRepository, error) { + server := trustServer(repoInfo.Index) + if !strings.HasPrefix(server, "https://") { + return nil, errors.New("unsupported scheme: https required for trust server") + } + + var cfg = tlsconfig.ClientDefault + cfg.InsecureSkipVerify = !repoInfo.Index.Secure + + // Get certificate base directory + certDir, err := cli.certificateDirectory(server) + if err != nil { + return nil, err + } + logrus.Debugf("reading certificate directory: %s", certDir) + + if err := registry.ReadCertsDirectory(&cfg, certDir); err != nil { + return nil, err + } + + base := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &cfg, + DisableKeepAlives: true, + } + + // Skip configuration headers since request is not going to Docker daemon + modifiers := registry.DockerHeaders(http.Header{}) + authTransport := transport.NewTransport(base, modifiers...) + pingClient := &http.Client{ + Transport: authTransport, + Timeout: 5 * time.Second, + } + endpointStr := server + "/v2/" + req, err := http.NewRequest("GET", endpointStr, nil) + if err != nil { + return nil, err + } + resp, err := pingClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + challengeManager := auth.NewSimpleChallengeManager() + if err := challengeManager.AddResponse(resp); err != nil { + return nil, err + } + + creds := simpleCredentialStore{auth: authConfig} + tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName, "push", "pull") + basicHandler := auth.NewBasicHandler(creds) + modifiers = append(modifiers, transport.RequestModifier(auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))) + tr := transport.NewTransport(base, modifiers...) + + return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName, server, tr, cli.getPassphraseRetriever()) +} + +func convertTarget(t client.Target) (target, error) { + h, ok := t.Hashes["sha256"] + if !ok { + return target{}, errors.New("no valid hash, expecting sha256") + } + return target{ + reference: registry.ParseReference(t.Name), + digest: digest.NewDigestFromHex("sha256", hex.EncodeToString(h)), + size: t.Length, + }, nil +} + +func (cli *DockerCli) getPassphraseRetriever() passphrase.Retriever { + baseRetriever := passphrase.PromptRetrieverWithInOut(cli.in, cli.out) + env := map[string]string{ + "root": os.Getenv("DOCKER_TRUST_ROOT_PASSPHRASE"), + "targets": os.Getenv("DOCKER_TRUST_TARGET_PASSPHRASE"), + "snapshot": os.Getenv("DOCKER_TRUST_SNAPSHOT_PASSPHRASE"), + } + return func(keyName string, alias string, createNew bool, numAttempts int) (string, bool, error) { + if v := env[alias]; v != "" { + return v, numAttempts > 1, nil + } + return baseRetriever(keyName, alias, createNew, numAttempts) + } +} + +func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (registry.Reference, error) { + repoInfo, err := registry.ParseRepositoryInfo(repo) + if err != nil { + return nil, err + } + + // Resolve the Auth config relevant for this server + authConfig := registry.ResolveAuthConfig(cli.configFile, repoInfo.Index) + + notaryRepo, err := cli.getNotaryRepository(repoInfo, authConfig) + if err != nil { + fmt.Fprintf(cli.out, "Error establishing connection to trust repository: %s\n", err) + return nil, err + } + + t, err := notaryRepo.GetTargetByName(ref.String()) + if err != nil { + return nil, err + } + r, err := convertTarget(*t) + if err != nil { + return nil, err + + } + + return registry.DigestReference(r.digest), nil +} + +func (cli *DockerCli) tagTrusted(repoInfo *registry.RepositoryInfo, trustedRef, ref registry.Reference) error { + fullName := trustedRef.ImageName(repoInfo.LocalName) + fmt.Fprintf(cli.out, "Tagging %s as %s\n", fullName, ref.ImageName(repoInfo.LocalName)) + tv := url.Values{} + tv.Set("repo", repoInfo.LocalName) + tv.Set("tag", ref.String()) + tv.Set("force", "1") + + if _, _, err := readBody(cli.call("POST", "/images/"+fullName+"/tag?"+tv.Encode(), nil, nil)); err != nil { + return err + } + + return nil +} + +func notaryError(err error) error { + switch err.(type) { + case *json.SyntaxError: + logrus.Debugf("Notary syntax error: %s", err) + return errors.New("no trust data available for remote repository") + case client.ErrExpired: + return fmt.Errorf("remote repository out-of-date: %v", err) + case trustmanager.ErrKeyNotFound: + return fmt.Errorf("signing keys not found: %v", err) + } + + return err +} + +func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registry.Reference, authConfig cliconfig.AuthConfig) error { + var ( + v = url.Values{} + refs = []target{} + ) + + notaryRepo, err := cli.getNotaryRepository(repoInfo, authConfig) + if err != nil { + fmt.Fprintf(cli.out, "Error establishing connection to trust repository: %s\n", err) + return err + } + + if ref.String() == "" { + // List all targets + targets, err := notaryRepo.ListTargets() + if err != nil { + return notaryError(err) + } + for _, tgt := range targets { + t, err := convertTarget(*tgt) + if err != nil { + fmt.Fprintf(cli.out, "Skipping target for %q\n", repoInfo.LocalName) + continue + } + refs = append(refs, t) + } + } else { + t, err := notaryRepo.GetTargetByName(ref.String()) + if err != nil { + return notaryError(err) + } + r, err := convertTarget(*t) + if err != nil { + return err + + } + refs = append(refs, r) + } + + v.Set("fromImage", repoInfo.LocalName) + for i, r := range refs { + displayTag := r.reference.String() + if displayTag != "" { + displayTag = ":" + displayTag + } + fmt.Fprintf(cli.out, "Pull (%d of %d): %s%s@%s\n", i+1, len(refs), repoInfo.LocalName, displayTag, r.digest) + v.Set("tag", r.digest.String()) + + _, _, err = cli.clientRequestAttemptLogin("POST", "/images/create?"+v.Encode(), nil, cli.out, repoInfo.Index, "pull") + if err != nil { + return err + } + + // If reference is not trusted, tag by trusted reference + if !r.reference.HasDigest() { + if err := cli.tagTrusted(repoInfo, registry.DigestReference(r.digest), r.reference); err != nil { + return err + + } + } + } + return nil +} + +func targetStream(in io.Writer) (io.WriteCloser, <-chan []target) { + r, w := io.Pipe() + out := io.MultiWriter(in, w) + targetChan := make(chan []target) + + go func() { + targets := []target{} + scanner := bufio.NewScanner(r) + scanner.Split(ansiescape.ScanANSILines) + for scanner.Scan() { + line := scanner.Bytes() + if matches := targetRegexp.FindSubmatch(line); len(matches) == 4 { + dgst, err := digest.ParseDigest(string(matches[2])) + if err != nil { + // Line does match what is expected, continue looking for valid lines + logrus.Debugf("Bad digest value %q in matched line, ignoring\n", string(matches[2])) + continue + } + s, err := strconv.ParseInt(string(matches[3]), 10, 64) + if err != nil { + // Line does match what is expected, continue looking for valid lines + logrus.Debugf("Bad size value %q in matched line, ignoring\n", string(matches[3])) + continue + } + + targets = append(targets, target{ + reference: registry.ParseReference(string(matches[1])), + digest: dgst, + size: s, + }) + } + } + targetChan <- targets + }() + + return ioutils.NewWriteCloserWrapper(out, w.Close), targetChan +} + +func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string, authConfig cliconfig.AuthConfig) error { + streamOut, targetChan := targetStream(cli.out) + + v := url.Values{} + v.Set("tag", tag) + + _, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push") + // Close stream channel to finish target parsing + if err := streamOut.Close(); err != nil { + return err + } + // Check error from request + if err != nil { + return err + } + + // Get target results + targets := <-targetChan + + if tag == "" { + fmt.Fprintf(cli.out, "No tag specified, skipping trust metadata push\n") + return nil + } + if len(targets) == 0 { + fmt.Fprintf(cli.out, "No targets found, skipping trust metadata push\n") + return nil + } + + fmt.Fprintf(cli.out, "Signing and pushing trust metadata\n") + + repo, err := cli.getNotaryRepository(repoInfo, authConfig) + if err != nil { + fmt.Fprintf(cli.out, "Error establishing connection to notary repository: %s\n", err) + return err + } + + for _, target := range targets { + h, err := hex.DecodeString(target.digest.Hex()) + if err != nil { + return err + } + t := &client.Target{ + Name: target.reference.String(), + Hashes: data.Hashes{ + string(target.digest.Algorithm()): h, + }, + Length: int64(target.size), + } + if err := repo.AddTarget(t); err != nil { + return err + } + } + + err = repo.Publish() + if _, ok := err.(*client.ErrRepoNotInitialized); !ok { + return notaryError(err) + } + + ks := repo.KeyStoreManager + keys := ks.RootKeyStore().ListKeys() + var rootKey string + + if len(keys) == 0 { + rootKey, err = ks.GenRootKey("ecdsa") + if err != nil { + return err + } + } else { + // TODO(dmcgowan): let user choose + rootKey = keys[0] + } + + cryptoService, err := ks.GetRootCryptoService(rootKey) + if err != nil { + return err + } + + if err := repo.Initialize(cryptoService); err != nil { + return notaryError(err) + } + fmt.Fprintf(cli.out, "Finished initializing %q\n", repoInfo.CanonicalName) + + return notaryError(repo.Publish()) +} diff --git a/graph/pull_v2.go b/graph/pull_v2.go index 92fdc16803..a70529754a 100644 --- a/graph/pull_v2.go +++ b/graph/pull_v2.go @@ -288,7 +288,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) { } } - manifestDigest, err := digestFromManifest(manifest, p.repoInfo.LocalName) + manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName) if err != nil { return false, err } diff --git a/graph/push_v2.go b/graph/push_v2.go index 4f3bfb5bdc..b8011bc139 100644 --- a/graph/push_v2.go +++ b/graph/push_v2.go @@ -184,12 +184,12 @@ func (p *v2Pusher) pushV2Tag(tag string) error { return err } - manifestDigest, err := digestFromManifest(signed, p.repo.Name()) + manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name()) if err != nil { return err } if manifestDigest != "" { - out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) + out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tag, manifestDigest, manifestSize)) } manSvc, err := p.repo.Manifests(context.Background()) diff --git a/graph/registry.go b/graph/registry.go index c4202cf664..9b744ed681 100644 --- a/graph/registry.go +++ b/graph/registry.go @@ -97,15 +97,15 @@ func NewV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd return client.NewRepository(ctx, repoName, endpoint.URL, tr) } -func digestFromManifest(m *manifest.SignedManifest, localName string) (digest.Digest, error) { +func digestFromManifest(m *manifest.SignedManifest, localName string) (digest.Digest, int, error) { payload, err := m.Payload() if err != nil { logrus.Debugf("could not retrieve manifest payload: %v", err) - return "", err + return "", 0, err } manifestDigest, err := digest.FromBytes(payload) if err != nil { logrus.Infof("Could not compute manifest digest for %s:%s : %v", localName, m.Tag, err) } - return manifestDigest, nil + return manifestDigest, len(payload), nil } diff --git a/integration-cli/docker_cli_by_digest_test.go b/integration-cli/docker_cli_by_digest_test.go index a53dea4235..cbc6dc1bad 100644 --- a/integration-cli/docker_cli_by_digest_test.go +++ b/integration-cli/docker_cli_by_digest_test.go @@ -10,8 +10,9 @@ import ( ) var ( - repoName = fmt.Sprintf("%v/dockercli/busybox-by-dgst", privateRegistryURL) - digestRegex = regexp.MustCompile("Digest: ([^\n]+)") + repoName = fmt.Sprintf("%v/dockercli/busybox-by-dgst", privateRegistryURL) + pushDigestRegex = regexp.MustCompile("[\\S]+: digest: ([\\S]+) size: [0-9]+") + digestRegex = regexp.MustCompile("Digest: ([\\S]+)") ) func setupImage(c *check.C) (string, error) { @@ -45,8 +46,7 @@ func setupImageWithTag(c *check.C, tag string) (string, error) { return "", fmt.Errorf("error deleting images prior to real test: %s, %v", rmiout, err) } - // the push output includes "Digest: ", so find that - matches := digestRegex.FindStringSubmatch(out) + matches := pushDigestRegex.FindStringSubmatch(out) if len(matches) != 2 { return "", fmt.Errorf("unable to parse digest from push output: %s", out) } diff --git a/pkg/ansiescape/split.go b/pkg/ansiescape/split.go new file mode 100644 index 0000000000..b097b25f87 --- /dev/null +++ b/pkg/ansiescape/split.go @@ -0,0 +1,89 @@ +package ansiescape + +import "bytes" + +// dropCR drops a leading or terminal \r from the data. +func dropCR(data []byte) []byte { + if len(data) > 0 && data[len(data)-1] == '\r' { + data = data[0 : len(data)-1] + } + if len(data) > 0 && data[0] == '\r' { + data = data[1:] + } + return data +} + +// escapeSequenceLength calculates the length of an ANSI escape sequence +// If there is not enough characters to match a sequence, -1 is returned, +// if there is no valid sequence 0 is returned, otherwise the number +// of bytes in the sequence is returned. Only returns length for +// line moving sequences. +func escapeSequenceLength(data []byte) int { + next := 0 + if len(data) <= next { + return -1 + } + if data[next] != '[' { + return 0 + } + for { + next = next + 1 + if len(data) <= next { + return -1 + } + if (data[next] > '9' || data[next] < '0') && data[next] != ';' { + break + } + } + if len(data) <= next { + return -1 + } + // Only match line moving codes + switch data[next] { + case 'A', 'B', 'E', 'F', 'H', 'h': + return next + 1 + } + + return 0 +} + +// ScanANSILines is a scanner function which splits the +// input based on ANSI escape codes and new lines. +func ScanANSILines(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + // Look for line moving escape sequence + if i := bytes.IndexByte(data, '\x1b'); i >= 0 { + last := 0 + for i >= 0 { + last = last + i + + // get length of ANSI escape sequence + sl := escapeSequenceLength(data[last+1:]) + if sl == -1 { + return 0, nil, nil + } + if sl == 0 { + // If no relevant sequence was found, skip + last = last + 1 + i = bytes.IndexByte(data[last:], '\x1b') + continue + } + + return last + 1 + sl, dropCR(data[0:(last)]), nil + } + } + if i := bytes.IndexByte(data, '\n'); i >= 0 { + // No escape sequence, check for new line + return i + 1, dropCR(data[0:i]), nil + } + + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), dropCR(data), nil + } + // Request more data. + return 0, nil, nil +} diff --git a/pkg/ansiescape/split_test.go b/pkg/ansiescape/split_test.go new file mode 100644 index 0000000000..ecb24b9398 --- /dev/null +++ b/pkg/ansiescape/split_test.go @@ -0,0 +1,53 @@ +package ansiescape + +import ( + "bufio" + "strings" + "testing" +) + +func TestSplit(t *testing.T) { + lines := []string{ + "test line 1", + "another test line", + "some test line", + "line with non-cursor moving sequence \x1b[1T", // Scroll Down + "line with \x1b[31;1mcolor\x1b[0m then reset", // "color" in Bold Red + "cursor forward \x1b[1C and backward \x1b[1D", + "invalid sequence \x1babcd", + "", + "after empty", + } + splitSequences := []string{ + "\x1b[1A", // Cursor up + "\x1b[1B", // Cursor down + "\x1b[1E", // Cursor next line + "\x1b[1F", // Cursor previous line + "\x1b[1;1H", // Move cursor to position + "\x1b[1;1h", // Move cursor to position + "\n", + "\r\n", + "\n\r", + "\x1b[1A\r", + "\r\x1b[1A", + } + + for _, sequence := range splitSequences { + scanner := bufio.NewScanner(strings.NewReader(strings.Join(lines, sequence))) + scanner.Split(ScanANSILines) + i := 0 + for scanner.Scan() { + if i >= len(lines) { + t.Fatalf("Too many scanned lines") + } + scanned := scanner.Text() + if scanned != lines[i] { + t.Fatalf("Wrong line scanned with sequence %q\n\tExpected: %q\n\tActual: %q", sequence, lines[i], scanned) + } + i++ + } + if i < len(lines) { + t.Errorf("Wrong number of lines for sequence %q: %d, expected %d", sequence, i, len(lines)) + } + } +} diff --git a/registry/config.go b/registry/config.go index d2108894f8..95f731298c 100644 --- a/registry/config.go +++ b/registry/config.go @@ -38,8 +38,8 @@ const ( IndexServer = DefaultV1Registry + "/v1/" // IndexName is the name of the index IndexName = "docker.io" - - // IndexServer = "https://registry-stage.hub.docker.com/v1/" + // NotaryServer is the endpoint serving the Notary trust server + NotaryServer = "https://notary.docker.io" ) var ( diff --git a/registry/reference.go b/registry/reference.go new file mode 100644 index 0000000000..e15f83eeeb --- /dev/null +++ b/registry/reference.go @@ -0,0 +1,68 @@ +package registry + +import ( + "strings" + + "github.com/docker/distribution/digest" +) + +// Reference represents a tag or digest within a repository +type Reference interface { + // HasDigest returns whether the reference has a verifiable + // content addressable reference which may be considered secure. + HasDigest() bool + + // ImageName returns an image name for the given repository + ImageName(string) string + + // Returns a string representation of the reference + String() string +} + +type tagReference struct { + tag string +} + +func (tr tagReference) HasDigest() bool { + return false +} + +func (tr tagReference) ImageName(repo string) string { + return repo + ":" + tr.tag +} + +func (tr tagReference) String() string { + return tr.tag +} + +type digestReference struct { + digest digest.Digest +} + +func (dr digestReference) HasDigest() bool { + return true +} + +func (dr digestReference) ImageName(repo string) string { + return repo + "@" + dr.String() +} + +func (dr digestReference) String() string { + return dr.digest.String() +} + +// ParseReference parses a reference into either a digest or tag reference +func ParseReference(ref string) Reference { + if strings.Contains(ref, ":") { + dgst, err := digest.ParseDigest(ref) + if err == nil { + return digestReference{digest: dgst} + } + } + return tagReference{tag: ref} +} + +// DigestReference creates a digest reference using a digest +func DigestReference(dgst digest.Digest) Reference { + return digestReference{digest: dgst} +} diff --git a/registry/registry.go b/registry/registry.go index fd85c21ca1..09143ba8c2 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -2,10 +2,14 @@ package registry import ( "crypto/tls" + "crypto/x509" "errors" + "fmt" + "io/ioutil" "net" "net/http" "os" + "path/filepath" "runtime" "strings" "time" @@ -54,6 +58,54 @@ func hasFile(files []os.FileInfo, name string) bool { return false } +// ReadCertsDirectory reads the directory for TLS certificates +// including roots and certificate pairs and updates the +// provided TLS configuration. +func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error { + fs, err := ioutil.ReadDir(directory) + if err != nil && !os.IsNotExist(err) { + return err + } + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if tlsConfig.RootCAs == nil { + // TODO(dmcgowan): Copy system pool + tlsConfig.RootCAs = x509.NewCertPool() + } + logrus.Debugf("crt: %s", filepath.Join(directory, f.Name())) + data, err := ioutil.ReadFile(filepath.Join(directory, f.Name())) + if err != nil { + return err + } + tlsConfig.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf("cert: %s", filepath.Join(directory, f.Name())) + if !hasFile(fs, keyName) { + return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName)) + if err != nil { + return err + } + tlsConfig.Certificates = append(tlsConfig.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf("key: %s", filepath.Join(directory, f.Name())) + if !hasFile(fs, certName) { + return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + return nil +} + // DockerHeaders returns request modifiers that ensure requests have // the User-Agent header set to dockerUserAgent and that metaHeaders // are added. diff --git a/registry/service.go b/registry/service.go index 274dfeb26d..fa35e3132e 100644 --- a/registry/service.go +++ b/registry/service.go @@ -2,12 +2,9 @@ package registry import ( "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" "net/http" "net/url" - "os" "path/filepath" "strings" @@ -110,57 +107,11 @@ func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { tlsConfig.InsecureSkipVerify = !isSecure if isSecure { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - hostDir := filepath.Join(CertsDir, hostname) logrus.Debugf("hostDir: %s", hostDir) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { + if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { return nil, err } - - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if tlsConfig.RootCAs == nil { - // TODO(dmcgowan): Copy system pool - tlsConfig.RootCAs = x509.NewCertPool() - } - logrus.Debugf("crt: %s", filepath.Join(hostDir, f.Name())) - data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) - if err != nil { - return nil, err - } - tlsConfig.RootCAs.AppendCertsFromPEM(data) - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logrus.Debugf("cert: %s", filepath.Join(hostDir, f.Name())) - if !hasFile(fs, keyName) { - return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), filepath.Join(hostDir, keyName)) - if err != nil { - return nil, err - } - tlsConfig.Certificates = append(tlsConfig.Certificates, cert) - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logrus.Debugf("key: %s", filepath.Join(hostDir, f.Name())) - if !hasFile(fs, certName) { - return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) - } - } - } } return &tlsConfig, nil