From d1502afb63a10df0bfce20ae2957774cfb3e58d8 Mon Sep 17 00:00:00 2001 From: Mike Goelzer Date: Tue, 8 Mar 2016 18:18:53 -0800 Subject: [PATCH] Pass upstream client's user agent through to registry on image pulls Changes how the Engine interacts with Registry servers on image pull. Previously, Engine sent a User-Agent string to the Registry server that included only the Engine's version information. This commit appends to that string the fields from the User-Agent sent by the client (e.g., Compose) of the Engine. This allows Registry server operators to understand what tools are actually generating pulls on their registries. Signed-off-by: Mike Goelzer --- api/client/trust.go | 2 +- api/server/httputils/httputils.go | 3 + api/server/middleware/user_agent.go | 2 + api/server/router/image/backend.go | 3 +- api/server/router/image/image_routes.go | 2 +- daemon/daemon.go | 10 +-- distribution/pull_v1.go | 4 +- distribution/push_v1.go | 4 +- distribution/registry.go | 4 +- dockerversion/useragent.go | 55 +++++++++++- .../docker_cli_registry_user_agent_test.go | 90 +++++++++++++++++++ 11 files changed, 163 insertions(+), 16 deletions(-) create mode 100644 integration-cli/docker_cli_registry_user_agent_test.go diff --git a/api/client/trust.go b/api/client/trust.go index cc7af987db..d897872ca3 100644 --- a/api/client/trust.go +++ b/api/client/trust.go @@ -152,7 +152,7 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut } // Skip configuration headers since request is not going to Docker daemon - modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), http.Header{}) + modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(""), http.Header{}) authTransport := transport.NewTransport(base, modifiers...) pingClient := &http.Client{ Transport: authTransport, diff --git a/api/server/httputils/httputils.go b/api/server/httputils/httputils.go index c81256b145..59ee0308b8 100644 --- a/api/server/httputils/httputils.go +++ b/api/server/httputils/httputils.go @@ -16,6 +16,9 @@ import ( // APIVersionKey is the client's requested API version. const APIVersionKey = "api-version" +// UAStringKey is used as key type for user-agent string in net/context struct +const UAStringKey = "upstream-user-agent" + // APIFunc is an adapter to allow the use of ordinary functions as Docker API endpoints. // Any function that has the appropriate signature can be registered as a API endpoint (e.g. getVersion). type APIFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error diff --git a/api/server/middleware/user_agent.go b/api/server/middleware/user_agent.go index be4e171cd2..188196bf63 100644 --- a/api/server/middleware/user_agent.go +++ b/api/server/middleware/user_agent.go @@ -16,6 +16,8 @@ func NewUserAgentMiddleware(versionCheck string) Middleware { return func(handler httputils.APIFunc) httputils.APIFunc { return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + ctx = context.WithValue(ctx, httputils.UAStringKey, r.Header.Get("User-Agent")) + if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { userAgent := strings.Split(r.Header.Get("User-Agent"), "/") diff --git a/api/server/router/image/backend.go b/api/server/router/image/backend.go index 8c76ef9260..c0bf3aeba5 100644 --- a/api/server/router/image/backend.go +++ b/api/server/router/image/backend.go @@ -7,6 +7,7 @@ import ( "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" "github.com/docker/engine-api/types/registry" + "golang.org/x/net/context" ) // Backend is all the methods that need to be implemented @@ -37,7 +38,7 @@ type importExportBackend interface { } type registryBackend interface { - PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error SearchRegistryForImages(term string, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error) } diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index dade346925..aa68e115da 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -129,7 +129,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite } } - err = s.backend.PullImage(ref, metaHeaders, authConfig, output) + err = s.backend.PullImage(ctx, ref, metaHeaders, authConfig, output) } } // Check the error from pulling an image to make sure the request diff --git a/daemon/daemon.go b/daemon/daemon.go index 8fd8edcc37..7b52f711d8 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -1007,14 +1007,14 @@ func isBrokenPipe(e error) bool { // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (daemon *Daemon) PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) writesDone := make(chan struct{}) - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc := context.WithCancel(ctx) go func() { writeDistributionProgress(cancelFunc, outStream, progressChan) @@ -1062,7 +1062,7 @@ func (daemon *Daemon) PullOnBuild(name string, authConfigs map[string]types.Auth pullRegistryAuth = &resolvedConfig } - if err := daemon.PullImage(ref, nil, pullRegistryAuth, output); err != nil { + if err := daemon.PullImage(context.Background(), ref, nil, pullRegistryAuth, output); err != nil { return nil, err } return daemon.GetImage(name) @@ -1519,7 +1519,7 @@ func configureVolumes(config *Config, rootUID, rootGID int) (*store.VolumeStore, // AuthenticateToRegistry checks the validity of credentials in authConfig func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (string, string, error) { - return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent()) + return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent("")) } // SearchRegistryForImages queries the registry for images matching @@ -1527,7 +1527,7 @@ func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (stri func (daemon *Daemon) SearchRegistryForImages(term string, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { - return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(), headers) + return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(""), headers) } // IsShuttingDown tells whether the daemon is shutting down or not diff --git a/distribution/pull_v1.go b/distribution/pull_v1.go index 3e0cbdb46c..9c23a676b8 100644 --- a/distribution/pull_v1.go +++ b/distribution/pull_v1.go @@ -49,10 +49,10 @@ func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error { tr := transport.NewTransport( // TODO(tiborvass): was ReceiveTimeout registry.NewTransport(tlsConfig), - registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)..., + registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)..., ) client := registry.HTTPClient(tr) - v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders) + v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders) if err != nil { logrus.Debugf("Could not get v1 endpoint: %v", err) return fallbackError{err: err} diff --git a/distribution/push_v1.go b/distribution/push_v1.go index e9b1065f7a..d82d120718 100644 --- a/distribution/push_v1.go +++ b/distribution/push_v1.go @@ -38,10 +38,10 @@ func (p *v1Pusher) Push(ctx context.Context) error { tr := transport.NewTransport( // TODO(tiborvass): was NoTimeout registry.NewTransport(tlsConfig), - registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)..., + registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)..., ) client := registry.HTTPClient(tr) - v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders) + v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders) if err != nil { logrus.Debugf("Could not get v1 endpoint: %v", err) return fallbackError{err: err} diff --git a/distribution/registry.go b/distribution/registry.go index a4de299530..54e314bbd6 100644 --- a/distribution/registry.go +++ b/distribution/registry.go @@ -37,6 +37,8 @@ func (dcs dumbCredentialStore) SetRefreshToken(*url.URL, string, string) { // providing timeout settings and authentication support, and also verifies the // remote API version. func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) { + upstreamUA := dockerversion.GetUserAgentFromContext(ctx) + repoName := repoInfo.FullName() // If endpoint does not support CanonicalName, use the RemoteName instead if endpoint.TrimHostname { @@ -57,7 +59,7 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end DisableKeepAlives: true, } - modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders) + modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(upstreamUA), metaHeaders) authTransport := transport.NewTransport(base, modifiers...) challengeManager, foundVersion, err := registry.PingV2Registry(endpoint, authTransport) diff --git a/dockerversion/useragent.go b/dockerversion/useragent.go index 47cc2d491d..abc0ed7f46 100644 --- a/dockerversion/useragent.go +++ b/dockerversion/useragent.go @@ -1,15 +1,19 @@ package dockerversion import ( + "fmt" "runtime" + "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/useragent" + "golang.org/x/net/context" ) // DockerUserAgent is the User-Agent the Docker client uses to identify itself. -// It is populated from version information of different components. -func DockerUserAgent() string { +// In accordance with RFC 7231 (5.5.3) is of the form: +// [docker client's UA] UpstreamClient([upstream client's UA]) +func DockerUserAgent(upstreamUA string) string { httpVersion := make([]useragent.VersionInfo, 0, 6) httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: Version}) httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()}) @@ -20,5 +24,50 @@ func DockerUserAgent() string { httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS}) httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH}) - return useragent.AppendVersions("", httpVersion...) + dockerUA := useragent.AppendVersions("", httpVersion...) + if len(upstreamUA) > 0 { + ret := insertUpstreamUserAgent(upstreamUA, dockerUA) + return ret + } + return dockerUA +} + +// GetUserAgentFromContext returns the previously saved user-agent context stored in ctx, if one exists +func GetUserAgentFromContext(ctx context.Context) string { + var upstreamUA string + if ctx != nil { + var ki interface{} = ctx.Value(httputils.UAStringKey) + if ki != nil { + upstreamUA = ctx.Value(httputils.UAStringKey).(string) + } + } + return upstreamUA +} + +// escapeStr returns s with every rune in charsToEscape escaped by a backslash +func escapeStr(s string, charsToEscape string) string { + var ret string + for _, currRune := range s { + appended := false + for _, escapeableRune := range charsToEscape { + if currRune == escapeableRune { + ret += "\\" + string(currRune) + appended = true + break + } + } + if !appended { + ret += string(currRune) + } + } + return ret +} + +// insertUpstreamUserAgent adds the upstream client useragent to create a user-agent +// string of the form: +// $dockerUA UpstreamClient($upstreamUA) +func insertUpstreamUserAgent(upstreamUA string, dockerUA string) string { + charsToEscape := "();\\" //["\\", ";", "(", ")"]string + upstreamUAEscaped := escapeStr(upstreamUA, charsToEscape) + return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, upstreamUAEscaped) } diff --git a/integration-cli/docker_cli_registry_user_agent_test.go b/integration-cli/docker_cli_registry_user_agent_test.go new file mode 100644 index 0000000000..ebb2093bcb --- /dev/null +++ b/integration-cli/docker_cli_registry_user_agent_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "net/http" + "regexp" + + "github.com/go-check/check" +) + +// unescapeBackslashSemicolonParens unescapes \;() +func unescapeBackslashSemicolonParens(s string) string { + re := regexp.MustCompile("\\\\;") + ret := re.ReplaceAll([]byte(s), []byte(";")) + + re = regexp.MustCompile("\\\\\\(") + ret = re.ReplaceAll([]byte(ret), []byte("(")) + + re = regexp.MustCompile("\\\\\\)") + ret = re.ReplaceAll([]byte(ret), []byte(")")) + + re = regexp.MustCompile("\\\\\\\\") + ret = re.ReplaceAll([]byte(ret), []byte("\\")) + + return string(ret) +} + +func regexpCheckUA(c *check.C, ua string) { + re := regexp.MustCompile("(?P.+) UpstreamClient(?P.+)") + substrArr := re.FindStringSubmatch(ua) + + c.Assert(substrArr, check.HasLen, 3, check.Commentf("Expected 'UpstreamClient()' with upstream client UA")) + dockerUA := substrArr[1] + upstreamUAEscaped := substrArr[2] + + // check dockerUA looks correct + reDockerUA := regexp.MustCompile("^docker/[0-9A-Za-z+]") + bMatchDockerUA := reDockerUA.MatchString(dockerUA) + c.Assert(bMatchDockerUA, check.Equals, true, check.Commentf("Docker Engine User-Agent malformed")) + + // check upstreamUA looks correct + // Expecting something like: Docker-Client/1.11.0-dev (linux) + upstreamUA := unescapeBackslashSemicolonParens(upstreamUAEscaped) + reUpstreamUA := regexp.MustCompile("^\\(Docker-Client/[0-9A-Za-z+]") + bMatchUpstreamUA := reUpstreamUA.MatchString(upstreamUA) + c.Assert(bMatchUpstreamUA, check.Equals, true, check.Commentf("(Upstream) Docker Client User-Agent malformed")) +} + +// TestUserAgentPassThroughOnPull verifies that when an image is pulled from +// a registry, the registry should see a User-Agent string of the form +// [docker engine UA] UptreamClientSTREAM-CLIENT([client UA]) +func (s *DockerRegistrySuite) TestUserAgentPassThroughOnPull(c *check.C) { + reg, err := newTestRegistry(c) + c.Assert(err, check.IsNil) + expectUpstreamUA := false + + reg.registerHandler("/v2/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + var ua string + for k, v := range r.Header { + if k == "User-Agent" { + ua = v[0] + } + } + c.Assert(ua, check.Not(check.Equals), "", check.Commentf("No User-Agent found in request")) + if r.URL.Path == "/v2/busybox/manifests/latest" { + if expectUpstreamUA { + regexpCheckUA(c, ua) + } + } + }) + + repoName := fmt.Sprintf("%s/busybox", reg.hostport) + err = s.d.Start("--insecure-registry", reg.hostport, "--disable-legacy-registry=true") + c.Assert(err, check.IsNil) + + dockerfileName, cleanup, err := makefile(fmt.Sprintf("FROM %s/busybox", reg.hostport)) + c.Assert(err, check.IsNil, check.Commentf("Unable to create test dockerfile")) + defer cleanup() + + s.d.Cmd("build", "--file", dockerfileName, ".") + + s.d.Cmd("run", repoName) + s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport) + s.d.Cmd("tag", "busybox", repoName) + s.d.Cmd("push", repoName) + + expectUpstreamUA = true + s.d.Cmd("pull", repoName) +}