From 69a75c673ccf0aebf19c7b350243f1b0cefad8be Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Fri, 18 Jul 2014 18:48:19 +0000 Subject: [PATCH] Add daemon flag to specify public registry mirrors Adds support for a --registry-mirror=scheme://[:port] daemon flag. The flag may be present multiple times. If provided, mirrors are prepended to the list of endpoints used for image pull. Note that only mirrors of the public index.docker.io registry are supported, and image/tag resolution is still performed via the official index. Docker-DCO-1.1-Signed-off-by: Tim Smith (github: timbot) --- daemon/config.go | 2 + daemon/daemon.go | 2 +- docs/man/docker.1.md | 3 + docs/sources/articles.md | 1 + docs/sources/articles/registry_mirror.md | 83 +++++++++++++++++++++++ docs/sources/reference/commandline/cli.md | 1 + graph/pull.go | 42 ++++++++---- graph/tags.go | 4 +- graph/tags_unit_test.go | 2 +- opts/opts.go | 23 +++++++ 10 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 docs/sources/articles/registry_mirror.md diff --git a/daemon/config.go b/daemon/config.go index 1ca8086933..07d41153a3 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -23,6 +23,7 @@ type Config struct { AutoRestart bool Dns []string DnsSearch []string + Mirrors []string EnableIptables bool EnableIpForward bool DefaultIp net.IP @@ -60,6 +61,7 @@ func (config *Config) InstallFlags() { // FIXME: why the inconsistency between "hosts" and "sockets"? opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers") opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains") + opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror") } func GetDefaultNetworkMtu() int { diff --git a/daemon/daemon.go b/daemon/daemon.go index fab28110c6..f24745901c 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -791,7 +791,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } diff --git a/docs/man/docker.1.md b/docs/man/docker.1.md index 3932097255..db6b3e3051 100644 --- a/docs/man/docker.1.md +++ b/docs/man/docker.1.md @@ -64,6 +64,9 @@ unix://[/path/to/socket] to use. **-p**="" Path to use for daemon PID file. Default is `/var/run/docker.pid` +**--registry-mirror=:// + Prepend a registry mirror to be used for image pulls. May be specified multiple times. + **-s**="" Force the Docker runtime to use a specific storage driver. diff --git a/docs/sources/articles.md b/docs/sources/articles.md index 325037b4ce..37f2cd80f1 100644 --- a/docs/sources/articles.md +++ b/docs/sources/articles.md @@ -12,3 +12,4 @@ - [Automatically Start Containers](host_integration/) - [Link via an Ambassador Container](ambassador_pattern_linking/) - [Increase a Boot2Docker Volume](b2d_volume_resize/) + - [Run a Local Registry Mirror](registry_mirror/) diff --git a/docs/sources/articles/registry_mirror.md b/docs/sources/articles/registry_mirror.md new file mode 100644 index 0000000000..2e6bfd79c6 --- /dev/null +++ b/docs/sources/articles/registry_mirror.md @@ -0,0 +1,83 @@ +page_title: Run a local registry mirror +page_description: How to set up and run a local registry mirror +page_keywords: docker, registry, mirror, examples + +# Run a local registry mirror + +## Why? + +If you have multiple instances of Docker running in your environment +(e.g., multiple physical or virtual machines, all running the Docker +daemon), each time one of them requires an image that it doesn't have +it will go out to the internet and fetch it from the public Docker +registry. By running a local registry mirror, you can keep most of the +image fetch traffic on your local network. + +## How does it work? + +The first time you request an image from your local registry mirror, +it pulls the image from the public Docker registry and stores it locally +before handing it back to you. On subsequent requests, the local registry +mirror is able to serve the image from its own storage. + +## How do I set up a local registry mirror? + +There are two steps to set up and use a local registry mirror. + +### Step 1: Configure your Docker daemons to use the local registry mirror + +You will need to pass the `--registry-mirror` option to your Docker daemon on +startup: + + docker --registry-mirror=http:// -d + +For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run: + + docker --registry-mirror=http://10.0.0.2:5000 -d + +**NOTE:** +Depending on your local host setup, you may be able to add the +`--registry-mirror` options to the `DOCKER_OPTS` variable in +`/etc/defaults/docker`. + +### Step 2: Run the local registry mirror + +You will need to start a local registry mirror service. The +[`registry` image](https://registry.hub.docker.com/_/registry/) provides this +functionality. For example, to run a local registry mirror that serves on +port `5000` and mirrors the content at `registry-1.docker.io`: + + docker run -p 5000:5000 \ + -e STANDALONE=false \ + -e MIRROR_SOURCE=https://registry-1.docker.io \ + -e MIRROR_SOURCE_INDEX=https://index.docker.io registry + +## Test it out + +With your mirror running, pull an image that you haven't pulled before (using +`time` to time it): + + $ time docker pull node:latest + Pulling repository node + [...] + + real 1m14.078s + user 0m0.176s + sys 0m0.120s + +Now, remove the image from your local machine: + + $ docker rmi node:latest + +Finally, re-pull the image: + + $ time docker pull node:latest + Pulling repository node + [...] + + real 0m51.376s + user 0m0.120s + sys 0m0.116s + +The second time around, the local registry mirror served the image from storage, +avoiding a trip out to the internet to refetch it. diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 714116ddc5..286ba4a7a8 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -71,6 +71,7 @@ expect an integer, and they can only be specified once. --mtu=0 Set the containers network MTU if no value is provided: default to the default route MTU or 1500 if no default route is available -p, --pidfile="/var/run/docker.pid" Path to use for daemon PID file + --registry-mirror=[] Specify a preferred Docker registry mirror -s, --storage-driver="" Force the Docker runtime to use a specific storage driver --selinux-enabled=false Enable selinux support. SELinux does not presently support the BTRFS storage driver --storage-opt=[] Set storage driver options diff --git a/graph/pull.go b/graph/pull.go index 1a4e516207..600e524d81 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -25,6 +25,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { sf = utils.NewStreamFormatter(job.GetenvBool("json")) authConfig = ®istry.AuthConfig{} metaHeaders map[string][]string + mirrors []string ) if len(job.Args) > 1 { tag = job.Args[1] @@ -64,16 +65,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if endpoint == registry.IndexServerAddress() { // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" localName = remoteName + + // Use provided mirrors, if any + mirrors = s.mirrors } - if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil { + if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil { return job.Error(err) } return engine.StatusOK } -func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error { +func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(remoteName) @@ -153,17 +157,31 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil)) success := false var lastErr error - for _, ep := range repoData.Endpoints { - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil)) - if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - // It's not ideal that only the last error is returned, it would be better to concatenate the errors. - // As the error is also given to the output stream the user will see the error. - lastErr = err - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil)) - continue + if mirrors != nil { + for _, ep := range mirrors { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil)) + if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + // Don't report errors when pulling from mirrors. + log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err) + continue + } + success = true + break + } + } + if !success { + for _, ep := range repoData.Endpoints { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil)) + if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + // It's not ideal that only the last error is returned, it would be better to concatenate the errors. + // As the error is also given to the output stream the user will see the error. + lastErr = err + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil)) + continue + } + success = true + break } - success = true - break } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, localName, lastErr) diff --git a/graph/tags.go b/graph/tags.go index eee3be9f2f..573b89e286 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -20,6 +20,7 @@ const DEFAULTTAG = "latest" type TagStore struct { path string graph *Graph + mirrors []string Repositories map[string]Repository sync.Mutex // FIXME: move push/pull-related fields @@ -48,7 +49,7 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -56,6 +57,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { store := &TagStore{ path: abspath, graph: graph, + mirrors: mirrors, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 36cda68a2b..e8cedb46c3 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -52,7 +52,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil) if err != nil { t.Fatal(err) } diff --git a/opts/opts.go b/opts/opts.go index 65806f3698..0ed0f62bcd 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -3,6 +3,7 @@ package opts import ( "fmt" "net" + "net/url" "os" "path/filepath" "regexp" @@ -33,6 +34,10 @@ func IPVar(value *net.IP, names []string, defaultValue, usage string) { flag.Var(NewIpOpt(value, defaultValue), names, usage) } +func MirrorListVar(values *[]string, names []string, usage string) { + flag.Var(newListOptsRef(values, ValidateMirror), names, usage) +} + // ListOpts type type ListOpts struct { values *[]string @@ -190,3 +195,21 @@ func validateDomain(val string) (string, error) { } return "", fmt.Errorf("%s is not a valid domain", val) } + +// Validates an HTTP(S) registry mirror +func ValidateMirror(val string) (string, error) { + uri, err := url.Parse(val) + if err != nil { + return "", fmt.Errorf("%s is not a valid URI", val) + } + + if uri.Scheme != "http" && uri.Scheme != "https" { + return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme) + } + + if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" { + return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") + } + + return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil +}