Add http(s) proxy properties to daemon configuration

This allows configuring the daemon's proxy server through the daemon.json con-
figuration file or command-line flags configuration file, in addition to the
existing option (through environment variables).

Configuring environment variables on Windows to configure a service is more
complicated than on Linux, and adding alternatives for this to the daemon con-
figuration makes the configuration more transparent and easier to use.

The configuration as set through command-line flags or through the daemon.json
configuration file takes precedence over env-vars in the daemon's environment,
which allows the daemon to use a different proxy. If both command-line flags
and a daemon.json configuration option is set, an error is produced when starting
the daemon.

Note that this configuration is not "live reloadable" due to Golang's use of
`sync.Once()` for proxy configuration, which means that changing the proxy
configuration requires a restart of the daemon (reload / SIGHUP will not update
the configuration.

With this patch:

    cat /etc/docker/daemon.json
    {
        "http-proxy": "http://proxytest.example.com:80",
        "https-proxy": "https://proxytest.example.com:443"
    }

    docker pull busybox
    Using default tag: latest
    Error response from daemon: Get "https://registry-1.docker.io/v2/": proxyconnect tcp: dial tcp: lookup proxytest.example.com on 127.0.0.11:53: no such host

    docker build .
    Sending build context to Docker daemon  89.28MB
    Step 1/3 : FROM golang:1.16-alpine AS base
    Get "https://registry-1.docker.io/v2/": proxyconnect tcp: dial tcp: lookup proxytest.example.com on 127.0.0.11:53: no such host

Integration tests were added to test the behavior:

- verify that the configuration through all means are used (env-var,
  command-line flags, damon.json), and used in the expected order of
  preference.
- verify that conflicting options produce an error.

Signed-off-by: Anca Iordache <anca.iordache@docker.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

cmd/dockerd/config.go

@@ -101,6 +101,10 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 
 	flags.StringVar(&conf.DefaultRuntime, "default-runtime", config.StockRuntimeName, "Default OCI runtime for containers")
 
+	flags.StringVar(&conf.HTTPProxy, "http-proxy", "", "HTTP proxy URL to use for outgoing traffic")
+	flags.StringVar(&conf.HTTPSProxy, "https-proxy", "", "HTTPS proxy URL to use for outgoing traffic")
+	flags.StringVar(&conf.NoProxy, "no-proxy", "", "Comma-separated list of hosts or IP addresses for which the proxy is skipped")
+
 	return nil
 }
 

cmd/dockerd/daemon.go

@@ -87,6 +87,8 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return nil
 	}
 
+	configureProxyEnv(cli.Config)
+
 	warnOnDeprecatedConfigOptions(cli.Config)
 
 	if err := configureDaemonLogs(cli.Config); err != nil {
@@ -779,3 +781,29 @@ func configureDaemonLogs(conf *config.Config) error {
 	})
 	return nil
 }
+
+func configureProxyEnv(conf *config.Config) {
+	if p := conf.HTTPProxy; p != "" {
+		overrideProxyEnv("HTTP_PROXY", p)
+		overrideProxyEnv("http_proxy", p)
+	}
+	if p := conf.HTTPSProxy; p != "" {
+		overrideProxyEnv("HTTPS_PROXY", p)
+		overrideProxyEnv("https_proxy", p)
+	}
+	if p := conf.NoProxy; p != "" {
+		overrideProxyEnv("NO_PROXY", p)
+		overrideProxyEnv("no_proxy", p)
+	}
+}
+
+func overrideProxyEnv(name, val string) {
+	if oldVal := os.Getenv(name); oldVal != "" && oldVal != val {
+		logrus.WithFields(logrus.Fields{
+			"name":      name,
+			"old-value": oldVal,
+			"new-value": val,
+		}).Warn("overriding existing proxy variable with value from configuration")
+	}
+	_ = os.Setenv(name, val)
+}

daemon/config/config.go

@@ -166,6 +166,7 @@ type CommonConfig struct {
 	ExecRoot              string                    `json:"exec-root,omitempty"`
 	SocketGroup           string                    `json:"group,omitempty"`
 	CorsHeaders           string                    `json:"api-cors-header,omitempty"`
+	ProxyConfig
 
 	// TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests
 	// when pushing to a registry which does not support schema 2. This field is marked as
@@ -276,6 +277,13 @@ type CommonConfig struct {
 	DefaultRuntime string `json:"default-runtime,omitempty"`
 }
 
+// ProxyConfig holds the proxy-configuration for the daemon.
+type ProxyConfig struct {
+	HTTPProxy  string `json:"http-proxy,omitempty"`
+	HTTPSProxy string `json:"https-proxy,omitempty"`
+	NoProxy    string `json:"no-proxy,omitempty"`
+}
+
 // IsValueSet returns true if a configuration value
 // was explicitly set in the configuration file.
 func (conf *Config) IsValueSet(name string) bool {

daemon/info.go

@@ -63,9 +63,9 @@ func (daemon *Daemon) SystemInfo() *types.Info {
 		Labels:             daemon.configStore.Labels,
 		ExperimentalBuild:  daemon.configStore.Experimental,
 		ServerVersion:      dockerversion.Version,
-		HTTPProxy:          config.MaskCredentials(getEnvAny("HTTP_PROXY", "http_proxy")),
+		HTTPProxy:          config.MaskCredentials(getConfigOrEnv(daemon.configStore.HTTPProxy, "HTTP_PROXY", "http_proxy")),
-		HTTPSProxy:         config.MaskCredentials(getEnvAny("HTTPS_PROXY", "https_proxy")),
+		HTTPSProxy:         config.MaskCredentials(getConfigOrEnv(daemon.configStore.HTTPSProxy, "HTTPS_PROXY", "https_proxy")),
-		NoProxy:            getEnvAny("NO_PROXY", "no_proxy"),
+		NoProxy:            getConfigOrEnv(daemon.configStore.NoProxy, "NO_PROXY", "no_proxy"),
 		LiveRestoreEnabled: daemon.configStore.LiveRestoreEnabled,
 		Isolation:          daemon.defaultIsolation,
 	}
@@ -296,3 +296,10 @@ func getEnvAny(names ...string) string {
 	}
 	return ""
 }
+
+func getConfigOrEnv(config string, env ...string) string {
+	if config != "" {
+		return config
+	}
+	return getEnvAny(env...)
+}

integration/daemon/daemon_test.go

@@ -1,16 +1,22 @@
 package daemon // import "github.com/docker/docker/integration/daemon"
 
 import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"testing"
 
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/testutil/daemon"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/env"
 	"gotest.tools/v3/skip"
 )
 
@@ -146,3 +152,150 @@ func TestConfigDaemonSeccompProfiles(t *testing.T) {
 		})
 	}
 }
+
+func TestDaemonProxy(t *testing.T) {
+	skip.If(t, runtime.GOOS == "windows", "cannot start multiple daemons on windows")
+
+	var received string
+	proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		received = r.Host
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte("OK"))
+	}))
+	defer proxyServer.Close()
+
+	// Configure proxy through env-vars
+	t.Run("environment variables", func(t *testing.T) {
+		defer env.Patch(t, "HTTP_PROXY", proxyServer.URL)()
+		defer env.Patch(t, "HTTPS_PROXY", proxyServer.URL)()
+		defer env.Patch(t, "NO_PROXY", "example.com")()
+
+		d := daemon.New(t)
+		c := d.NewClientT(t)
+		defer func() { _ = c.Close() }()
+		ctx := context.Background()
+		d.Start(t)
+
+		_, err := c.ImagePull(ctx, "example.org:5000/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5000")
+
+		// Test NoProxy: example.com should not hit the proxy, and "received" variable should not be changed.
+		_, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5000", "should not have used proxy")
+
+		info := d.Info(t)
+		assert.Equal(t, info.HTTPProxy, proxyServer.URL)
+		assert.Equal(t, info.HTTPSProxy, proxyServer.URL)
+		assert.Equal(t, info.NoProxy, "example.com")
+		d.Stop(t)
+	})
+
+	// Configure proxy through command-line flags
+	t.Run("command-line options", func(t *testing.T) {
+		defer env.Patch(t, "HTTP_PROXY", "http://from-env-http.invalid")()
+		defer env.Patch(t, "http_proxy", "http://from-env-http.invalid")()
+		defer env.Patch(t, "HTTPS_PROXY", "https://from-env-https.invalid")()
+		defer env.Patch(t, "https_proxy", "https://from-env-http.invalid")()
+		defer env.Patch(t, "NO_PROXY", "ignore.invalid")()
+		defer env.Patch(t, "no_proxy", "ignore.invalid")()
+
+		d := daemon.New(t)
+		d.Start(t, "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com")
+
+		logs, err := d.ReadLogFile()
+		assert.NilError(t, err)
+		assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration"))
+		for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
+			assert.Assert(t, is.Contains(string(logs), "name="+v))
+		}
+
+		c := d.NewClientT(t)
+		defer func() { _ = c.Close() }()
+		ctx := context.Background()
+
+		_, err = c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5001")
+
+		// Test NoProxy: example.com should not hit the proxy, and "received" variable should not be changed.
+		_, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5001", "should not have used proxy")
+
+		info := d.Info(t)
+		assert.Equal(t, info.HTTPProxy, proxyServer.URL)
+		assert.Equal(t, info.HTTPSProxy, proxyServer.URL)
+		assert.Equal(t, info.NoProxy, "example.com")
+
+		d.Stop(t)
+	})
+
+	// Configure proxy through configuration file
+	t.Run("configuration file", func(t *testing.T) {
+		defer env.Patch(t, "HTTP_PROXY", "http://from-env-http.invalid")()
+		defer env.Patch(t, "http_proxy", "http://from-env-http.invalid")()
+		defer env.Patch(t, "HTTPS_PROXY", "https://from-env-https.invalid")()
+		defer env.Patch(t, "https_proxy", "https://from-env-http.invalid")()
+		defer env.Patch(t, "NO_PROXY", "ignore.invalid")()
+		defer env.Patch(t, "no_proxy", "ignore.invalid")()
+
+		d := daemon.New(t)
+		c := d.NewClientT(t)
+		defer func() { _ = c.Close() }()
+		ctx := context.Background()
+
+		configFile := filepath.Join(d.RootDir(), "daemon.json")
+		configJSON := fmt.Sprintf(`{"http-proxy":%[1]q, "https-proxy": %[1]q, "no-proxy": "example.com"}`, proxyServer.URL)
+		assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644))
+
+		d.Start(t, "--config-file", configFile)
+
+		logs, err := d.ReadLogFile()
+		assert.NilError(t, err)
+		assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration"))
+		for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
+			assert.Assert(t, is.Contains(string(logs), "name="+v))
+		}
+
+		_, err = c.ImagePull(ctx, "example.org:5002/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5002")
+
+		// Test NoProxy: example.com should not hit the proxy, and "received" variable should not be changed.
+		_, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{})
+		assert.ErrorContains(t, err, "", "pulling should have failed")
+		assert.Equal(t, received, "example.org:5002", "should not have used proxy")
+
+		info := d.Info(t)
+		assert.Equal(t, info.HTTPProxy, proxyServer.URL)
+		assert.Equal(t, info.HTTPSProxy, proxyServer.URL)
+		assert.Equal(t, info.NoProxy, "example.com")
+
+		d.Stop(t)
+	})
+
+	// Conflicting options (passed both through command-line options and config file)
+	t.Run("conflicting options", func(t *testing.T) {
+		const (
+			proxyRawURL = "https://myuser:mypassword@example.org"
+		)
+
+		d := daemon.New(t)
+
+		configFile := filepath.Join(d.RootDir(), "daemon.json")
+		configJSON := fmt.Sprintf(`{"http-proxy":%[1]q, "https-proxy": %[1]q, "no-proxy": "example.com"}`, proxyRawURL)
+		assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644))
+
+		err := d.StartWithError("--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com", "--config-file", configFile, "--validate")
+		assert.ErrorContains(t, err, "daemon exited during startup")
+		logs, err := d.ReadLogFile()
+		assert.NilError(t, err)
+		expected := fmt.Sprintf(
+			`the following directives are specified both as a flag and in the configuration file: http-proxy: (from flag: %[1]s, from file: %[1]s), https-proxy: (from flag: %[1]s, from file: %[1]s), no-proxy: (from flag: example.com, from file: example.com)`,
+			proxyRawURL,
+		)
+		assert.Assert(t, is.Contains(string(logs), expected))
+	})
+}