Add support for metrics plugins

Allows for a plugin type that can be used to scrape metrics.
This is useful because metrics are not neccessarily at a standard
location... `--metrics-addr` must be set, and must currently be a TCP
socket.
Even if metrics are done via a unix socket, there's no guarentee where
the socket may be located on the system, making bind-mounting such a
socket into a container difficult (and racey, failure-prone on daemon
restart).

Metrics plugins side-step this issue by always listening on a unix
socket and then bind-mounting that into a known path in the plugin
container.

Note there has been similar work in the past (and ultimately punted at
the time) for consistent access to the Docker API from within a
container.

Why not add metrics to the Docker API and just provide a plugin with
access to the Docker API? Certainly this can be useful, but gives a lot
of control/access to a plugin that may only need the metrics. We can
look at supporting API plugins separately for this reason.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

daemon/daemon.go

@@ -106,6 +106,7 @@ type Daemon struct {
 	defaultIsolation          containertypes.Isolation // Default isolation mode on Windows
 	clusterProvider           cluster.Provider
 	cluster                   Cluster
+	metricsPluginListener     net.Listener
 
 	machineMemory uint64
 
@@ -593,6 +594,12 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
 	d.PluginStore = pluginStore
 	logger.RegisterPluginGetter(d.PluginStore)
 
+	metricsSockPath, err := d.listenMetricsSock()
+	if err != nil {
+		return nil, err
+	}
+	registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
+
 	// Plugin system initialization should happen before restore. Do not change order.
 	d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
 		Root:               filepath.Join(config.Root, "plugins"),
@@ -821,6 +828,8 @@ func (daemon *Daemon) Shutdown() error {
 	if daemon.configStore.LiveRestoreEnabled && daemon.containers != nil {
 		// check if there are any running containers, if none we should do some cleanup
 		if ls, err := daemon.Containers(&types.ContainerListOptions{}); len(ls) != 0 || err != nil {
+			// metrics plugins still need some cleanup
+			daemon.cleanupMetricsPlugins()
 			return nil
 		}
 	}
@@ -861,6 +870,8 @@ func (daemon *Daemon) Shutdown() error {
 		daemon.DaemonLeavesCluster()
 	}
 
+	daemon.cleanupMetricsPlugins()
+
 	// Shutdown plugins after containers and layerstore. Don't change the order.
 	daemon.pluginShutdown()
 

daemon/metrics.go

@@ -1,12 +1,19 @@
 package daemon
 
 import (
+	"path/filepath"
 	"sync"
 
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/go-metrics"
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const metricsPluginType = "MetricsCollector"
+
 var (
 	containerActions          metrics.LabeledTimer
 	containerStates           metrics.LabeledGauge
@@ -106,3 +113,62 @@ func (ctr *stateCounter) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(ctr.desc, prometheus.GaugeValue, float64(paused), "paused")
 	ch <- prometheus.MustNewConstMetric(ctr.desc, prometheus.GaugeValue, float64(stopped), "stopped")
 }
+
+func (d *Daemon) cleanupMetricsPlugins() {
+	ls := d.PluginStore.GetAllManagedPluginsByCap(metricsPluginType)
+	var wg sync.WaitGroup
+	wg.Add(len(ls))
+
+	for _, p := range ls {
+		go func() {
+			defer wg.Done()
+			pluginStopMetricsCollection(p)
+		}()
+	}
+	wg.Wait()
+
+	if d.metricsPluginListener != nil {
+		d.metricsPluginListener.Close()
+	}
+}
+
+type metricsPlugin struct {
+	plugingetter.CompatPlugin
+}
+
+func (p metricsPlugin) sock() string {
+	return "metrics.sock"
+}
+
+func (p metricsPlugin) sockBase() string {
+	return filepath.Join(p.BasePath(), "run", "docker")
+}
+
+func pluginStartMetricsCollection(p plugingetter.CompatPlugin) error {
+	type metricsPluginResponse struct {
+		Err string
+	}
+	var res metricsPluginResponse
+	if err := p.Client().Call(metricsPluginType+".StartMetrics", nil, &res); err != nil {
+		return errors.Wrap(err, "could not start metrics plugin")
+	}
+	if res.Err != "" {
+		return errors.New(res.Err)
+	}
+	return nil
+}
+
+func pluginStopMetricsCollection(p plugingetter.CompatPlugin) {
+	if err := p.Client().Call(metricsPluginType+".StopMetrics", nil, nil); err != nil {
+		logrus.WithError(err).WithField("name", p.Name()).Error("error stopping metrics collector")
+	}
+
+	mp := metricsPlugin{p}
+	sockPath := filepath.Join(mp.sockBase(), mp.sock())
+	if err := mount.Unmount(sockPath); err != nil {
+		if mounted, _ := mount.Mounted(sockPath); mounted {
+			logrus.WithError(err).WithField("name", p.Name()).WithField("socket", sockPath).Error("error unmounting metrics socket for plugin")
+		}
+	}
+	return
+}

daemon/metrics_unix.go

@@ -0,0 +1,86 @@
+// +build !windows
+
+package daemon
+
+import (
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/plugingetter"
+	"github.com/docker/docker/pkg/plugins"
+	metrics "github.com/docker/go-metrics"
+	"github.com/pkg/errors"
+)
+
+func (daemon *Daemon) listenMetricsSock() (string, error) {
+	path := filepath.Join(daemon.configStore.ExecRoot, "metrics.sock")
+	syscall.Unlink(path)
+	l, err := net.Listen("unix", path)
+	if err != nil {
+		return "", errors.Wrap(err, "error setting up metrics plugin listener")
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", metrics.Handler())
+	go func() {
+		http.Serve(l, mux)
+	}()
+	daemon.metricsPluginListener = l
+	return path, nil
+}
+
+func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) {
+	getter.Handle(metricsPluginType, func(name string, client *plugins.Client) {
+		// Use lookup since nothing in the system can really reference it, no need
+		// to protect against removal
+		p, err := getter.Get(name, metricsPluginType, plugingetter.Lookup)
+		if err != nil {
+			return
+		}
+
+		mp := metricsPlugin{p}
+		sockBase := mp.sockBase()
+		if err := os.MkdirAll(sockBase, 0755); err != nil {
+			logrus.WithError(err).WithField("name", name).WithField("path", sockBase).Error("error creating metrics plugin base path")
+			return
+		}
+
+		defer func() {
+			if err != nil {
+				os.RemoveAll(sockBase)
+			}
+		}()
+
+		pluginSockPath := filepath.Join(sockBase, mp.sock())
+		_, err = os.Stat(pluginSockPath)
+		if err == nil {
+			mount.Unmount(pluginSockPath)
+		} else {
+			logrus.WithField("path", pluginSockPath).Debugf("creating plugin socket")
+			f, err := os.OpenFile(pluginSockPath, os.O_CREATE, 0600)
+			if err != nil {
+				return
+			}
+			f.Close()
+		}
+
+		if err := mount.Mount(sockPath, pluginSockPath, "none", "bind,ro"); err != nil {
+			logrus.WithError(err).WithField("name", name).Error("could not mount metrics socket to plugin")
+			return
+		}
+
+		if err := pluginStartMetricsCollection(p); err != nil {
+			if err := mount.Unmount(pluginSockPath); err != nil {
+				if mounted, _ := mount.Mounted(pluginSockPath); mounted {
+					logrus.WithError(err).WithField("sock_path", pluginSockPath).Error("error unmounting metrics socket from plugin during cleanup")
+				}
+			}
+			logrus.WithError(err).WithField("name", name).Error("error while initializing metrics plugin")
+		}
+	})
+}

daemon/metrics_unsupported.go

@@ -0,0 +1,12 @@
+// +build windows
+
+package daemon
+
+import "github.com/docker/docker/pkg/plugingetter"
+
+func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) {
+}
+
+func (daemon *Daemon) listenMetricsSock() (string, error) {
+	return "", nil
+}

docs/extend/config.md

@@ -61,6 +61,8 @@ Config provides the base accessible fields for working with V0 plugin format
 
         - **docker.logdriver/1.0**
 
+        - **docker.metricscollector/1.0**
+
     - **`socket`** *string*
 
       socket is the name of the socket the engine should use to communicate with the plugins.

docs/extend/plugins_metrics.md

@@ -0,0 +1,85 @@
+---
+title: "Docker metrics collector plugins"
+description: "Metrics plugins."
+keywords: "Examples, Usage, plugins, docker, documentation, user guide, metrics"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# Metrics Collector Plugins
+
+Docker exposes internal metrics based on the prometheus format. Metrics plugins
+enable accessing these metrics in a consistent way by providing a Unix
+socket at a predefined path where the plugin can scrape the metrics.
+
+> **Note**: that while the plugin interface for metrics is non-experimental, the naming
+of the metrics and metric labels is still considered experimental and may change
+in a future version.
+
+## Creating a metrics plugin
+
+You must currently set `PropagatedMount` in the plugin `config.json` to
+`/run/docker`. This allows the plugin to receive updated mounts
+(the bind-mounted socket) from Docker after the plugin is already configured.
+
+## MetricsCollector protocol
+
+Metrics plugins must register as implementing the`MetricsCollector` interface
+in `config.json`.
+
+On Unix platforms, the socket is located at `/run/docker/metrics.sock` in the
+plugin's rootfs.
+
+`MetricsCollector` must implement two endpoints:
+
+### `MetricsCollector.StartMetrics`
+
+Signals to the plugin that the metrics socket is now available for scraping
+
+**Request**
+```json
+{}
+```
+
+The request has no playload.
+
+**Response**
+```json
+{
+	"Err": ""
+}
+```
+
+If an error occurred during this request, add an error message to the `Err` field
+in the response. If no error then you can either send an empty response (`{}`)
+or an empty value for the `Err` field. Errors will only be logged.
+
+### `MetricsCollector.StopMetrics`
+
+Signals to the plugin that the metrics socket is no longer available.
+This may happen when the daemon is shutting down.
+
+**Request**
+```json
+{}
+```
+
+The request has no playload.
+
+**Response**
+```json
+{
+	"Err": ""
+}
+```
+
+If an error occurred during this request, add an error message to the `Err` field
+in the response. If no error then you can either send an empty response (`{}`)
+or an empty value for the `Err` field. Errors will only be logged.

docs/reference/commandline/plugin_ls.md

@@ -55,7 +55,7 @@ than one filter, then pass multiple flags (e.g., `--filter "foo=bar" --filter "b
 The currently supported filters are:
 
 * enabled (boolean - true or false, 0 or 1)
-* capability (string - currently `volumedriver`, `networkdriver`, `ipamdriver`, or `authz`)
+* capability (string - currently `volumedriver`, `networkdriver`, `ipamdriver`, `logdriver`, `metricscollector`, or `authz`)
 
 #### enabled
 
@@ -65,7 +65,7 @@ The `enabled` filter matches on plugins enabled or disabled.
 
 The `capability` filter matches on plugin capabilities. One plugin
 might have multiple capabilities. Currently `volumedriver`, `networkdriver`,
-`ipamdriver`, and `authz` are supported capabilities.
+`ipamdriver`, `logdriver`, `metricscollector`, and `authz` are supported capabilities.
 
 ```bash
 $ docker plugin install --disable tiborvass/no-remove

integration-cli/docker_cli_plugins_test.go

@@ -3,12 +3,14 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli"
+	"github.com/docker/docker/integration-cli/daemon"
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
 	"github.com/go-check/check"
 )
@@ -455,3 +457,24 @@ func (s *DockerSuite) TestPluginUpgrade(c *check.C) {
 	dockerCmd(c, "volume", "inspect", "bananas")
 	dockerCmd(c, "run", "--rm", "-v", "bananas:/apple", "busybox", "sh", "-c", "ls -lh /apple/core")
 }
+
+func (s *DockerSuite) TestPluginMetricsCollector(c *check.C) {
+	testRequires(c, DaemonIsLinux, Network, SameHostDaemon, IsAmd64)
+	d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{})
+	d.Start(c)
+	defer d.Stop(c)
+
+	name := "cpuguy83/docker-metrics-plugin-test:latest"
+	r := cli.Docker(cli.Args("plugin", "install", "--grant-all-permissions", name), cli.Daemon(d))
+	c.Assert(r.Error, checker.IsNil, check.Commentf(r.Combined()))
+
+	// plugin lisens on localhost:19393 and proxies the metrics
+	resp, err := http.Get("http://localhost:19393/metrics")
+	c.Assert(err, checker.IsNil)
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	c.Assert(err, checker.IsNil)
+	// check that a known metric is there... don't epect this metric to change over time.. probably safe
+	c.Assert(string(b), checker.Contains, "container_actions")
+}