From 0e8e8f0f318656be80e34db9b5e390ffeef3fd0d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 13 Apr 2017 21:56:50 -0400 Subject: [PATCH] Add support for metrics plugins Allows for a plugin type that can be used to scrape metrics. This is useful because metrics are not neccessarily at a standard location... `--metrics-addr` must be set, and must currently be a TCP socket. Even if metrics are done via a unix socket, there's no guarentee where the socket may be located on the system, making bind-mounting such a socket into a container difficult (and racey, failure-prone on daemon restart). Metrics plugins side-step this issue by always listening on a unix socket and then bind-mounting that into a known path in the plugin container. Note there has been similar work in the past (and ultimately punted at the time) for consistent access to the Docker API from within a container. Why not add metrics to the Docker API and just provide a plugin with access to the Docker API? Certainly this can be useful, but gives a lot of control/access to a plugin that may only need the metrics. We can look at supporting API plugins separately for this reason. Signed-off-by: Brian Goff --- daemon/daemon.go | 11 +++ daemon/metrics.go | 66 +++++++++++++++++ daemon/metrics_unix.go | 86 ++++++++++++++++++++++ daemon/metrics_unsupported.go | 12 +++ docs/extend/config.md | 2 + docs/extend/plugins_metrics.md | 85 +++++++++++++++++++++ docs/reference/commandline/plugin_ls.md | 4 +- integration-cli/docker_cli_plugins_test.go | 23 ++++++ 8 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 daemon/metrics_unix.go create mode 100644 daemon/metrics_unsupported.go create mode 100644 docs/extend/plugins_metrics.md diff --git a/daemon/daemon.go b/daemon/daemon.go index 823fb04f9c..3f6f5684e2 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -106,6 +106,7 @@ type Daemon struct { defaultIsolation containertypes.Isolation // Default isolation mode on Windows clusterProvider cluster.Provider cluster Cluster + metricsPluginListener net.Listener machineMemory uint64 @@ -593,6 +594,12 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.PluginStore = pluginStore logger.RegisterPluginGetter(d.PluginStore) + metricsSockPath, err := d.listenMetricsSock() + if err != nil { + return nil, err + } + registerMetricsPluginCallback(d.PluginStore, metricsSockPath) + // Plugin system initialization should happen before restore. Do not change order. d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{ Root: filepath.Join(config.Root, "plugins"), @@ -821,6 +828,8 @@ func (daemon *Daemon) Shutdown() error { if daemon.configStore.LiveRestoreEnabled && daemon.containers != nil { // check if there are any running containers, if none we should do some cleanup if ls, err := daemon.Containers(&types.ContainerListOptions{}); len(ls) != 0 || err != nil { + // metrics plugins still need some cleanup + daemon.cleanupMetricsPlugins() return nil } } @@ -861,6 +870,8 @@ func (daemon *Daemon) Shutdown() error { daemon.DaemonLeavesCluster() } + daemon.cleanupMetricsPlugins() + // Shutdown plugins after containers and layerstore. Don't change the order. daemon.pluginShutdown() diff --git a/daemon/metrics.go b/daemon/metrics.go index dd67a0f71e..bf9e49d044 100644 --- a/daemon/metrics.go +++ b/daemon/metrics.go @@ -1,12 +1,19 @@ package daemon import ( + "path/filepath" "sync" + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/plugingetter" "github.com/docker/go-metrics" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) +const metricsPluginType = "MetricsCollector" + var ( containerActions metrics.LabeledTimer containerStates metrics.LabeledGauge @@ -106,3 +113,62 @@ func (ctr *stateCounter) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(ctr.desc, prometheus.GaugeValue, float64(paused), "paused") ch <- prometheus.MustNewConstMetric(ctr.desc, prometheus.GaugeValue, float64(stopped), "stopped") } + +func (d *Daemon) cleanupMetricsPlugins() { + ls := d.PluginStore.GetAllManagedPluginsByCap(metricsPluginType) + var wg sync.WaitGroup + wg.Add(len(ls)) + + for _, p := range ls { + go func() { + defer wg.Done() + pluginStopMetricsCollection(p) + }() + } + wg.Wait() + + if d.metricsPluginListener != nil { + d.metricsPluginListener.Close() + } +} + +type metricsPlugin struct { + plugingetter.CompatPlugin +} + +func (p metricsPlugin) sock() string { + return "metrics.sock" +} + +func (p metricsPlugin) sockBase() string { + return filepath.Join(p.BasePath(), "run", "docker") +} + +func pluginStartMetricsCollection(p plugingetter.CompatPlugin) error { + type metricsPluginResponse struct { + Err string + } + var res metricsPluginResponse + if err := p.Client().Call(metricsPluginType+".StartMetrics", nil, &res); err != nil { + return errors.Wrap(err, "could not start metrics plugin") + } + if res.Err != "" { + return errors.New(res.Err) + } + return nil +} + +func pluginStopMetricsCollection(p plugingetter.CompatPlugin) { + if err := p.Client().Call(metricsPluginType+".StopMetrics", nil, nil); err != nil { + logrus.WithError(err).WithField("name", p.Name()).Error("error stopping metrics collector") + } + + mp := metricsPlugin{p} + sockPath := filepath.Join(mp.sockBase(), mp.sock()) + if err := mount.Unmount(sockPath); err != nil { + if mounted, _ := mount.Mounted(sockPath); mounted { + logrus.WithError(err).WithField("name", p.Name()).WithField("socket", sockPath).Error("error unmounting metrics socket for plugin") + } + } + return +} diff --git a/daemon/metrics_unix.go b/daemon/metrics_unix.go new file mode 100644 index 0000000000..cda7355e8e --- /dev/null +++ b/daemon/metrics_unix.go @@ -0,0 +1,86 @@ +// +build !windows + +package daemon + +import ( + "net" + "net/http" + "os" + "path/filepath" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/plugingetter" + "github.com/docker/docker/pkg/plugins" + metrics "github.com/docker/go-metrics" + "github.com/pkg/errors" +) + +func (daemon *Daemon) listenMetricsSock() (string, error) { + path := filepath.Join(daemon.configStore.ExecRoot, "metrics.sock") + syscall.Unlink(path) + l, err := net.Listen("unix", path) + if err != nil { + return "", errors.Wrap(err, "error setting up metrics plugin listener") + } + + mux := http.NewServeMux() + mux.Handle("/metrics", metrics.Handler()) + go func() { + http.Serve(l, mux) + }() + daemon.metricsPluginListener = l + return path, nil +} + +func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) { + getter.Handle(metricsPluginType, func(name string, client *plugins.Client) { + // Use lookup since nothing in the system can really reference it, no need + // to protect against removal + p, err := getter.Get(name, metricsPluginType, plugingetter.Lookup) + if err != nil { + return + } + + mp := metricsPlugin{p} + sockBase := mp.sockBase() + if err := os.MkdirAll(sockBase, 0755); err != nil { + logrus.WithError(err).WithField("name", name).WithField("path", sockBase).Error("error creating metrics plugin base path") + return + } + + defer func() { + if err != nil { + os.RemoveAll(sockBase) + } + }() + + pluginSockPath := filepath.Join(sockBase, mp.sock()) + _, err = os.Stat(pluginSockPath) + if err == nil { + mount.Unmount(pluginSockPath) + } else { + logrus.WithField("path", pluginSockPath).Debugf("creating plugin socket") + f, err := os.OpenFile(pluginSockPath, os.O_CREATE, 0600) + if err != nil { + return + } + f.Close() + } + + if err := mount.Mount(sockPath, pluginSockPath, "none", "bind,ro"); err != nil { + logrus.WithError(err).WithField("name", name).Error("could not mount metrics socket to plugin") + return + } + + if err := pluginStartMetricsCollection(p); err != nil { + if err := mount.Unmount(pluginSockPath); err != nil { + if mounted, _ := mount.Mounted(pluginSockPath); mounted { + logrus.WithError(err).WithField("sock_path", pluginSockPath).Error("error unmounting metrics socket from plugin during cleanup") + } + } + logrus.WithError(err).WithField("name", name).Error("error while initializing metrics plugin") + } + }) +} diff --git a/daemon/metrics_unsupported.go b/daemon/metrics_unsupported.go new file mode 100644 index 0000000000..64dc1817a3 --- /dev/null +++ b/daemon/metrics_unsupported.go @@ -0,0 +1,12 @@ +// +build windows + +package daemon + +import "github.com/docker/docker/pkg/plugingetter" + +func registerMetricsPluginCallback(getter plugingetter.PluginGetter, sockPath string) { +} + +func (daemon *Daemon) listenMetricsSock() (string, error) { + return "", nil +} diff --git a/docs/extend/config.md b/docs/extend/config.md index c7444525fc..bb6c7f2ceb 100644 --- a/docs/extend/config.md +++ b/docs/extend/config.md @@ -61,6 +61,8 @@ Config provides the base accessible fields for working with V0 plugin format - **docker.logdriver/1.0** + - **docker.metricscollector/1.0** + - **`socket`** *string* socket is the name of the socket the engine should use to communicate with the plugins. diff --git a/docs/extend/plugins_metrics.md b/docs/extend/plugins_metrics.md new file mode 100644 index 0000000000..a86c7f22d2 --- /dev/null +++ b/docs/extend/plugins_metrics.md @@ -0,0 +1,85 @@ +--- +title: "Docker metrics collector plugins" +description: "Metrics plugins." +keywords: "Examples, Usage, plugins, docker, documentation, user guide, metrics" +--- + + + +# Metrics Collector Plugins + +Docker exposes internal metrics based on the prometheus format. Metrics plugins +enable accessing these metrics in a consistent way by providing a Unix +socket at a predefined path where the plugin can scrape the metrics. + +> **Note**: that while the plugin interface for metrics is non-experimental, the naming +of the metrics and metric labels is still considered experimental and may change +in a future version. + +## Creating a metrics plugin + +You must currently set `PropagatedMount` in the plugin `config.json` to +`/run/docker`. This allows the plugin to receive updated mounts +(the bind-mounted socket) from Docker after the plugin is already configured. + +## MetricsCollector protocol + +Metrics plugins must register as implementing the`MetricsCollector` interface +in `config.json`. + +On Unix platforms, the socket is located at `/run/docker/metrics.sock` in the +plugin's rootfs. + +`MetricsCollector` must implement two endpoints: + +### `MetricsCollector.StartMetrics` + +Signals to the plugin that the metrics socket is now available for scraping + +**Request** +```json +{} +``` + +The request has no playload. + +**Response** +```json +{ + "Err": "" +} +``` + +If an error occurred during this request, add an error message to the `Err` field +in the response. If no error then you can either send an empty response (`{}`) +or an empty value for the `Err` field. Errors will only be logged. + +### `MetricsCollector.StopMetrics` + +Signals to the plugin that the metrics socket is no longer available. +This may happen when the daemon is shutting down. + +**Request** +```json +{} +``` + +The request has no playload. + +**Response** +```json +{ + "Err": "" +} +``` + +If an error occurred during this request, add an error message to the `Err` field +in the response. If no error then you can either send an empty response (`{}`) +or an empty value for the `Err` field. Errors will only be logged. diff --git a/docs/reference/commandline/plugin_ls.md b/docs/reference/commandline/plugin_ls.md index 7808c79f6d..3ba29fee03 100644 --- a/docs/reference/commandline/plugin_ls.md +++ b/docs/reference/commandline/plugin_ls.md @@ -55,7 +55,7 @@ than one filter, then pass multiple flags (e.g., `--filter "foo=bar" --filter "b The currently supported filters are: * enabled (boolean - true or false, 0 or 1) -* capability (string - currently `volumedriver`, `networkdriver`, `ipamdriver`, or `authz`) +* capability (string - currently `volumedriver`, `networkdriver`, `ipamdriver`, `logdriver`, `metricscollector`, or `authz`) #### enabled @@ -65,7 +65,7 @@ The `enabled` filter matches on plugins enabled or disabled. The `capability` filter matches on plugin capabilities. One plugin might have multiple capabilities. Currently `volumedriver`, `networkdriver`, -`ipamdriver`, and `authz` are supported capabilities. +`ipamdriver`, `logdriver`, `metricscollector`, and `authz` are supported capabilities. ```bash $ docker plugin install --disable tiborvass/no-remove diff --git a/integration-cli/docker_cli_plugins_test.go b/integration-cli/docker_cli_plugins_test.go index 6644c6a809..46b0b26468 100644 --- a/integration-cli/docker_cli_plugins_test.go +++ b/integration-cli/docker_cli_plugins_test.go @@ -3,12 +3,14 @@ package main import ( "fmt" "io/ioutil" + "net/http" "os" "path/filepath" "strings" "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli" + "github.com/docker/docker/integration-cli/daemon" icmd "github.com/docker/docker/pkg/testutil/cmd" "github.com/go-check/check" ) @@ -455,3 +457,24 @@ func (s *DockerSuite) TestPluginUpgrade(c *check.C) { dockerCmd(c, "volume", "inspect", "bananas") dockerCmd(c, "run", "--rm", "-v", "bananas:/apple", "busybox", "sh", "-c", "ls -lh /apple/core") } + +func (s *DockerSuite) TestPluginMetricsCollector(c *check.C) { + testRequires(c, DaemonIsLinux, Network, SameHostDaemon, IsAmd64) + d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{}) + d.Start(c) + defer d.Stop(c) + + name := "cpuguy83/docker-metrics-plugin-test:latest" + r := cli.Docker(cli.Args("plugin", "install", "--grant-all-permissions", name), cli.Daemon(d)) + c.Assert(r.Error, checker.IsNil, check.Commentf(r.Combined())) + + // plugin lisens on localhost:19393 and proxies the metrics + resp, err := http.Get("http://localhost:19393/metrics") + c.Assert(err, checker.IsNil) + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + c.Assert(err, checker.IsNil) + // check that a known metric is there... don't epect this metric to change over time.. probably safe + c.Assert(string(b), checker.Contains, "container_actions") +}