daemon/logger: Add logging driver for Google Cloud Logging
Signed-off-by: Mike Danese <mikedanese@google.com>
This commit is contained in:
Mike Danese
parent
123f22004b
commit
ed1b9fa07a
|
|
@ -397,6 +397,7 @@ __docker_complete_log_drivers() {
|
|||
awslogs
|
||||
etwlogs
|
||||
fluentd
|
||||
gcplogs
|
||||
gelf
|
||||
journald
|
||||
json-file
|
||||
|
|
@ -410,13 +411,14 @@ __docker_complete_log_options() {
|
|||
# see docs/reference/logging/index.md
|
||||
local awslogs_options="awslogs-region awslogs-group awslogs-stream"
|
||||
local fluentd_options="env fluentd-address labels tag"
|
||||
local gcplogs_options="env gcp-log-cmd gcp-project labels"
|
||||
local gelf_options="env gelf-address labels tag"
|
||||
local journald_options="env labels tag"
|
||||
local json_file_options="env labels max-file max-size"
|
||||
local syslog_options="syslog-address syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify syslog-facility tag"
|
||||
local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag"
|
||||
|
||||
local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
|
||||
local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
|
||||
|
||||
case $(__docker_value_of_option --log-driver) in
|
||||
'')
|
||||
|
|
@ -428,6 +430,9 @@ __docker_complete_log_options() {
|
|||
fluentd)
|
||||
COMPREPLY=( $( compgen -W "$fluentd_options" -S = -- "$cur" ) )
|
||||
;;
|
||||
gcplogs)
|
||||
COMPREPLY=( $( compgen -W "$gcplogs_options" -S = -- "$cur" ) )
|
||||
;;
|
||||
gelf)
|
||||
COMPREPLY=( $( compgen -W "$gelf_options" -S = -- "$cur" ) )
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -201,6 +201,7 @@ __docker_get_log_options() {
|
|||
|
||||
awslogs_options=("awslogs-region" "awslogs-group" "awslogs-stream")
|
||||
fluentd_options=("env" "fluentd-address" "labels" "tag")
|
||||
gcplogs_options=("env" "gcp-log-cmd" "gcp-project" "labels")
|
||||
gelf_options=("env" "gelf-address" "labels" "tag")
|
||||
journald_options=("env" "labels")
|
||||
json_file_options=("env" "labels" "max-file" "max-size")
|
||||
|
|
@ -209,6 +210,7 @@ __docker_get_log_options() {
|
|||
|
||||
[[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0
|
||||
[[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0
|
||||
[[ $log_driver = (gcplogs|all) ]] && _describe -t gcplogs-options "gcplogs options" gcplogs_options "$@" && ret=0
|
||||
[[ $log_driver = (gelf|all) ]] && _describe -t gelf-options "gelf options" gelf_options "$@" && ret=0
|
||||
[[ $log_driver = (journald|all) ]] && _describe -t journald-options "journald options" journald_options "$@" && ret=0
|
||||
[[ $log_driver = (json-file|all) ]] && _describe -t json-file-options "json-file options" json_file_options "$@" && ret=0
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
// therefore they register themselves to the logdriver factory.
|
||||
_ "github.com/docker/docker/daemon/logger/awslogs"
|
||||
_ "github.com/docker/docker/daemon/logger/fluentd"
|
||||
_ "github.com/docker/docker/daemon/logger/gcplogs"
|
||||
_ "github.com/docker/docker/daemon/logger/gelf"
|
||||
_ "github.com/docker/docker/daemon/logger/journald"
|
||||
_ "github.com/docker/docker/daemon/logger/jsonfilelog"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
package gcplogs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/daemon/logger"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/cloud/compute/metadata"
|
||||
"google.golang.org/cloud/logging"
|
||||
)
|
||||
|
||||
const (
|
||||
name = "gcplogs"
|
||||
|
||||
projectOptKey = "gcp-project"
|
||||
logLabelsKey = "labels"
|
||||
logEnvKey = "env"
|
||||
logCmdKey = "gcp-log-cmd"
|
||||
)
|
||||
|
||||
var (
|
||||
// The number of logs the gcplogs driver has dropped.
|
||||
droppedLogs uint64
|
||||
|
||||
onGCE = metadata.OnGCE()
|
||||
|
||||
// instance metadata populated from the metadata server if available
|
||||
projectID string
|
||||
zone string
|
||||
instanceName string
|
||||
instanceID string
|
||||
)
|
||||
|
||||
func init() {
|
||||
if onGCE {
|
||||
// These will fail on instances if the metadata service is
|
||||
// down or the client is compiled with an API version that
|
||||
// has been removed. Since these are not vital, let's ignore
|
||||
// them and make their fields in the dockeLogEntry ,omitempty
|
||||
projectID, _ = metadata.ProjectID()
|
||||
zone, _ = metadata.Zone()
|
||||
instanceName, _ = metadata.InstanceName()
|
||||
instanceID, _ = metadata.InstanceID()
|
||||
}
|
||||
|
||||
if err := logger.RegisterLogDriver(name, New); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type gcplogs struct {
|
||||
client *logging.Client
|
||||
instance *instanceInfo
|
||||
container *containerInfo
|
||||
}
|
||||
|
||||
type dockerLogEntry struct {
|
||||
Instance *instanceInfo `json:"instance,omitempty"`
|
||||
Container *containerInfo `json:"container,omitempty"`
|
||||
Data string `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
type instanceInfo struct {
|
||||
Zone string `json:"zone,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type containerInfo struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
ImageName string `json:"imageName,omitempty"`
|
||||
ImageID string `json:"imageId,omitempty"`
|
||||
Created time.Time `json:"created,omitempty"`
|
||||
Command string `json:"command,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// New creates a new logger that logs to Google Cloud Logging using the application
|
||||
// default credentials.
|
||||
//
|
||||
// See https://developers.google.com/identity/protocols/application-default-credentials
|
||||
func New(ctx logger.Context) (logger.Logger, error) {
|
||||
|
||||
var project string
|
||||
if projectID != "" {
|
||||
project = projectID
|
||||
}
|
||||
if projectID, found := ctx.Config[projectOptKey]; found {
|
||||
project = projectID
|
||||
}
|
||||
if project == "" {
|
||||
return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
|
||||
}
|
||||
|
||||
c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := c.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
|
||||
}
|
||||
|
||||
l := &gcplogs{
|
||||
client: c,
|
||||
container: &containerInfo{
|
||||
Name: ctx.ContainerName,
|
||||
ID: ctx.ContainerID,
|
||||
ImageName: ctx.ContainerImageName,
|
||||
ImageID: ctx.ContainerImageID,
|
||||
Created: ctx.ContainerCreated,
|
||||
Metadata: ctx.ExtraAttributes(nil),
|
||||
},
|
||||
}
|
||||
|
||||
if ctx.Config[logCmdKey] == "true" {
|
||||
l.container.Command = ctx.Command()
|
||||
}
|
||||
|
||||
if onGCE {
|
||||
l.instance = &instanceInfo{
|
||||
Zone: zone,
|
||||
Name: instanceName,
|
||||
ID: instanceID,
|
||||
}
|
||||
}
|
||||
|
||||
// The logger "overflows" at a rate of 10,000 logs per second and this
|
||||
// overflow func is called. We want to surface the error to the user
|
||||
// without overly spamming /var/log/docker.log so we log the first time
|
||||
// we overflow and every 1000th time after.
|
||||
c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
|
||||
if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
|
||||
logrus.Errorf("gcplogs driver has dropped %v logs", i)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs
|
||||
// driver doesn't take any arguments.
|
||||
func ValidateLogOpts(cfg map[string]string) error {
|
||||
for k := range cfg {
|
||||
switch k {
|
||||
case projectOptKey, logLabelsKey, logEnvKey, logCmdKey:
|
||||
default:
|
||||
return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *gcplogs) Log(m *logger.Message) error {
|
||||
return l.client.Log(logging.Entry{
|
||||
Time: m.Timestamp,
|
||||
Payload: &dockerLogEntry{
|
||||
Instance: l.instance,
|
||||
Container: l.container,
|
||||
Data: string(m.Line),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (l *gcplogs) Close() error {
|
||||
return l.client.Flush()
|
||||
}
|
||||
|
||||
func (l *gcplogs) Name() string {
|
||||
return name
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
<!--[metadata]>
|
||||
+++
|
||||
title = "Google Cloud Logging driver"
|
||||
description = "Describes how to use the Google Cloud Logging driver."
|
||||
keywords = ["gcplogs, google, docker, logging, driver"]
|
||||
[menu.main]
|
||||
parent = "smn_logging"
|
||||
weight = 2
|
||||
+++
|
||||
<![end-metadata]-->
|
||||
|
||||
# Google Cloud Logging driver
|
||||
|
||||
The Google Cloud Logging driver sends container logs to <a href="https://cloud.google.com/logging/docs/" target="_blank">Google Cloud
|
||||
Logging</a>.
|
||||
|
||||
## Usage
|
||||
|
||||
You can configure the default logging driver by passing the `--log-driver`
|
||||
option to the Docker daemon:
|
||||
|
||||
docker daemon --log-driver=gcplogs
|
||||
|
||||
You can set the logging driver for a specific container by using the
|
||||
`--log-driver` option to `docker run`:
|
||||
|
||||
docker run --log-driver=gcplogs ...
|
||||
|
||||
This log driver does not implement a reader so it is incompatible with
|
||||
`docker logs`.
|
||||
|
||||
If Docker detects that it is running in a Google Cloud Project, it will discover configuration
|
||||
from the <a href="https://cloud.google.com/compute/docs/metadata" target="_blank">instance metadata service</a>.
|
||||
Otherwise, the user must specify which project to log to using the `--gcp-project`
|
||||
log option and Docker will attempt to obtain credentials from the
|
||||
<a href="https://developers.google.com/identity/protocols/application-default-credentials" target="_blank">Google Application Default Credential</a>.
|
||||
The `--gcp-project` takes precedence over information discovered from the metadata server
|
||||
so a Docker daemon running in a Google Cloud Project can be overriden to log to a different
|
||||
Google Cloud Project using `--gcp-project`.
|
||||
|
||||
## gcplogs options
|
||||
|
||||
You can use the `--log-opt NAME=VALUE` flag to specify these additional Google
|
||||
Cloud Logging driver options:
|
||||
|
||||
| Option | Required | Description |
|
||||
|-----------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `gcp-project` | optional | Which GCP project to log to. Defaults to discovering this value from the GCE metadata service. |
|
||||
| `gcp-log-cmd` | optional | Whether to log the command that the container was started with. Defaults to false. |
|
||||
| `labels` | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. |
|
||||
| `env` | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. |
|
||||
|
||||
If there is collision between `label` and `env` keys, the value of the `env`
|
||||
takes precedence. Both options add additional fields to the attributes of a
|
||||
logging message.
|
||||
|
||||
Below is an example of the logging options required to log to the default
|
||||
logging destination which is discovered by querying the GCE metadata server.
|
||||
|
||||
docker run --log-driver=gcplogs \
|
||||
--log-opt labels=location
|
||||
--log-opt env=TEST
|
||||
--log-opt gcp-log-cmd=true
|
||||
--env "TEST=false"
|
||||
--label location=west
|
||||
your/application
|
||||
|
||||
This configuration also directs the driver to include in the payload the label
|
||||
`location`, the environment variable `ENV`, and the command used to start the
|
||||
container.
|
||||
|
|
@ -27,6 +27,7 @@ container's logging driver. The following options are supported:
|
|||
| `awslogs` | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs. |
|
||||
| `splunk` | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector. |
|
||||
| `etwlogs` | ETW logging driver for Docker on Windows. Writes log messages as ETW events. |
|
||||
| `gcplogs` | Google Cloud Logging driver for Docker. Writes log messages to Google Cloud Logging. |
|
||||
|
||||
The `docker logs`command is available only for the `json-file` and `journald`
|
||||
logging drivers.
|
||||
|
|
@ -213,4 +214,14 @@ as an ETW event. An ETW listener can then be created to listen for these events.
|
|||
|
||||
For detailed information on working with this logging driver, see [the ETW logging driver](etwlogs.md) reference documentation.
|
||||
|
||||
## Google Cloud Logging
|
||||
|
||||
The Google Cloud Logging driver supports the following options:
|
||||
|
||||
--log-opt gcp-project=<gcp_projext>
|
||||
--log-opt labels=<label1>,<label2>
|
||||
--log-opt env=<envvar1>,<envvar2>
|
||||
--log-opt log-cmd=true
|
||||
|
||||
For detailed information about working with this logging driver, see the [Google Cloud Logging driver](gcplogs.md).
|
||||
reference documentation.
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ millions of trillions.
|
|||
Add link to another container in the form of <name or id>:alias or just
|
||||
<name or id> in which case the alias will match the name.
|
||||
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
|
||||
Logging driver for container. Default is defined by daemon `--log-driver` flag.
|
||||
**Warning**: the `docker logs` command works only for the `json-file` and
|
||||
`journald` logging drivers.
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ unix://[/path/to/socket] to use.
|
|||
**--label**="[]"
|
||||
Set key=value labels to the daemon (displayed in `docker info`)
|
||||
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
|
||||
Default driver for container logs. Default is `json-file`.
|
||||
**Warning**: `docker logs` command works only for `json-file` logging driver.
|
||||
|
||||
|
|
|
|||
|
|
@ -320,7 +320,7 @@ container can access the exposed port via a private networking interface. Docker
|
|||
will set some environment variables in the client container to help indicate
|
||||
which interface and port to use.
|
||||
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
|
||||
**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
|
||||
Logging driver for container. Default is defined by daemon `--log-driver` flag.
|
||||
**Warning**: the `docker logs` command works only for the `json-file` and
|
||||
`journald` logging drivers.
|
||||
|
|
|
|||
Loading…
Reference in New Issue