Merge pull request #15384 from phil-monroe/15058-include-name-in-syslog-tag

log driver - Interpolate fields into log tag

daemon/logger/context.go

@@ -0,0 +1,72 @@
+package logger
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// Context provides enough information for a logging driver to do its function.
+type Context struct {
+	Config              map[string]string
+	ContainerID         string
+	ContainerName       string
+	ContainerEntrypoint string
+	ContainerArgs       []string
+	ContainerImageID    string
+	ContainerImageName  string
+	ContainerCreated    time.Time
+	LogPath             string
+}
+
+// Hostname returns the hostname from the underlying OS.
+func (ctx *Context) Hostname() (string, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return "", fmt.Errorf("logger: can not resolve hostname: %v", err)
+	}
+	return hostname, nil
+}
+
+// Command returns the command that the container being logged was
+// started with. The Entrypoint is prepended to the container
+// arguments.
+func (ctx *Context) Command() string {
+	terms := []string{ctx.ContainerEntrypoint}
+	for _, arg := range ctx.ContainerArgs {
+		terms = append(terms, arg)
+	}
+	command := strings.Join(terms, " ")
+	return command
+}
+
+// ID Returns the Container ID shortened to 12 characters.
+func (ctx *Context) ID() string {
+	return ctx.ContainerID[:12]
+}
+
+// FullID is an alias of ContainerID.
+func (ctx *Context) FullID() string {
+	return ctx.ContainerID
+}
+
+// Name returns the ContainerName without a preceding '/'.
+func (ctx *Context) Name() string {
+	return ctx.ContainerName[1:]
+}
+
+// ImageID returns the ContainerImageID shortened to 12 characters.
+func (ctx *Context) ImageID() string {
+	return ctx.ContainerImageID[:12]
+}
+
+// ImageFullID is an alias of ContainerID.
+func (ctx *Context) ImageFullID() string {
+	return ctx.ContainerImageID
+}
+
+// ImageName is an alias of ContainerImageName
+func (ctx *Context) ImageName() string {
+	return ctx.ContainerImageName
+}

daemon/logger/factory.go

@@ -2,10 +2,7 @@ package logger
 
 import (
 	"fmt"
-	"os"
-	"strings"
 	"sync"
-	"time"
 )
 
 // Creator builds a logging driver instance with given context.
@@ -15,40 +12,6 @@ type Creator func(Context) (Logger, error)
 // logging implementation.
 type LogOptValidator func(cfg map[string]string) error
 
-// Context provides enough information for a logging driver to do its function.
-type Context struct {
-	Config              map[string]string
-	ContainerID         string
-	ContainerName       string
-	ContainerEntrypoint string
-	ContainerArgs       []string
-	ContainerImageID    string
-	ContainerImageName  string
-	ContainerCreated    time.Time
-	LogPath             string
-}
-
-// Hostname returns the hostname from the underlying OS.
-func (ctx *Context) Hostname() (string, error) {
-	hostname, err := os.Hostname()
-	if err != nil {
-		return "", fmt.Errorf("logger: can not resolve hostname: %v", err)
-	}
-	return hostname, nil
-}
-
-// Command returns the command that the container being logged was
-// started with. The Entrypoint is prepended to the container
-// arguments.
-func (ctx *Context) Command() string {
-	terms := []string{ctx.ContainerEntrypoint}
-	for _, arg := range ctx.ContainerArgs {
-		terms = append(terms, arg)
-	}
-	command := strings.Join(terms, " ")
-	return command
-}
-
 type logdriverFactory struct {
 	registry     map[string]Creator
 	optValidator map[string]LogOptValidator

daemon/logger/fluentd/fluentd.go

@@ -3,16 +3,15 @@
 package fluentd
 
 import (
-	"bytes"
 	"fmt"
 	"math"
 	"net"
 	"strconv"
 	"strings"
-	"text/template"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/loggerutils"
 	"github.com/fluent/fluent-logger-golang/fluent"
 )
 
@@ -23,12 +22,6 @@ type fluentd struct {
 	writer        *fluent.Fluent
 }
 
-type receiver struct {
-	ID     string
-	FullID string
-	Name   string
-}
-
 const (
 	name             = "fluentd"
 	defaultHostName  = "localhost"
@@ -48,10 +41,14 @@ func init() {
 func parseConfig(ctx logger.Context) (string, int, string, error) {
 	host := defaultHostName
 	port := defaultPort
-	tag := "docker." + ctx.ContainerID[:12]
 
 	config := ctx.Config
 
+	tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}")
+	if err != nil {
+		return "", 0, "", err
+	}
+
 	if address := config["fluentd-address"]; address != "" {
 		if h, p, err := net.SplitHostPort(address); err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
@@ -68,23 +65,6 @@ func parseConfig(ctx logger.Context) (string, int, string, error) {
 		}
 	}
 
-	if config["fluentd-tag"] != "" {
-		receiver := &receiver{
-			ID:     ctx.ContainerID[:12],
-			FullID: ctx.ContainerID,
-			Name:   ctx.ContainerName,
-		}
-		tmpl, err := template.New("tag").Parse(config["fluentd-tag"])
-		if err != nil {
-			return "", 0, "", err
-		}
-		buf := new(bytes.Buffer)
-		if err := tmpl.Execute(buf, receiver); err != nil {
-			return "", 0, "", err
-		}
-		tag = buf.String()
-	}
-
 	return host, port, tag, nil
 }
 
@@ -130,6 +110,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		switch key {
 		case "fluentd-address":
 		case "fluentd-tag":
+		case "tag":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for fluentd log driver", key)
 		}

daemon/logger/gelf/gelf.go

@@ -14,6 +14,7 @@ import (
 	"github.com/Graylog2/go-gelf/gelf"
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/loggerutils"
 	"github.com/docker/docker/pkg/urlutil"
 )
 
@@ -64,6 +65,12 @@ func New(ctx logger.Context) (logger.Logger, error) {
 	// remove trailing slash from container name
 	containerName := bytes.TrimLeft([]byte(ctx.ContainerName), "/")
 
+	// parse log tag
+	tag, err := loggerutils.ParseLogTag(ctx, "")
+	if err != nil {
+		return nil, err
+	}
+
 	fields := gelfFields{
 		hostname:      hostname,
 		containerID:   ctx.ContainerID,
@@ -71,7 +78,7 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		imageID:       ctx.ContainerImageID,
 		imageName:     ctx.ContainerImageName,
 		command:       ctx.Command(),
-		tag:           ctx.Config["gelf-tag"],
+		tag:           tag,
 		created:       ctx.ContainerCreated,
 	}
 
@@ -135,6 +142,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		switch key {
 		case "gelf-address":
 		case "gelf-tag":
+		case "tag":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for gelf log driver", key)
 		}

daemon/logger/loggerutils/log_tag.go

@@ -0,0 +1,46 @@
+package loggerutils
+
+import (
+	"bytes"
+	"fmt"
+	"text/template"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/daemon/logger"
+)
+
+// ParseLogTag generates a context aware tag for consistency across different
+// log drivers based on the context of the running container.
+func ParseLogTag(ctx logger.Context, defaultTemplate string) (string, error) {
+	tagTemplate := lookupTagTemplate(ctx, defaultTemplate)
+
+	tmpl, err := template.New("log-tag").Parse(tagTemplate)
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	if err := tmpl.Execute(buf, &ctx); err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}
+
+func lookupTagTemplate(ctx logger.Context, defaultTemplate string) string {
+	tagTemplate := ctx.Config["tag"]
+
+	deprecatedConfigs := []string{"syslog-tag", "gelf-tag", "fluentd-tag"}
+	for i := 0; tagTemplate == "" && i < len(deprecatedConfigs); i++ {
+		cfg := deprecatedConfigs[i]
+		if ctx.Config[cfg] != "" {
+			tagTemplate = ctx.Config[cfg]
+			logrus.Warn(fmt.Sprintf("Using log tag from deprecated log-opt '%s'. Please use: --log-opt tag=\"%s\"", cfg, tagTemplate))
+		}
+	}
+
+	if tagTemplate == "" {
+		tagTemplate = defaultTemplate
+	}
+
+	return tagTemplate
+}

daemon/logger/loggerutils/log_tag_test.go

@@ -0,0 +1,58 @@
+package loggerutils
+
+import (
+	"testing"
+
+	"github.com/docker/docker/daemon/logger"
+)
+
+func TestParseLogTagDefaultTag(t *testing.T) {
+	ctx := buildContext(map[string]string{})
+	tag, e := ParseLogTag(ctx, "{{.ID}}")
+	assertTag(t, e, tag, ctx.ID())
+}
+
+func TestParseLogTag(t *testing.T) {
+	ctx := buildContext(map[string]string{"tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"})
+	tag, e := ParseLogTag(ctx, "{{.ID}}")
+	assertTag(t, e, tag, "test-image/test-container/container-ab")
+}
+
+func TestParseLogTagSyslogTag(t *testing.T) {
+	ctx := buildContext(map[string]string{"syslog-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"})
+	tag, e := ParseLogTag(ctx, "{{.ID}}")
+	assertTag(t, e, tag, "test-image/test-container/container-ab")
+}
+
+func TestParseLogTagGelfTag(t *testing.T) {
+	ctx := buildContext(map[string]string{"gelf-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"})
+	tag, e := ParseLogTag(ctx, "{{.ID}}")
+	assertTag(t, e, tag, "test-image/test-container/container-ab")
+}
+
+func TestParseLogTagFluentdTag(t *testing.T) {
+	ctx := buildContext(map[string]string{"fluentd-tag": "{{.ImageName}}/{{.Name}}/{{.ID}}"})
+	tag, e := ParseLogTag(ctx, "{{.ID}}")
+	assertTag(t, e, tag, "test-image/test-container/container-ab")
+}
+
+// Helpers
+
+func buildContext(cfg map[string]string) logger.Context {
+	return logger.Context{
+		ContainerID:        "container-abcdefghijklmnopqrstuvwxyz01234567890",
+		ContainerName:      "/test-container",
+		ContainerImageID:   "image-abcdefghijklmnopqrstuvwxyz01234567890",
+		ContainerImageName: "test-image",
+		Config:             cfg,
+	}
+}
+
+func assertTag(t *testing.T, e error, tag string, expected string) {
+	if e != nil {
+		t.Fatalf("Error generating tag: %q", e)
+	}
+	if tag != expected {
+		t.Fatalf("Wrong tag: %q, should be %q", tag, expected)
+	}
+}

daemon/logger/syslog/syslog.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/loggerutils"
 	"github.com/docker/docker/pkg/urlutil"
 )
 
@@ -61,9 +62,9 @@ func init() {
 // the context. Supported context configuration variables are
 // syslog-address, syslog-facility, & syslog-tag.
 func New(ctx logger.Context) (logger.Logger, error) {
-	tag := ctx.Config["syslog-tag"]
-	if tag == "" {
-		tag = ctx.ContainerID[:12]
+	tag, err := loggerutils.ParseLogTag(ctx, "{{.ID}}")
+	if err != nil {
+		return nil, err
 	}
 
 	proto, address, err := parseAddress(ctx.Config["syslog-address"])
@@ -146,6 +147,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case "syslog-address":
 		case "syslog-facility":
 		case "syslog-tag":
+		case "tag":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for syslog log driver", key)
 		}

docs/misc/deprecated.md

@@ -12,6 +12,19 @@ parent = "mn_use_docker"
 
 The following list of features are deprecated.
 
+### Driver Specific Log Tags
+**Deprecated In Release: v1.9**
+
+**Target For Removal In Release: v1.11**
+
+Log tags are now generated in a standard way across different logging drivers.
+Because of which, the driver specific log tag options `syslog-tag`, `gelf-tag` and
+`fluentd-tag` have been deprecated in favor of the generic `tag` option.
+
+    docker --log-driver=syslog --log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}"
+
+
+
 ### LXC built-in exec driver
 **Deprecated In Release: v1.8**
 

docs/reference/logging/fluentd.md

@@ -5,6 +5,7 @@ description = "Describes how to use the fluentd logging driver."
 keywords = ["Fluentd, docker, logging, driver"]
 [menu.main]
 parent = "smn_logging"
+weight=2
 +++
 <![end-metadata]-->
 
@@ -32,7 +33,7 @@ The `docker logs` command is not available for this logging driver.
 Some options are supported by specifying `--log-opt` as many times as needed:
 
  - `fluentd-address`: specify `host:port` to connect `localhost:24224`
- - `fluentd-tag`: specify tag for fluentd message, which interpret some markup, ex `{{.ID}}`, `{{.FullID}}` or `{{.Name}}` `docker.{{.ID}}`
+ - `tag`: specify tag for fluentd message, which interpret some markup, ex `{{.ID}}`, `{{.FullID}}` or `{{.Name}}` `docker.{{.ID}}`
 
 
 Configure the default logging driver by passing the
@@ -65,24 +66,12 @@ By default, the logging driver connects to `localhost:24224`. Supply the
 
     docker run --log-driver=fluentd --log-opt fluentd-address=myhost.local:24224
 
-### fluentd-tag
+### tag
 
-Every Fluentd's event has a tag that indicates where the log comes from. By
-default, the driver uses the `docker.{{.ID}}` tag.  Use the `fluentd-tag` option
-to change this behavior.
+By default, Docker uses the first 12 characters of the container ID to tag log messages.
+Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing
+the log tag format.
 
-When specifying a `fluentd-tag` value, you can use the following markup tags:
-
- - `{{.ID}}`: short container id (12 characters)
- - `{{.FullID}}`: full container id
- - `{{.Name}}`: container name
-
-## Note regarding container names
-
-At startup time, the system sets the `container_name` field and `{{.Name}}`
-in the tags to their values at startup. If you use `docker rename` to rename a
-container, the new name is not be reflected in  `fluentd` messages. Instead,
-these messages continue to use the original container name.
 
 ## Fluentd daemon management with Docker
 

docs/reference/logging/index.md

@@ -1,7 +1,7 @@
 <!--[metadata]>
 +++
-title = "Logging Drivers"
-description = "Logging Drivers"
+title = "Logging"
+description = "Logging and Logging Drivers"
 keywords = [" docker, logging, driver"]
 [menu.main]
 parent = "smn_administrate"
@@ -14,6 +14,7 @@ weight=8
 # Logging Drivers
 
 * [Configuring logging drivers](overview)
+* [Configuring log tags](log_tags)
 * [Fluentd logging driver](fluentd)
 * [Journald logging driver](journald)
 * [Amazon CloudWatch Logs logging driver](awslogs)

docs/reference/logging/journald.md

@@ -5,6 +5,7 @@ description = "Describes how to use the fluentd logging driver."
 keywords = ["Fluentd, docker, logging, driver"]
 [menu.main]
 parent = "smn_logging"
+weight = 2
 +++
 <![end-metadata]-->
 

docs/reference/logging/log_tags.md

@@ -0,0 +1,50 @@
+<!--[metadata]>
++++
+title = "Log tags for logging driver"
+description = "Describes how to format tags for."
+keywords = ["docker, logging, driver, syslog, Fluentd, gelf"]
+[menu.main]
+parent = "smn_logging"
+weight = 1
++++
+<![end-metadata]-->
+
+# Log Tags
+
+The `tag` log option specifies how to format a tag that identifies the
+container's log messages. By default, the system uses the first 12 characters of
+the container id. To override this behavior, specify a `tag` option:
+
+```
+docker run --log-driver=fluentd --log-opt fluentd-address=myhost.local:24224 --log-opt tag="mailer"
+```
+
+Docker supports some special template markup you can use when specifying a tag's value:
+
+| Markup             | Description                                          |
+|--------------------|------------------------------------------------------|
+| `{{.ID}}`          | The first 12 characters of the container id.         |
+| `{{.FullID}}`      | The full container id.                               |
+| `{{.Name}}`        | The container name.                                  |
+| `{{.ImageID}}`     | The first 12 characters of the container's image id. |
+| `{{.ImageFullID}}` | The container's full image identifier.               |
+| `{{.ImageName}}`   | The name of the image used by the container.         |
+
+For example, specifying a `--log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}"` value yields `syslog` log lines like:
+
+```
+Aug  7 18:33:19 HOSTNAME docker/hello-world/foobar/5790672ab6a0[9103]: Hello from Docker.
+```
+
+At startup time, the system sets the `container_name` field and `{{.Name}}` in
+the tags. If you use `docker rename` to rename a container, the new name is not
+reflected in the log messages. Instead, these messages continue to use the
+original container name.
+
+For advanced usage, the generated tag's use [go
+templates](http://golang.org/pkg/text/template/) and the container's [logging
+context](https://github.com/docker/docker/blob/master/daemon/logger/context.go).
+
+>**Note**:The driver specific log options `syslog-tag`, `fluentd-tag` and
+>`gelf-tag` still work for backwards compatibility. However, going forward you
+>should standardize on using the generic `tag` log option instead.

docs/reference/logging/overview.md

@@ -2,7 +2,7 @@
 +++
 title = "Configuring Logging Drivers"
 description = "Configure logging driver."
-keywords = ["Fluentd, docker, logging, driver"]
+keywords = ["docker, logging, driver, Fluentd"]
 [menu.main]
 parent = "smn_logging"
 weight=-1
@@ -27,7 +27,7 @@ container's logging driver. The following options are supported:
 
 The `docker logs`command is available only for the `json-file` logging driver.
 
-### The json-file options
+## json-file options
 
 The following logging options are supported for the `json-file` logging driver:
 
@@ -41,14 +41,14 @@ Logs that reach `max-size` are rolled over. You can set the size in kilobytes(k)
 
 If `max-size` and `max-file` are set, `docker logs` only returns the log lines from the newest log file.
 
-### The syslog options
+## syslog options
 
 The following logging options are supported for the `syslog` logging driver:
 
     --log-opt syslog-address=[tcp|udp]://host:port
     --log-opt syslog-address=unix://path
     --log-opt syslog-facility=daemon
-    --log-opt syslog-tag="mailer"
+    --log-opt tag="mailer"
 
 `syslog-address` specifies the remote syslog server address where the driver connects to.
 If not specified it defaults to the local unix socket of the running system.
@@ -83,22 +83,23 @@ the following named facilities:
 * `local6`
 * `local7`
 
-The `syslog-tag` specifies a tag that identifies the container's syslog messages. By default,
-the system uses the first 12 characters of the container id. To override this behavior, specify
-a `syslog-tag` option
+By default, Docker uses the first 12 characters of the container ID to tag log messages.
+Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing
+the log tag format.
 
-## Specify journald options
+
+## journald options
 
 The `journald` logging driver stores the container id in the journal's `CONTAINER_ID` field. For detailed information on
 working with this logging driver, see [the journald logging driver](/reference/logging/journald/)
 reference documentation.
 
-## Specify gelf options
+## gelf options
 
 The GELF logging driver supports the following options:
 
     --log-opt gelf-address=udp://host:port
-    --log-opt gelf-tag="database"
+    --log-opt tag="database"
 
 The `gelf-address` option specifies the remote GELF server address that the
 driver connects to. Currently, only `udp` is supported as the transport and you must
@@ -107,24 +108,21 @@ driver to a GELF remote server at `192.168.0.42` on port `12201`
 
     $ docker run --log-driver=gelf --log-opt gelf-address=udp://192.168.0.42:12201
 
-The `gelf-tag` option specifies a tag for easy container identification.
+By default, Docker uses the first 12 characters of the container ID to tag log messages.
+Refer to the [log tag option documentation](/reference/logging/log_tags/) for customizing
+the log tag format.
 
-## Specify fluentd options
+
+## fluentd options
 
 You can use the `--log-opt NAME=VALUE` flag to specify these additional Fluentd logging driver options.
 
  - `fluentd-address`: specify `host:port` to connect [localhost:24224]
- - `fluentd-tag`: specify tag for `fluentd` message, 
-
-When specifying a `fluentd-tag` value, you can use the following markup tags:
-
- - `{{.ID}}`: short container id (12 characters)
- - `{{.FullID}}`: full container id
- - `{{.Name}}`: container name
+ - `tag`: specify tag for `fluentd` message,
 
 For example, to specify both additional options:
 
-`docker run --log-driver=fluentd --log-opt fluentd-address=localhost:24224 --log-opt fluentd-tag=docker.{{.Name}}`
+`docker run --log-driver=fluentd --log-opt fluentd-address=localhost:24224 --log-opt tag=docker.{{.Name}}`
 
 If container cannot connect to the Fluentd daemon on the specified address,
 the container stops immediately. For detailed information on working with this