From bd9d14a07b9f1c82625dc8483245caf3fa7fe9e6 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 8 Apr 2016 12:15:08 -0400 Subject: [PATCH] Add support for reading logs extra attrs The jsonlog logger currently allows specifying envs and labels that should be propagated to the log message, however there has been no way to read that back. This adds a new API option to enable inserting these attrs back to the log reader. With timestamps, this looks like so: ``` 92016-04-08T15:28:09.835913720Z foo=bar,hello=world hello ``` The extra attrs are comma separated before the log message but after timestamps. Without timestaps it looks like so: ``` foo=bar,hello=world hello ``` Signed-off-by: Brian Goff --- api/client/logs.go | 2 ++ .../router/container/container_routes.go | 1 + daemon/logger/jsonfilelog/read.go | 1 + daemon/logger/logger.go | 27 +++++++++++++++++++ daemon/logs.go | 3 +++ docs/reference/api/docker_remote_api.md | 1 + docs/reference/api/docker_remote_api_v1.24.md | 1 + docs/reference/commandline/logs.md | 5 ++++ integration-cli/docker_cli_logs_test.go | 13 +++++++++ man/docker-logs.1.md | 7 +++++ pkg/jsonlog/jsonlog.go | 2 ++ pkg/jsonlog/jsonlog_marshalling_test.go | 22 +++++++-------- 12 files changed, 74 insertions(+), 11 deletions(-) diff --git a/api/client/logs.go b/api/client/logs.go index 85545b9304..25c9004c70 100644 --- a/api/client/logs.go +++ b/api/client/logs.go @@ -25,6 +25,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error { follow := cmd.Bool([]string{"f", "-follow"}, false, "Follow log output") since := cmd.String([]string{"-since"}, "", "Show logs since timestamp") times := cmd.Bool([]string{"t", "-timestamps"}, false, "Show timestamps") + details := cmd.Bool([]string{"-details"}, false, "Show extra details provided to logs") tail := cmd.String([]string{"-tail"}, "all", "Number of lines to show from the end of the logs") cmd.Require(flag.Exact, 1) @@ -48,6 +49,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error { Timestamps: *times, Follow: *follow, Tail: *tail, + Details: *details, } responseBody, err := cli.client.ContainerLogs(context.Background(), name, options) if err != nil { diff --git a/api/server/router/container/container_routes.go b/api/server/router/container/container_routes.go index a343d64c0b..b0cc83f1cd 100644 --- a/api/server/router/container/container_routes.go +++ b/api/server/router/container/container_routes.go @@ -99,6 +99,7 @@ func (s *containerRouter) getContainersLogs(ctx context.Context, w http.Response Tail: r.Form.Get("tail"), ShowStdout: stdout, ShowStderr: stderr, + Details: httputils.BoolValue(r, "details"), }, OutStream: w, } diff --git a/daemon/logger/jsonfilelog/read.go b/daemon/logger/jsonfilelog/read.go index 0c8fb5e5cd..1e197f3e84 100644 --- a/daemon/logger/jsonfilelog/read.go +++ b/daemon/logger/jsonfilelog/read.go @@ -27,6 +27,7 @@ func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, erro Source: l.Stream, Timestamp: l.Created, Line: []byte(l.Log), + Attrs: l.Attrs, } return msg, nil } diff --git a/daemon/logger/logger.go b/daemon/logger/logger.go index cf8d571fa4..27c01a59d0 100644 --- a/daemon/logger/logger.go +++ b/daemon/logger/logger.go @@ -9,6 +9,8 @@ package logger import ( "errors" + "sort" + "strings" "time" "github.com/docker/docker/pkg/jsonlog" @@ -29,6 +31,31 @@ type Message struct { Line []byte Source string Timestamp time.Time + Attrs LogAttributes +} + +// LogAttributes is used to hold the extra attributes available in the log message +// Primarily used for converting the map type to string and sorting. +type LogAttributes map[string]string +type byKey []string + +func (s byKey) Len() int { return len(s) } +func (s byKey) Less(i, j int) bool { + keyI := strings.Split(s[i], "=") + keyJ := strings.Split(s[j], "=") + return keyI[0] < keyJ[0] +} +func (s byKey) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (a LogAttributes) String() string { + var ss byKey + for k, v := range a { + ss = append(ss, k+"="+v) + } + sort.Sort(ss) + return strings.Join(ss, ",") } // Logger is the interface for docker logging drivers. diff --git a/daemon/logs.go b/daemon/logs.go index 57d44acb3b..4473e1a311 100644 --- a/daemon/logs.go +++ b/daemon/logs.go @@ -90,6 +90,9 @@ func (daemon *Daemon) ContainerLogs(ctx context.Context, containerName string, c return nil } logLine := msg.Line + if config.Details { + logLine = append([]byte(msg.Attrs.String()+" "), logLine...) + } if config.Timestamps { logLine = append([]byte(msg.Timestamp.Format(logger.TimeFormat)+" "), logLine...) } diff --git a/docs/reference/api/docker_remote_api.md b/docs/reference/api/docker_remote_api.md index d536cbf629..234c3d2a29 100644 --- a/docs/reference/api/docker_remote_api.md +++ b/docs/reference/api/docker_remote_api.md @@ -136,6 +136,7 @@ This section lists each version from latest to oldest. Each listing includes a * `POST /auth` now returns an `IdentityToken` when supported by a registry. * `POST /containers/create` with both `Hostname` and `Domainname` fields specified will result in the container's hostname being set to `Hostname`, rather than `Hostname.Domainname`. * `GET /volumes` now supports more filters, new added filters are `name` and `driver`. +* `GET /containers/(id or name)/logs` now accepts a `details` query parameter to stream the extra attributes that were provided to the containers `LogOpts`, such as environment variables and labels, with the logs. ### v1.22 API changes diff --git a/docs/reference/api/docker_remote_api_v1.24.md b/docs/reference/api/docker_remote_api_v1.24.md index 6678cbfe3f..399d8e6a8c 100644 --- a/docs/reference/api/docker_remote_api_v1.24.md +++ b/docs/reference/api/docker_remote_api_v1.24.md @@ -770,6 +770,7 @@ Get `stdout` and `stderr` logs from the container ``id`` Query Parameters: +- **details** - 1/True/true or 0/False/flase, Show extra details provided to logs. Default `false`. - **follow** – 1/True/true or 0/False/false, return stream. Default `false`. - **stdout** – 1/True/true or 0/False/false, show `stdout` log. Default `false`. - **stderr** – 1/True/true or 0/False/false, show `stderr` log. Default `false`. diff --git a/docs/reference/commandline/logs.md b/docs/reference/commandline/logs.md index 91558ffa63..dd90c4dcc0 100644 --- a/docs/reference/commandline/logs.md +++ b/docs/reference/commandline/logs.md @@ -14,6 +14,7 @@ parent = "smn_cli" Fetch the logs of a container + --details Show extra details provided to logs -f, --follow Follow log output --help Print usage --since="" Show logs since timestamp @@ -36,6 +37,10 @@ The `docker logs --timestamps` command will add an [RFC3339Nano timestamp](https log entry. To ensure that the timestamps are aligned the nano-second part of the timestamp will be padded with zero when necessary. +The `docker logs --details` command will add on extra attributes, such as +environment variables and labels, provided to `--log-opt` when creating the +container. + The `--since` option shows only the container logs generated after a given date. You can specify the date as an RFC 3339 date, a UNIX timestamp, or a Go duration string (e.g. `1m30s`, `3h`). Besides RFC3339 date diff --git a/integration-cli/docker_cli_logs_test.go b/integration-cli/docker_cli_logs_test.go index a862cb31c5..317cb202ec 100644 --- a/integration-cli/docker_cli_logs_test.go +++ b/integration-cli/docker_cli_logs_test.go @@ -307,3 +307,16 @@ func (s *DockerSuite) TestLogsCLIContainerNotFound(c *check.C) { message := fmt.Sprintf("Error: No such container: %s\n", name) c.Assert(out, checker.Equals, message) } + +func (s *DockerSuite) TestLogsWithDetails(c *check.C) { + dockerCmd(c, "run", "--name=test", "--label", "foo=bar", "-e", "baz=qux", "--log-opt", "labels=foo", "--log-opt", "env=baz", "busybox", "echo", "hello") + out, _ := dockerCmd(c, "logs", "--details", "--timestamps", "test") + + logFields := strings.Fields(strings.TrimSpace(out)) + c.Assert(len(logFields), checker.Equals, 3, check.Commentf(out)) + + details := strings.Split(logFields[1], ",") + c.Assert(details, checker.HasLen, 2) + c.Assert(details[0], checker.Equals, "baz=qux") + c.Assert(details[1], checker.Equals, "foo=bar") +} diff --git a/man/docker-logs.1.md b/man/docker-logs.1.md index f910b53574..db23a0f137 100644 --- a/man/docker-logs.1.md +++ b/man/docker-logs.1.md @@ -30,6 +30,9 @@ logging drivers. **--help** Print usage statement +**--details**=*true*|*false* + Show extra details provided to logs + **-f**, **--follow**=*true*|*false* Follow log output. The default is *false*. @@ -55,6 +58,10 @@ epoch or Unix time), and the optional .nanoseconds field is a fraction of a second no more than nine digits long. You can combine the `--since` option with either or both of the `--follow` or `--tail` options. +The `docker logs --details` command will add on extra attributes, such as +environment variables and labels, provided to `--log-opt` when creating the +container. + # HISTORY April 2014, Originally compiled by William Henry (whenry at redhat dot com) based on docker.com source material and internal work. diff --git a/pkg/jsonlog/jsonlog.go b/pkg/jsonlog/jsonlog.go index 422e4bbd92..4734c31119 100644 --- a/pkg/jsonlog/jsonlog.go +++ b/pkg/jsonlog/jsonlog.go @@ -15,6 +15,8 @@ type JSONLog struct { Stream string `json:"stream,omitempty"` // Created is the created timestamp of log Created time.Time `json:"time"` + // Attrs is the list of extra attributes provided by the user + Attrs map[string]string `json:"attrs,omitempty"` } // Format returns the log formatted according to format diff --git a/pkg/jsonlog/jsonlog_marshalling_test.go b/pkg/jsonlog/jsonlog_marshalling_test.go index 5e455685ab..3edb271410 100644 --- a/pkg/jsonlog/jsonlog_marshalling_test.go +++ b/pkg/jsonlog/jsonlog_marshalling_test.go @@ -6,18 +6,18 @@ import ( ) func TestJSONLogMarshalJSON(t *testing.T) { - logs := map[JSONLog]string{ - JSONLog{Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, - JSONLog{Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, - JSONLog{Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, - JSONLog{Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, - JSONLog{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, - JSONLog{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, - JSONLog{}: `^{\"time\":\".{20,}\"}$`, + logs := map[*JSONLog]string{ + &JSONLog{Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, + &JSONLog{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, + &JSONLog{}: `^{\"time\":\".{20,}\"}$`, // These ones are a little weird - JSONLog{Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, - JSONLog{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, - JSONLog{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, + &JSONLog{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, } for jsonLog, expression := range logs { data, err := jsonLog.MarshalJSON()