From 3cf82ff1ab14e1ddd2b629524e894ac359168388 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Tue, 17 Nov 2015 15:41:56 +0100 Subject: [PATCH] Added flag to ignore fluentd connect error on container start Signed-off-by: Jussi Nummelin Changed buffer size to 1M and removed unnecessary fmt call Signed-off-by: Jussi Nummelin Updated docs for the new fluentd opts Signed-off-by: Jussi Nummelin --- daemon/logger/fluentd/fluentd.go | 38 +++++++++++--- .../logger/loggerutils/log_option_helpers.go | 26 ++++++++++ .../loggerutils/log_option_helpers_test.go | 51 +++++++++++++++++++ docs/admin/logging/fluentd.md | 17 ++++++- docs/admin/logging/overview.md | 2 + 5 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 daemon/logger/loggerutils/log_option_helpers.go create mode 100644 daemon/logger/loggerutils/log_option_helpers_test.go diff --git a/daemon/logger/fluentd/fluentd.go b/daemon/logger/fluentd/fluentd.go index 825de3a1ae..788482b8bb 100644 --- a/daemon/logger/fluentd/fluentd.go +++ b/daemon/logger/fluentd/fluentd.go @@ -24,10 +24,12 @@ type fluentd struct { } const ( - name = "fluentd" - defaultHostName = "localhost" - defaultPort = 24224 - defaultTagPrefix = "docker" + name = "fluentd" + defaultHostName = "localhost" + defaultPort = 24224 + defaultTagPrefix = "docker" + defaultIgnoreConnectErrorOnStart = false // So that we do not break existing behaviour + defaultBufferLimit = 1 * 1024 * 1024 // 1M buffer by default ) func init() { @@ -52,13 +54,24 @@ func New(ctx logger.Context) (logger.Logger, error) { if err != nil { return nil, err } + failOnStartupError, err := loggerutils.ParseFailOnStartupErrorFlag(ctx) + if err != nil { + return nil, err + } + bufferLimit, err := parseBufferLimit(ctx.Config["buffer-limit"]) + if err != nil { + return nil, err + } extra := ctx.ExtraAttributes(nil) logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s, extra:%v.", ctx.ContainerID, host, port, tag, extra) // logger tries to reconnect 2**32 - 1 times // failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds] - log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32}) + log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32, BufferLimit: bufferLimit}) if err != nil { - return nil, err + if failOnStartupError { + return nil, err + } + logrus.Warnf("fluentd cannot connect to configured endpoint. Ignoring as instructed. Error: %q", err) } return &fluentd{ tag: tag, @@ -101,6 +114,8 @@ func ValidateLogOpt(cfg map[string]string) error { case "tag": case "labels": case "env": + case "fail-on-startup-error": + case "buffer-limit": default: return fmt.Errorf("unknown log opt '%s' for fluentd log driver", key) } @@ -132,3 +147,14 @@ func parseAddress(address string) (string, int, error) { } return host, portnum, nil } + +func parseBufferLimit(bufferLimit string) (int, error) { + if bufferLimit == "" { + return defaultBufferLimit, nil + } + limit, err := strconv.Atoi(bufferLimit) + if err != nil { + return 0, fmt.Errorf("invalid buffer limit %s: %s", bufferLimit, err) + } + return limit, nil +} diff --git a/daemon/logger/loggerutils/log_option_helpers.go b/daemon/logger/loggerutils/log_option_helpers.go new file mode 100644 index 0000000000..7c1d97f999 --- /dev/null +++ b/daemon/logger/loggerutils/log_option_helpers.go @@ -0,0 +1,26 @@ +package loggerutils + +import ( + "fmt" + "strconv" + + "github.com/docker/docker/daemon/logger" +) + +const ( + defaultFailOnStartupError = true // So that we do not break existing behaviour +) + +// ParseFailOnStartupErrorFlag parses a log driver flag that determines if +// the driver should ignore possible connection errors during startup +func ParseFailOnStartupErrorFlag(ctx logger.Context) (bool, error) { + failOnStartupError := ctx.Config["fail-on-startup-error"] + if failOnStartupError == "" { + return defaultFailOnStartupError, nil + } + failOnStartupErrorFlag, err := strconv.ParseBool(failOnStartupError) + if err != nil { + return defaultFailOnStartupError, fmt.Errorf("invalid connect error flag %s: %s", failOnStartupError, err) + } + return failOnStartupErrorFlag, nil +} diff --git a/daemon/logger/loggerutils/log_option_helpers_test.go b/daemon/logger/loggerutils/log_option_helpers_test.go new file mode 100644 index 0000000000..fbbd501d9c --- /dev/null +++ b/daemon/logger/loggerutils/log_option_helpers_test.go @@ -0,0 +1,51 @@ +package loggerutils + +import ( + "testing" + + "github.com/docker/docker/daemon/logger" +) + +func TestParseDefaultIgnoreFlag(t *testing.T) { + ctx := buildContext(map[string]string{}) + flag, e := ParseFailOnStartupErrorFlag(ctx) + assertFlag(t, e, flag, true) +} + +func TestParseIgnoreFlagWhenFalse(t *testing.T) { + ctx := buildContext(map[string]string{"fail-on-startup-error": "false"}) + flag, e := ParseFailOnStartupErrorFlag(ctx) + assertFlag(t, e, flag, false) +} + +func TestParseIgnoreFlagWhenTrue(t *testing.T) { + ctx := buildContext(map[string]string{"fail-on-startup-error": "true"}) + flag, e := ParseFailOnStartupErrorFlag(ctx) + assertFlag(t, e, flag, true) +} + +func TestParseIgnoreFlagWithError(t *testing.T) { + ctx := buildContext(map[string]string{"fail-on-startup-error": "maybe :)"}) + flag, e := ParseFailOnStartupErrorFlag(ctx) + if e == nil { + t.Fatalf("Error should have happened") + } + assertFlag(t, nil, flag, true) +} + +// Helpers + +func buildConfig(cfg map[string]string) logger.Context { + return logger.Context{ + Config: cfg, + } +} + +func assertFlag(t *testing.T, e error, flag bool, expected bool) { + if e != nil { + t.Fatalf("Error parsing ignore connect error flag: %q", e) + } + if flag != expected { + t.Fatalf("Wrong flag: %t, should be %t", flag, expected) + } +} diff --git a/docs/admin/logging/fluentd.md b/docs/admin/logging/fluentd.md index ba9b630a0d..a87b1dca69 100644 --- a/docs/admin/logging/fluentd.md +++ b/docs/admin/logging/fluentd.md @@ -35,7 +35,8 @@ Some options are supported by specifying `--log-opt` as many times as needed: - `fluentd-address`: specify `host:port` to connect `localhost:24224` - `tag`: specify tag for fluentd message, which interpret some markup, ex `{{.ID}}`, `{{.FullID}}` or `{{.Name}}` `docker.{{.ID}}` - + - `fail-on-startup-error`: true/false; Should the logging driver fail container startup in case of connect error during startup. Default: true (backwards compatible) + - `buffer-limit`: Size limit (bytes) for the buffer which is used to buffer messages in case of connection outages. Default: 1M Configure the default logging driver by passing the `--log-driver` option to the Docker daemon: @@ -78,6 +79,20 @@ the log tag format. The `labels` and `env` options each take a comma-separated list of keys. If there is collision between `label` and `env` keys, the value of the `env` takes precedence. Both options add additional fields to the extra attributes of a logging message. +### fail-on-startup-error + +By default, if the logging driver cannot connect to the backend it will fail the entire startup of the container. If you wish to ignore potential connect error during container startup supply the `fail-on-startup-error` flag. + + docker run --log-driver=fluentd --log-opt fail-on-startup-error=false + + +### buffer-limit + +When fluent driver loses connection, or cannot connect at container startup, it will buffer the log events locally for re-transmission. Buffer limit option controls how much data will be buffered locally, **per container**. Specified in bytes. + + docker run --log-driver=fluentd --log-opt buffer-limit=5242880 + +The above would result to use 5M buffer locally. Keep in mind that during possible connection errors all your containers will start buffering locally and thus might result in considerable memory usage. ## Fluentd daemon management with Docker diff --git a/docs/admin/logging/overview.md b/docs/admin/logging/overview.md index 9878ba8445..531b338ed0 100644 --- a/docs/admin/logging/overview.md +++ b/docs/admin/logging/overview.md @@ -172,6 +172,8 @@ You can use the `--log-opt NAME=VALUE` flag to specify these additional Fluentd - `fluentd-address`: specify `host:port` to connect [localhost:24224] - `tag`: specify tag for `fluentd` message, + - `fail-on-startup-error`: true/false; Should the logging driver fail container startup in case of connect error during startup. Default: true (backwards compatible) + - `buffer-limit`: Size limit (bytes) for the buffer which is used to buffer messages in case of connection outages. Default: 1M For example, to specify both additional options: