From 603fd0831513257bc26d20ca1f64efcc4965eae6 Mon Sep 17 00:00:00 2001 From: Denis Gladkikh Date: Thu, 9 Jun 2016 21:06:44 -0700 Subject: [PATCH 1/2] Splunk Logging Driver: formats and verifyconnection `--log-opt splunk-format=inline|json|raw` allows to change how logging driver sends data to Splunk, where `inline` - default value, format used before, message is injected as a line in JSON payload `json` - driver will try to parse each line as a JSON object and embed it inside of the JSON payload `raw` - driver will send Raw payload instead of JSON, tag and attributes will be prefixed before the message `--log-opt splunk-verify-connection=true|false` - allows to skip verification for Splunk Url Signed-off-by: Denis Gladkikh --- contrib/completion/bash/docker | 8 +- contrib/completion/zsh/_docker | 2 +- daemon/logger/splunk/splunk.go | 172 ++++++++++++++++++++++++++++----- docs/admin/logging/splunk.md | 66 +++++++++++++ 4 files changed, 219 insertions(+), 29 deletions(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 6d7278bc57..d2af8a6287 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -520,7 +520,7 @@ __docker_complete_log_options() { local journald_options="env labels tag" local json_file_options="env labels max-file max-size" local syslog_options="env labels syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag" - local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag" + local splunk_options="env labels splunk-caname splunk-capath splunk-format splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag" local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options" @@ -629,10 +629,14 @@ __docker_complete_log_driver_options() { __ltrim_colon_completions "${cur}" return ;; - splunk-insecureskipverify) + splunk-insecureskipverify|splunk-verify-connection) COMPREPLY=( $( compgen -W "false true" -- "${cur##*=}" ) ) return ;; + splunk-format) + COMPREPLY=( $( compgen -W "inline json raw" -- "${cur##*=}" ) ) + return + ;; esac return 1 } diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 733477cbb9..544d492016 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -228,7 +228,7 @@ __docker_get_log_options() { journald_options=("env" "labels" "tag") json_file_options=("env" "labels" "max-file" "max-size") syslog_options=("env" "labels" "syslog-address" "syslog-facility" "syslog-format" "syslog-tls-ca-cert" "syslog-tls-cert" "syslog-tls-key" "syslog-tls-skip-verify" "tag") - splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "tag") + splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-format" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "splunk-verify-connection" "tag") [[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0 [[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0 diff --git a/daemon/logger/splunk/splunk.go b/daemon/logger/splunk/splunk.go index b5cafffb30..0dd64e15be 100644 --- a/daemon/logger/splunk/splunk.go +++ b/daemon/logger/splunk/splunk.go @@ -30,6 +30,8 @@ const ( splunkCAPathKey = "splunk-capath" splunkCANameKey = "splunk-caname" splunkInsecureSkipVerifyKey = "splunk-insecureskipverify" + splunkFormatKey = "splunk-format" + splunkVerifyConnectionKey = "splunk-verify-connection" envKey = "env" labelsKey = "labels" tagKey = "tag" @@ -44,22 +46,44 @@ type splunkLogger struct { nullMessage *splunkMessage } +type splunkLoggerInline struct { + splunkLogger + + nullEvent *splunkMessageEvent +} + +type splunkLoggerJSON struct { + splunkLoggerInline +} + +type splunkLoggerRaw struct { + splunkLogger + + prefix []byte +} + type splunkMessage struct { - Event splunkMessageEvent `json:"event"` - Time string `json:"time"` - Host string `json:"host"` - Source string `json:"source,omitempty"` - SourceType string `json:"sourcetype,omitempty"` - Index string `json:"index,omitempty"` + Event interface{} `json:"event"` + Time string `json:"time"` + Host string `json:"host"` + Source string `json:"source,omitempty"` + SourceType string `json:"sourcetype,omitempty"` + Index string `json:"index,omitempty"` } type splunkMessageEvent struct { - Line string `json:"line"` + Line interface{} `json:"line"` Source string `json:"source"` Tag string `json:"tag,omitempty"` Attrs map[string]string `json:"attrs,omitempty"` } +const ( + splunkFormatRaw = "raw" + splunkFormatJSON = "json" + splunkFormatInline = "inline" +) + func init() { if err := logger.RegisterLogDriver(driverName, New); err != nil { logrus.Fatal(err) @@ -122,21 +146,23 @@ func New(ctx logger.Context) (logger.Logger, error) { Transport: transport, } - var nullMessage = &splunkMessage{ - Host: hostname, - } + source := ctx.Config[splunkSourceKey] + sourceType := ctx.Config[splunkSourceTypeKey] + index := ctx.Config[splunkIndexKey] - // Optional parameters for messages - nullMessage.Source = ctx.Config[splunkSourceKey] - nullMessage.SourceType = ctx.Config[splunkSourceTypeKey] - nullMessage.Index = ctx.Config[splunkIndexKey] + var nullMessage = &splunkMessage{ + Host: hostname, + Source: source, + SourceType: sourceType, + Index: index, + } tag, err := loggerutils.ParseLogTag(ctx, loggerutils.DefaultTemplate) if err != nil { return nil, err } - nullMessage.Event.Tag = tag - nullMessage.Event.Attrs = ctx.ExtraAttributes(nil) + + attrs := ctx.ExtraAttributes(nil) logger := &splunkLogger{ client: client, @@ -146,21 +172,107 @@ func New(ctx logger.Context) (logger.Logger, error) { nullMessage: nullMessage, } - err = verifySplunkConnection(logger) - if err != nil { - return nil, err + // By default we verify connection, but we allow use to skip that + verifyConnection := true + if verifyConnectionStr, ok := ctx.Config[splunkVerifyConnectionKey]; ok { + var err error + verifyConnection, err = strconv.ParseBool(verifyConnectionStr) + if err != nil { + return nil, err + } + } + if verifyConnection { + err = verifySplunkConnection(logger) + if err != nil { + return nil, err + } } - return logger, nil + var splunkFormat string + if splunkFormatParsed, ok := ctx.Config[splunkFormatKey]; ok { + switch splunkFormatParsed { + case splunkFormatInline: + case splunkFormatJSON: + case splunkFormatRaw: + default: + return nil, fmt.Errorf("Unknown format specified %s, supported formats are inline, json and raw", splunkFormat) + } + splunkFormat = splunkFormatParsed + } else { + splunkFormat = splunkFormatInline + } + + switch splunkFormat { + case splunkFormatInline: + nullEvent := &splunkMessageEvent{ + Tag: tag, + Attrs: attrs, + } + + return &splunkLoggerInline{*logger, nullEvent}, nil + case splunkFormatJSON: + nullEvent := &splunkMessageEvent{ + Tag: tag, + Attrs: attrs, + } + + return &splunkLoggerJSON{splunkLoggerInline{*logger, nullEvent}}, nil + case splunkFormatRaw: + var prefix bytes.Buffer + prefix.WriteString(tag) + prefix.WriteString(" ") + for key, value := range attrs { + prefix.WriteString(key) + prefix.WriteString("=") + prefix.WriteString(value) + prefix.WriteString(" ") + } + + return &splunkLoggerRaw{*logger, prefix.Bytes()}, nil + default: + return nil, fmt.Errorf("Unexpected format %s", splunkFormat) + } } -func (l *splunkLogger) Log(msg *logger.Message) error { - // Construct message as a copy of nullMessage - message := *l.nullMessage - message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000) - message.Event.Line = string(msg.Line) - message.Event.Source = msg.Source +func (l *splunkLoggerInline) Log(msg *logger.Message) error { + message := l.createSplunkMessage(msg) + event := *l.nullEvent + event.Line = string(msg.Line) + event.Source = msg.Source + + message.Event = &event + + return l.postMessage(&message) +} + +func (l *splunkLoggerJSON) Log(msg *logger.Message) error { + message := l.createSplunkMessage(msg) + event := *l.nullEvent + + var rawJSONMessage json.RawMessage + if err := json.Unmarshal(msg.Line, &rawJSONMessage); err == nil { + event.Line = &rawJSONMessage + } else { + event.Line = string(msg.Line) + } + + event.Source = msg.Source + + message.Event = &event + + return l.postMessage(&message) +} + +func (l *splunkLoggerRaw) Log(msg *logger.Message) error { + message := l.createSplunkMessage(msg) + + message.Event = string(append(l.prefix, msg.Line...)) + + return l.postMessage(&message) +} + +func (l *splunkLogger) postMessage(message *splunkMessage) error { jsonEvent, err := json.Marshal(&message) if err != nil { return err @@ -196,6 +308,12 @@ func (l *splunkLogger) Name() string { return driverName } +func (l *splunkLogger) createSplunkMessage(msg *logger.Message) splunkMessage { + message := *l.nullMessage + message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000) + return message +} + // ValidateLogOpt looks for all supported by splunk driver options func ValidateLogOpt(cfg map[string]string) error { for key := range cfg { @@ -208,6 +326,8 @@ func ValidateLogOpt(cfg map[string]string) error { case splunkCAPathKey: case splunkCANameKey: case splunkInsecureSkipVerifyKey: + case splunkFormatKey: + case splunkVerifyConnectionKey: case envKey: case labelsKey: case tagKey: diff --git a/docs/admin/logging/splunk.md b/docs/admin/logging/splunk.md index d33f964778..5f03c32ff5 100644 --- a/docs/admin/logging/splunk.md +++ b/docs/admin/logging/splunk.md @@ -42,6 +42,8 @@ logging driver options: | `splunk-capath` | optional | Path to root certificate. | | `splunk-caname` | optional | Name to use for validating server certificate; by default the hostname of the `splunk-url` will be used. | | `splunk-insecureskipverify` | optional | Ignore server certificate validation. | +| `splunk-format` | optional | Message format. Can be `inline`, `json` or `raw`. Defaults to `inline`. | +| `splunk-verify-connection` | optional | Verify on start, that docker can connect to Splunk server. Defaults to true. | | `tag` | optional | Specify tag for message, which interpret some markup. Default value is `{{.ID}}` (12 characters of the container ID). Refer to the [log tag option documentation](log_tags.md) for customizing the log tag format. | | `labels` | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. | | `env` | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. | @@ -66,3 +68,67 @@ The `SplunkServerDefaultCert` is automatically generated by Splunk certificates. --env "TEST=false" --label location=west your/application + +### Message formats + +By default Logging Driver sends messages as `inline` format, where each message +will be embedded as a string, for example + +``` +{ + "attrs": { + "env1": "val1", + "label1": "label1" + }, + "tag": "MyImage/MyContainer", + "source": "stdout", + "line": "my message" +} +{ + "attrs": { + "env1": "val1", + "label1": "label1" + }, + "tag": "MyImage/MyContainer", + "source": "stdout", + "line": "{\"foo\": \"bar\"}" +} +``` + +In case if your messages are JSON objects you may want to embed them in the +message we send to Splunk. By specifying `--log-opt splunk-format=json` driver +will try to parse every line as a JSON object and send it as embedded object. In +case if it cannot parse it - message will be send as `inline`. For example + + +``` +{ + "attrs": { + "env1": "val1", + "label1": "label1" + }, + "tag": "MyImage/MyContainer", + "source": "stdout", + "line": "my message" +} +{ + "attrs": { + "env1": "val1", + "label1": "label1" + }, + "tag": "MyImage/MyContainer", + "source": "stdout", + "line": { + "foo": "bar" + } +} +``` + +Third format is a `raw` message. You can specify it by using +`--log-opt splunk-format=raw`. Attributes (environment variables and labels) and +tag will be prefixed to the message. For example + +``` +MyImage/MyContainer env1=val1 label1=label1 my message +MyImage/MyContainer env1=val1 label1=label1 {"foo": "bar"} +``` From 9407322972a93fa5a47436a8a9f0133b49806e6c Mon Sep 17 00:00:00 2001 From: Denis Gladkikh Date: Thu, 18 Aug 2016 09:17:06 -0700 Subject: [PATCH 2/2] Splunk Logging Driver: PR feedback Signed-off-by: Denis Gladkikh --- daemon/logger/splunk/splunk.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/daemon/logger/splunk/splunk.go b/daemon/logger/splunk/splunk.go index 0dd64e15be..15bff45110 100644 --- a/daemon/logger/splunk/splunk.go +++ b/daemon/logger/splunk/splunk.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "strconv" + "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" @@ -47,17 +48,17 @@ type splunkLogger struct { } type splunkLoggerInline struct { - splunkLogger + *splunkLogger nullEvent *splunkMessageEvent } type splunkLoggerJSON struct { - splunkLoggerInline + *splunkLoggerInline } type splunkLoggerRaw struct { - splunkLogger + *splunkLogger prefix []byte } @@ -209,14 +210,14 @@ func New(ctx logger.Context) (logger.Logger, error) { Attrs: attrs, } - return &splunkLoggerInline{*logger, nullEvent}, nil + return &splunkLoggerInline{logger, nullEvent}, nil case splunkFormatJSON: nullEvent := &splunkMessageEvent{ Tag: tag, Attrs: attrs, } - return &splunkLoggerJSON{splunkLoggerInline{*logger, nullEvent}}, nil + return &splunkLoggerJSON{&splunkLoggerInline{logger, nullEvent}}, nil case splunkFormatRaw: var prefix bytes.Buffer prefix.WriteString(tag) @@ -228,7 +229,7 @@ func New(ctx logger.Context) (logger.Logger, error) { prefix.WriteString(" ") } - return &splunkLoggerRaw{*logger, prefix.Bytes()}, nil + return &splunkLoggerRaw{logger, prefix.Bytes()}, nil default: return nil, fmt.Errorf("Unexpected format %s", splunkFormat) } @@ -243,7 +244,7 @@ func (l *splunkLoggerInline) Log(msg *logger.Message) error { message.Event = &event - return l.postMessage(&message) + return l.postMessage(message) } func (l *splunkLoggerJSON) Log(msg *logger.Message) error { @@ -261,7 +262,7 @@ func (l *splunkLoggerJSON) Log(msg *logger.Message) error { message.Event = &event - return l.postMessage(&message) + return l.postMessage(message) } func (l *splunkLoggerRaw) Log(msg *logger.Message) error { @@ -269,11 +270,11 @@ func (l *splunkLoggerRaw) Log(msg *logger.Message) error { message.Event = string(append(l.prefix, msg.Line...)) - return l.postMessage(&message) + return l.postMessage(message) } func (l *splunkLogger) postMessage(message *splunkMessage) error { - jsonEvent, err := json.Marshal(&message) + jsonEvent, err := json.Marshal(message) if err != nil { return err } @@ -308,10 +309,10 @@ func (l *splunkLogger) Name() string { return driverName } -func (l *splunkLogger) createSplunkMessage(msg *logger.Message) splunkMessage { +func (l *splunkLogger) createSplunkMessage(msg *logger.Message) *splunkMessage { message := *l.nullMessage - message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000) - return message + message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/float64(time.Second)) + return &message } // ValidateLogOpt looks for all supported by splunk driver options