Splunk Logging Driver: formats and verifyconnection

`--log-opt splunk-format=inline|json|raw` allows to change how logging
driver sends data to Splunk, where

`inline` - default value, format used before, message is injected as a
line in JSON payload
`json` - driver will try to parse each line as a JSON object and embed it
inside of the JSON payload
`raw` - driver will send Raw payload instead of JSON, tag and attributes
will be prefixed before the message

`--log-opt splunk-verify-connection=true|false` - allows to skip
verification for Splunk Url

Signed-off-by: Denis Gladkikh <denis@gladkikh.email>

contrib/completion/bash/docker

@@ -520,7 +520,7 @@ __docker_complete_log_options() {
 	local journald_options="env labels tag"
 	local json_file_options="env labels max-file max-size"
 	local syslog_options="env labels syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag"
-	local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag"
+	local splunk_options="env labels splunk-caname splunk-capath splunk-format splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag"
 
 	local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
 
@@ -629,10 +629,14 @@ __docker_complete_log_driver_options() {
 			__ltrim_colon_completions "${cur}"
 			return
 			;;
-		splunk-insecureskipverify)
+		splunk-insecureskipverify|splunk-verify-connection)
 			COMPREPLY=( $( compgen -W "false true" -- "${cur##*=}" ) )
 			return
 			;;
+		splunk-format)
+			COMPREPLY=( $( compgen -W "inline json raw" -- "${cur##*=}" ) )
+			return
+			;;
 	esac
 	return 1
 }

contrib/completion/zsh/_docker

@@ -228,7 +228,7 @@ __docker_get_log_options() {
     journald_options=("env" "labels" "tag")
     json_file_options=("env" "labels" "max-file" "max-size")
     syslog_options=("env" "labels" "syslog-address" "syslog-facility" "syslog-format" "syslog-tls-ca-cert" "syslog-tls-cert" "syslog-tls-key" "syslog-tls-skip-verify" "tag")
-    splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "tag")
+    splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-format" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "splunk-verify-connection" "tag")
 
     [[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0
     [[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0

daemon/logger/splunk/splunk.go

@@ -30,6 +30,8 @@ const (
 	splunkCAPathKey             = "splunk-capath"
 	splunkCANameKey             = "splunk-caname"
 	splunkInsecureSkipVerifyKey = "splunk-insecureskipverify"
+	splunkFormatKey             = "splunk-format"
+	splunkVerifyConnectionKey   = "splunk-verify-connection"
 	envKey                      = "env"
 	labelsKey                   = "labels"
 	tagKey                      = "tag"
@@ -44,8 +46,24 @@ type splunkLogger struct {
 	nullMessage *splunkMessage
 }
 
+type splunkLoggerInline struct {
+	splunkLogger
+
+	nullEvent *splunkMessageEvent
+}
+
+type splunkLoggerJSON struct {
+	splunkLoggerInline
+}
+
+type splunkLoggerRaw struct {
+	splunkLogger
+
+	prefix []byte
+}
+
 type splunkMessage struct {
-	Event      splunkMessageEvent `json:"event"`
+	Event      interface{} `json:"event"`
 	Time       string      `json:"time"`
 	Host       string      `json:"host"`
 	Source     string      `json:"source,omitempty"`
@@ -54,12 +72,18 @@ type splunkMessage struct {
 }
 
 type splunkMessageEvent struct {
-	Line   string            `json:"line"`
+	Line   interface{}       `json:"line"`
 	Source string            `json:"source"`
 	Tag    string            `json:"tag,omitempty"`
 	Attrs  map[string]string `json:"attrs,omitempty"`
 }
 
+const (
+	splunkFormatRaw    = "raw"
+	splunkFormatJSON   = "json"
+	splunkFormatInline = "inline"
+)
+
 func init() {
 	if err := logger.RegisterLogDriver(driverName, New); err != nil {
 		logrus.Fatal(err)
@@ -122,21 +146,23 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		Transport: transport,
 	}
 
+	source := ctx.Config[splunkSourceKey]
+	sourceType := ctx.Config[splunkSourceTypeKey]
+	index := ctx.Config[splunkIndexKey]
+
 	var nullMessage = &splunkMessage{
 		Host:       hostname,
+		Source:     source,
+		SourceType: sourceType,
+		Index:      index,
 	}
 
-	// Optional parameters for messages
-	nullMessage.Source = ctx.Config[splunkSourceKey]
-	nullMessage.SourceType = ctx.Config[splunkSourceTypeKey]
-	nullMessage.Index = ctx.Config[splunkIndexKey]
-
 	tag, err := loggerutils.ParseLogTag(ctx, loggerutils.DefaultTemplate)
 	if err != nil {
 		return nil, err
 	}
-	nullMessage.Event.Tag = tag
+
-	nullMessage.Event.Attrs = ctx.ExtraAttributes(nil)
+	attrs := ctx.ExtraAttributes(nil)
 
 	logger := &splunkLogger{
 		client:      client,
@@ -146,21 +172,107 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		nullMessage: nullMessage,
 	}
 
+	// By default we verify connection, but we allow use to skip that
+	verifyConnection := true
+	if verifyConnectionStr, ok := ctx.Config[splunkVerifyConnectionKey]; ok {
+		var err error
+		verifyConnection, err = strconv.ParseBool(verifyConnectionStr)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if verifyConnection {
 		err = verifySplunkConnection(logger)
 		if err != nil {
 			return nil, err
 		}
-
-	return logger, nil
 	}
 
-func (l *splunkLogger) Log(msg *logger.Message) error {
+	var splunkFormat string
-	// Construct message as a copy of nullMessage
+	if splunkFormatParsed, ok := ctx.Config[splunkFormatKey]; ok {
-	message := *l.nullMessage
+		switch splunkFormatParsed {
-	message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000)
+		case splunkFormatInline:
-	message.Event.Line = string(msg.Line)
+		case splunkFormatJSON:
-	message.Event.Source = msg.Source
+		case splunkFormatRaw:
+		default:
+			return nil, fmt.Errorf("Unknown format specified %s, supported formats are inline, json and raw", splunkFormat)
+		}
+		splunkFormat = splunkFormatParsed
+	} else {
+		splunkFormat = splunkFormatInline
+	}
 
+	switch splunkFormat {
+	case splunkFormatInline:
+		nullEvent := &splunkMessageEvent{
+			Tag:   tag,
+			Attrs: attrs,
+		}
+
+		return &splunkLoggerInline{*logger, nullEvent}, nil
+	case splunkFormatJSON:
+		nullEvent := &splunkMessageEvent{
+			Tag:   tag,
+			Attrs: attrs,
+		}
+
+		return &splunkLoggerJSON{splunkLoggerInline{*logger, nullEvent}}, nil
+	case splunkFormatRaw:
+		var prefix bytes.Buffer
+		prefix.WriteString(tag)
+		prefix.WriteString(" ")
+		for key, value := range attrs {
+			prefix.WriteString(key)
+			prefix.WriteString("=")
+			prefix.WriteString(value)
+			prefix.WriteString(" ")
+		}
+
+		return &splunkLoggerRaw{*logger, prefix.Bytes()}, nil
+	default:
+		return nil, fmt.Errorf("Unexpected format %s", splunkFormat)
+	}
+}
+
+func (l *splunkLoggerInline) Log(msg *logger.Message) error {
+	message := l.createSplunkMessage(msg)
+
+	event := *l.nullEvent
+	event.Line = string(msg.Line)
+	event.Source = msg.Source
+
+	message.Event = &event
+
+	return l.postMessage(&message)
+}
+
+func (l *splunkLoggerJSON) Log(msg *logger.Message) error {
+	message := l.createSplunkMessage(msg)
+	event := *l.nullEvent
+
+	var rawJSONMessage json.RawMessage
+	if err := json.Unmarshal(msg.Line, &rawJSONMessage); err == nil {
+		event.Line = &rawJSONMessage
+	} else {
+		event.Line = string(msg.Line)
+	}
+
+	event.Source = msg.Source
+
+	message.Event = &event
+
+	return l.postMessage(&message)
+}
+
+func (l *splunkLoggerRaw) Log(msg *logger.Message) error {
+	message := l.createSplunkMessage(msg)
+
+	message.Event = string(append(l.prefix, msg.Line...))
+
+	return l.postMessage(&message)
+}
+
+func (l *splunkLogger) postMessage(message *splunkMessage) error {
 	jsonEvent, err := json.Marshal(&message)
 	if err != nil {
 		return err
@@ -196,6 +308,12 @@ func (l *splunkLogger) Name() string {
 	return driverName
 }
 
+func (l *splunkLogger) createSplunkMessage(msg *logger.Message) splunkMessage {
+	message := *l.nullMessage
+	message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000)
+	return message
+}
+
 // ValidateLogOpt looks for all supported by splunk driver options
 func ValidateLogOpt(cfg map[string]string) error {
 	for key := range cfg {
@@ -208,6 +326,8 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case splunkCAPathKey:
 		case splunkCANameKey:
 		case splunkInsecureSkipVerifyKey:
+		case splunkFormatKey:
+		case splunkVerifyConnectionKey:
 		case envKey:
 		case labelsKey:
 		case tagKey:

docs/admin/logging/splunk.md

@@ -42,6 +42,8 @@ logging driver options:
 | `splunk-capath`             | optional | Path to root certificate.                                                                                                                                                                                          |
 | `splunk-caname`             | optional | Name to use for validating server certificate; by default the hostname of the `splunk-url` will be used.                                                                                                           |
 | `splunk-insecureskipverify` | optional | Ignore server certificate validation.                                                                                                                                                                              |
+| `splunk-format`             | optional | Message format. Can be `inline`, `json` or `raw`. Defaults to `inline`.                                                                                                                                            |
+| `splunk-verify-connection`   | optional | Verify on start, that docker can connect to Splunk server. Defaults to true.                                                                                                                                       |
 | `tag`                       | optional | Specify tag for message, which interpret some markup. Default value is `{{.ID}}` (12 characters of the container ID). Refer to the [log tag option documentation](log_tags.md) for customizing the log tag format. |
 | `labels`                    | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container.                                                                                          |
 | `env`                       | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container.                                                                        |
@@ -66,3 +68,67 @@ The `SplunkServerDefaultCert` is automatically generated by Splunk certificates.
         --env "TEST=false"
         --label location=west
         your/application
+
+### Message formats
+
+By default Logging Driver sends messages as `inline` format, where each message
+will be embedded as a string, for example
+
+```
+{
+    "attrs": {
+        "env1": "val1",
+        "label1": "label1"
+    },
+    "tag": "MyImage/MyContainer",
+    "source":  "stdout",
+    "line": "my message"
+}
+{
+    "attrs": {
+        "env1": "val1",
+        "label1": "label1"
+    },
+    "tag": "MyImage/MyContainer",
+    "source":  "stdout",
+    "line": "{\"foo\": \"bar\"}"
+}
+```
+
+In case if your messages are JSON objects you may want to embed them in the
+message we send to Splunk. By specifying `--log-opt splunk-format=json` driver
+will try to parse every line as a JSON object and send it as embedded object. In
+case if it cannot parse it - message will be send as `inline`. For example
+
+
+```
+{
+    "attrs": {
+        "env1": "val1",
+        "label1": "label1"
+    },
+    "tag": "MyImage/MyContainer",
+    "source":  "stdout",
+    "line": "my message"
+}
+{
+    "attrs": {
+        "env1": "val1",
+        "label1": "label1"
+    },
+    "tag": "MyImage/MyContainer",
+    "source":  "stdout",
+    "line": {
+        "foo": "bar"
+    }
+}
+```
+
+Third format is a `raw` message. You can specify it by using
+`--log-opt splunk-format=raw`. Attributes (environment variables and labels) and
+tag will be prefixed to the message. For example
+
+```
+MyImage/MyContainer env1=val1 label1=label1 my message
+MyImage/MyContainer env1=val1 label1=label1 {"foo": "bar"}
+```