kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

Merge pull request #25786 from splunk/splunk-logging-driver-raw-format-using-json

Browse source 
Splunk Logging Driver: formats and verifyconnection
This commit is contained in:
Alexander Morozov 2016-08-25 09:12:31 -07:00 committed by GitHub
commit 78771b6225
4 changed files with 221 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
contrib/completion/bash/docker
View file

@ -520,7 +520,7 @@ __docker_complete_log_options() {
local journald_options="env labels tag"
local json_file_options="env labels max-file max-size"
local syslog_options="env labels syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag"
local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag"
local splunk_options="env labels splunk-caname splunk-capath splunk-format splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag"
local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
@ -629,10 +629,14 @@ __docker_complete_log_driver_options() {
__ltrim_colon_completions "${cur}"
return
;;
splunk-insecureskipverify)
splunk-insecureskipverify|splunk-verify-connection)
COMPREPLY=( $( compgen -W "false true" -- "${cur##*=}" ) )
return
;;
splunk-format)
COMPREPLY=( $( compgen -W "inline json raw" -- "${cur##*=}" ) )
return
;;
esac
return 1
}

2
contrib/completion/zsh/_docker
View file

@ -228,7 +228,7 @@ __docker_get_log_options() {
journald_options=("env" "labels" "tag")
json_file_options=("env" "labels" "max-file" "max-size")
syslog_options=("env" "labels" "syslog-address" "syslog-facility" "syslog-format" "syslog-tls-ca-cert" "syslog-tls-cert" "syslog-tls-key" "syslog-tls-skip-verify" "tag")
splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "tag")
splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-format" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "splunk-verify-connection" "tag")
[[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0
[[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0

175
daemon/logger/splunk/splunk.go
View file

@ -13,6 +13,7 @@ import (
"net/http"
"net/url"
"strconv"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/daemon/logger"
@ -30,6 +31,8 @@ const (
splunkCAPathKey = "splunk-capath"
splunkCANameKey = "splunk-caname"
splunkInsecureSkipVerifyKey = "splunk-insecureskipverify"
splunkFormatKey = "splunk-format"
splunkVerifyConnectionKey = "splunk-verify-connection"
envKey = "env"
labelsKey = "labels"
tagKey = "tag"
@ -44,22 +47,44 @@ type splunkLogger struct {
nullMessage *splunkMessage
}
type splunkLoggerInline struct {
*splunkLogger
nullEvent *splunkMessageEvent
}
type splunkLoggerJSON struct {
*splunkLoggerInline
}
type splunkLoggerRaw struct {
*splunkLogger
prefix []byte
}
type splunkMessage struct {
Event splunkMessageEvent `json:"event"`
Time string `json:"time"`
Host string `json:"host"`
Source string `json:"source,omitempty"`
SourceType string `json:"sourcetype,omitempty"`
Index string `json:"index,omitempty"`
Event interface{} `json:"event"`
Time string `json:"time"`
Host string `json:"host"`
Source string `json:"source,omitempty"`
SourceType string `json:"sourcetype,omitempty"`
Index string `json:"index,omitempty"`
}
type splunkMessageEvent struct {
Line string `json:"line"`
Line interface{} `json:"line"`
Source string `json:"source"`
Tag string `json:"tag,omitempty"`
Attrs map[string]string `json:"attrs,omitempty"`
}
const (
splunkFormatRaw = "raw"
splunkFormatJSON = "json"
splunkFormatInline = "inline"
)
func init() {
if err := logger.RegisterLogDriver(driverName, New); err != nil {
logrus.Fatal(err)
@ -122,21 +147,23 @@ func New(ctx logger.Context) (logger.Logger, error) {
Transport: transport,
}
var nullMessage = &splunkMessage{
Host: hostname,
}
source := ctx.Config[splunkSourceKey]
sourceType := ctx.Config[splunkSourceTypeKey]
index := ctx.Config[splunkIndexKey]
// Optional parameters for messages
nullMessage.Source = ctx.Config[splunkSourceKey]
nullMessage.SourceType = ctx.Config[splunkSourceTypeKey]
nullMessage.Index = ctx.Config[splunkIndexKey]
var nullMessage = &splunkMessage{
Host: hostname,
Source: source,
SourceType: sourceType,
Index: index,
}
tag, err := loggerutils.ParseLogTag(ctx, loggerutils.DefaultTemplate)
if err != nil {
return nil, err
}
nullMessage.Event.Tag = tag
nullMessage.Event.Attrs = ctx.ExtraAttributes(nil)
attrs := ctx.ExtraAttributes(nil)
logger := &splunkLogger{
client: client,
@ -146,22 +173,108 @@ func New(ctx logger.Context) (logger.Logger, error) {
nullMessage: nullMessage,
}
err = verifySplunkConnection(logger)
if err != nil {
return nil, err
// By default we verify connection, but we allow use to skip that
verifyConnection := true
if verifyConnectionStr, ok := ctx.Config[splunkVerifyConnectionKey]; ok {
var err error
verifyConnection, err = strconv.ParseBool(verifyConnectionStr)
if err != nil {
return nil, err
}
}
if verifyConnection {
err = verifySplunkConnection(logger)
if err != nil {
return nil, err
}
}
return logger, nil
var splunkFormat string
if splunkFormatParsed, ok := ctx.Config[splunkFormatKey]; ok {
switch splunkFormatParsed {
case splunkFormatInline:
case splunkFormatJSON:
case splunkFormatRaw:
default:
return nil, fmt.Errorf("Unknown format specified %s, supported formats are inline, json and raw", splunkFormat)
}
splunkFormat = splunkFormatParsed
} else {
splunkFormat = splunkFormatInline
}
switch splunkFormat {
case splunkFormatInline:
nullEvent := &splunkMessageEvent{
Tag: tag,
Attrs: attrs,
}
return &splunkLoggerInline{logger, nullEvent}, nil
case splunkFormatJSON:
nullEvent := &splunkMessageEvent{
Tag: tag,
Attrs: attrs,
}
return &splunkLoggerJSON{&splunkLoggerInline{logger, nullEvent}}, nil
case splunkFormatRaw:
var prefix bytes.Buffer
prefix.WriteString(tag)
prefix.WriteString(" ")
for key, value := range attrs {
prefix.WriteString(key)
prefix.WriteString("=")
prefix.WriteString(value)
prefix.WriteString(" ")
}
return &splunkLoggerRaw{logger, prefix.Bytes()}, nil
default:
return nil, fmt.Errorf("Unexpected format %s", splunkFormat)
}
}
func (l *splunkLogger) Log(msg *logger.Message) error {
// Construct message as a copy of nullMessage
message := *l.nullMessage
message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000)
message.Event.Line = string(msg.Line)
message.Event.Source = msg.Source
func (l *splunkLoggerInline) Log(msg *logger.Message) error {
message := l.createSplunkMessage(msg)
jsonEvent, err := json.Marshal(&message)
event := *l.nullEvent
event.Line = string(msg.Line)
event.Source = msg.Source
message.Event = &event
return l.postMessage(message)
}
func (l *splunkLoggerJSON) Log(msg *logger.Message) error {
message := l.createSplunkMessage(msg)
event := *l.nullEvent
var rawJSONMessage json.RawMessage
if err := json.Unmarshal(msg.Line, &rawJSONMessage); err == nil {
event.Line = &rawJSONMessage
} else {
event.Line = string(msg.Line)
}
event.Source = msg.Source
message.Event = &event
return l.postMessage(message)
}
func (l *splunkLoggerRaw) Log(msg *logger.Message) error {
message := l.createSplunkMessage(msg)
message.Event = string(append(l.prefix, msg.Line...))
return l.postMessage(message)
}
func (l *splunkLogger) postMessage(message *splunkMessage) error {
jsonEvent, err := json.Marshal(message)
if err != nil {
return err
}
@ -196,6 +309,12 @@ func (l *splunkLogger) Name() string {
return driverName
}
func (l *splunkLogger) createSplunkMessage(msg *logger.Message) *splunkMessage {
message := *l.nullMessage
message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/float64(time.Second))
return &message
}
// ValidateLogOpt looks for all supported by splunk driver options
func ValidateLogOpt(cfg map[string]string) error {
for key := range cfg {
@ -208,6 +327,8 @@ func ValidateLogOpt(cfg map[string]string) error {
case splunkCAPathKey:
case splunkCANameKey:
case splunkInsecureSkipVerifyKey:
case splunkFormatKey:
case splunkVerifyConnectionKey:
case envKey:
case labelsKey:
case tagKey:

66
docs/admin/logging/splunk.md
View file

@ -42,6 +42,8 @@ logging driver options:
| `splunk-capath` | optional | Path to root certificate. |
| `splunk-caname` | optional | Name to use for validating server certificate; by default the hostname of the `splunk-url` will be used. |
| `splunk-insecureskipverify` | optional | Ignore server certificate validation. |
| `splunk-format` | optional | Message format. Can be `inline`, `json` or `raw`. Defaults to `inline`. |
| `splunk-verify-connection` | optional | Verify on start, that docker can connect to Splunk server. Defaults to true. |
| `tag` | optional | Specify tag for message, which interpret some markup. Default value is `{{.ID}}` (12 characters of the container ID). Refer to the [log tag option documentation](log_tags.md) for customizing the log tag format. |
| `labels` | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. |
| `env` | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. |
@ -66,3 +68,67 @@ The `SplunkServerDefaultCert` is automatically generated by Splunk certificates.
--env "TEST=false"
--label location=west
your/application
### Message formats
By default Logging Driver sends messages as `inline` format, where each message
will be embedded as a string, for example
```
{
"attrs": {
"env1": "val1",
"label1": "label1"
},
"tag": "MyImage/MyContainer",
"source": "stdout",
"line": "my message"
}
{
"attrs": {
"env1": "val1",
"label1": "label1"
},
"tag": "MyImage/MyContainer",
"source": "stdout",
"line": "{\"foo\": \"bar\"}"
}
```
In case if your messages are JSON objects you may want to embed them in the
message we send to Splunk. By specifying `--log-opt splunk-format=json` driver
will try to parse every line as a JSON object and send it as embedded object. In
case if it cannot parse it - message will be send as `inline`. For example
```
{
"attrs": {
"env1": "val1",
"label1": "label1"
},
"tag": "MyImage/MyContainer",
"source": "stdout",
"line": "my message"
}
{
"attrs": {
"env1": "val1",
"label1": "label1"
},
"tag": "MyImage/MyContainer",
"source": "stdout",
"line": {
"foo": "bar"
}
}
```
Third format is a `raw` message. You can specify it by using
`--log-opt splunk-format=raw`. Attributes (environment variables and labels) and
tag will be prefixed to the message. For example
```
MyImage/MyContainer env1=val1 label1=label1 my message
MyImage/MyContainer env1=val1 label1=label1 {"foo": "bar"}
```