From 9758a2a72445eb767ea81b44a73aabb5a8005bab Mon Sep 17 00:00:00 2001 From: Joseph Rothrock Date: Tue, 8 Nov 2016 16:34:47 -0800 Subject: [PATCH] Use a regex to match environment variables #27565 Signed-off-by: Joseph Rothrock --- daemon/logger/fluentd/fluentd.go | 6 +++- daemon/logger/gcplogs/gcplogging.go | 24 ++++++++----- daemon/logger/gelf/gelf.go | 8 ++++- daemon/logger/journald/journald.go | 6 +++- daemon/logger/jsonfilelog/jsonfilelog.go | 7 +++- daemon/logger/jsonfilelog/jsonfilelog_test.go | 15 ++++---- daemon/logger/logentries/logentries.go | 1 + daemon/logger/loginfo.go | 34 ++++++++++++++----- daemon/logger/splunk/splunk.go | 7 +++- daemon/logger/splunk/splunk_test.go | 14 +++++--- daemon/logger/syslog/syslog.go | 1 + 11 files changed, 89 insertions(+), 34 deletions(-) diff --git a/daemon/logger/fluentd/fluentd.go b/daemon/logger/fluentd/fluentd.go index 8c0da26e76..6b1fc67039 100644 --- a/daemon/logger/fluentd/fluentd.go +++ b/daemon/logger/fluentd/fluentd.go @@ -78,7 +78,10 @@ func New(info logger.Info) (logger.Logger, error) { return nil, err } - extra := info.ExtraAttributes(nil) + extra, err := info.ExtraAttributes(nil) + if err != nil { + return nil, err + } bufferLimit := defaultBufferLimit if info.Config[bufferLimitKey] != "" { @@ -169,6 +172,7 @@ func ValidateLogOpt(cfg map[string]string) error { for key := range cfg { switch key { case "env": + case "env-regex": case "labels": case "tag": case addressKey: diff --git a/daemon/logger/gcplogs/gcplogging.go b/daemon/logger/gcplogs/gcplogging.go index ff1cb39c30..82f7ab45d0 100644 --- a/daemon/logger/gcplogs/gcplogging.go +++ b/daemon/logger/gcplogs/gcplogging.go @@ -17,13 +17,14 @@ import ( const ( name = "gcplogs" - projectOptKey = "gcp-project" - logLabelsKey = "labels" - logEnvKey = "env" - logCmdKey = "gcp-log-cmd" - logZoneKey = "gcp-meta-zone" - logNameKey = "gcp-meta-name" - logIDKey = "gcp-meta-id" + projectOptKey = "gcp-project" + logLabelsKey = "labels" + logEnvKey = "env" + logEnvRegexKey = "env-regex" + logCmdKey = "gcp-log-cmd" + logZoneKey = "gcp-meta-zone" + logNameKey = "gcp-meta-name" + logIDKey = "gcp-meta-id" ) var ( @@ -133,6 +134,11 @@ func New(info logger.Info) (logger.Logger, error) { return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err) } + extraAttributes, err := info.ExtraAttributes(nil) + if err != nil { + return nil, err + } + l := &gcplogs{ logger: lg, container: &containerInfo{ @@ -141,7 +147,7 @@ func New(info logger.Info) (logger.Logger, error) { ImageName: info.ContainerImageName, ImageID: info.ContainerImageID, Created: info.ContainerCreated, - Metadata: info.ExtraAttributes(nil), + Metadata: extraAttributes, }, } @@ -185,7 +191,7 @@ func New(info logger.Info) (logger.Logger, error) { func ValidateLogOpts(cfg map[string]string) error { for k := range cfg { switch k { - case projectOptKey, logLabelsKey, logEnvKey, logCmdKey, logZoneKey, logNameKey, logIDKey: + case projectOptKey, logLabelsKey, logEnvKey, logEnvRegexKey, logCmdKey, logZoneKey, logNameKey, logIDKey: default: return fmt.Errorf("%q is not a valid option for the gcplogs driver", k) } diff --git a/daemon/logger/gelf/gelf.go b/daemon/logger/gelf/gelf.go index 42b9570495..63b4bbf02b 100644 --- a/daemon/logger/gelf/gelf.go +++ b/daemon/logger/gelf/gelf.go @@ -69,12 +69,17 @@ func New(info logger.Info) (logger.Logger, error) { "_created": info.ContainerCreated, } - extraAttrs := info.ExtraAttributes(func(key string) string { + extraAttrs, err := info.ExtraAttributes(func(key string) string { if key[0] == '_' { return key } return "_" + key }) + + if err != nil { + return nil, err + } + for k, v := range extraAttrs { extra[k] = v } @@ -156,6 +161,7 @@ func ValidateLogOpt(cfg map[string]string) error { case "tag": case "labels": case "env": + case "env-regex": case "gelf-compression-level": i, err := strconv.Atoi(val) if err != nil || i < flate.DefaultCompression || i > flate.BestCompression { diff --git a/daemon/logger/journald/journald.go b/daemon/logger/journald/journald.go index 0a16aafd94..d5204206df 100644 --- a/daemon/logger/journald/journald.go +++ b/daemon/logger/journald/journald.go @@ -75,7 +75,10 @@ func New(info logger.Info) (logger.Logger, error) { "CONTAINER_NAME": info.Name(), "CONTAINER_TAG": tag, } - extraAttrs := info.ExtraAttributes(sanitizeKeyMod) + extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod) + if err != nil { + return nil, err + } for k, v := range extraAttrs { vars[k] = v } @@ -89,6 +92,7 @@ func validateLogOpt(cfg map[string]string) error { switch key { case "labels": case "env": + case "env-regex": case "tag": default: return fmt.Errorf("unknown log opt '%s' for journald log driver", key) diff --git a/daemon/logger/jsonfilelog/jsonfilelog.go b/daemon/logger/jsonfilelog/jsonfilelog.go index eb25e419af..4a720df6a2 100644 --- a/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/daemon/logger/jsonfilelog/jsonfilelog.go @@ -67,7 +67,11 @@ func New(info logger.Info) (logger.Logger, error) { } var extra []byte - if attrs := info.ExtraAttributes(nil); len(attrs) > 0 { + attrs, err := info.ExtraAttributes(nil) + if err != nil { + return nil, err + } + if len(attrs) > 0 { var err error extra, err = json.Marshal(attrs) if err != nil { @@ -121,6 +125,7 @@ func ValidateLogOpt(cfg map[string]string) error { case "max-size": case "labels": case "env": + case "env-regex": default: return fmt.Errorf("unknown log opt '%s' for json-file log driver", key) } diff --git a/daemon/logger/jsonfilelog/jsonfilelog_test.go b/daemon/logger/jsonfilelog/jsonfilelog_test.go index d6091efd14..d2d36e943b 100644 --- a/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -160,13 +160,13 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) { } defer os.RemoveAll(tmp) filename := filepath.Join(tmp, "container.log") - config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"} + config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl", "env-regex": "^dc"} l, err := New(logger.Info{ ContainerID: cid, LogPath: filename, Config: config, ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"}, - ContainerEnv: []string{"environ=production", "debug=false", "port=10001", "ssl=true"}, + ContainerEnv: []string{"environ=production", "debug=false", "port=10001", "ssl=true", "dc_region=west"}, }) if err != nil { t.Fatal(err) @@ -189,11 +189,12 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) { t.Fatal(err) } expected := map[string]string{ - "rack": "101", - "dc": "lhr", - "environ": "production", - "debug": "false", - "ssl": "true", + "rack": "101", + "dc": "lhr", + "environ": "production", + "debug": "false", + "ssl": "true", + "dc_region": "west", } if !reflect.DeepEqual(extra, expected) { t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected) diff --git a/daemon/logger/logentries/logentries.go b/daemon/logger/logentries/logentries.go index 114ddd59d4..6ea9fb5daf 100644 --- a/daemon/logger/logentries/logentries.go +++ b/daemon/logger/logentries/logentries.go @@ -78,6 +78,7 @@ func ValidateLogOpt(cfg map[string]string) error { for key := range cfg { switch key { case "env": + case "env-regex": case "labels": case "tag": case key: diff --git a/daemon/logger/loginfo.go b/daemon/logger/loginfo.go index c0104e06b8..4c930b9056 100644 --- a/daemon/logger/loginfo.go +++ b/daemon/logger/loginfo.go @@ -3,6 +3,7 @@ package logger import ( "fmt" "os" + "regexp" "strings" "time" ) @@ -26,7 +27,7 @@ type Info struct { // ExtraAttributes returns the user-defined extra attributes (labels, // environment variables) in key-value format. This can be used by log drivers // that support metadata to add more context to a log. -func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string { +func (info *Info) ExtraAttributes(keyMod func(string) string) (map[string]string, error) { extra := make(map[string]string) labels, ok := info.Config["labels"] if ok && len(labels) > 0 { @@ -40,14 +41,15 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string } } + envMapping := make(map[string]string) + for _, e := range info.ContainerEnv { + if kv := strings.SplitN(e, "=", 2); len(kv) == 2 { + envMapping[kv[0]] = kv[1] + } + } + env, ok := info.Config["env"] if ok && len(env) > 0 { - envMapping := make(map[string]string) - for _, e := range info.ContainerEnv { - if kv := strings.SplitN(e, "=", 2); len(kv) == 2 { - envMapping[kv[0]] = kv[1] - } - } for _, l := range strings.Split(env, ",") { if v, ok := envMapping[l]; ok { if keyMod != nil { @@ -58,7 +60,23 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string } } - return extra + envRegex, ok := info.Config["env-regex"] + if ok && len(envRegex) > 0 { + re, err := regexp.Compile(envRegex) + if err != nil { + return nil, err + } + for k, v := range envMapping { + if re.MatchString(k) { + if keyMod != nil { + k = keyMod(k) + } + extra[k] = v + } + } + } + + return extra, nil } // Hostname returns the hostname from the underlying OS. diff --git a/daemon/logger/splunk/splunk.go b/daemon/logger/splunk/splunk.go index 3ae6da71b3..f2c6bd243b 100644 --- a/daemon/logger/splunk/splunk.go +++ b/daemon/logger/splunk/splunk.go @@ -39,6 +39,7 @@ const ( splunkGzipCompressionKey = "splunk-gzip" splunkGzipCompressionLevelKey = "splunk-gzip-level" envKey = "env" + envRegexKey = "env-regex" labelsKey = "labels" tagKey = "tag" ) @@ -235,7 +236,10 @@ func New(info logger.Info) (logger.Logger, error) { } } - attrs := info.ExtraAttributes(nil) + attrs, err := info.ExtraAttributes(nil) + if err != nil { + return nil, err + } var ( postMessagesFrequency = getAdvancedOptionDuration(envVarPostMessagesFrequency, defaultPostMessagesFrequency) @@ -538,6 +542,7 @@ func ValidateLogOpt(cfg map[string]string) error { case splunkGzipCompressionKey: case splunkGzipCompressionLevelKey: case envKey: + case envRegexKey: case labelsKey: case tagKey: default: diff --git a/daemon/logger/splunk/splunk_test.go b/daemon/logger/splunk/splunk_test.go index e7e3d68744..84f7529886 100644 --- a/daemon/logger/splunk/splunk_test.go +++ b/daemon/logger/splunk/splunk_test.go @@ -25,9 +25,10 @@ func TestValidateLogOpt(t *testing.T) { splunkVerifyConnectionKey: "true", splunkGzipCompressionKey: "true", splunkGzipCompressionLevelKey: "1", - envKey: "a", - labelsKey: "b", - tagKey: "c", + envKey: "a", + envRegexKey: "^foo", + labelsKey: "b", + tagKey: "c", }) if err != nil { t.Fatal(err) @@ -215,8 +216,9 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) { splunkIndexKey: "myindex", splunkFormatKey: splunkFormatInline, splunkGzipCompressionKey: "true", - tagKey: "{{.ImageName}}/{{.Name}}", - labelsKey: "a", + tagKey: "{{.ImageName}}/{{.Name}}", + labelsKey: "a", + envRegexKey: "^foo", }, ContainerID: "containeriid", ContainerName: "/container_name", @@ -225,6 +227,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) { ContainerLabels: map[string]string{ "a": "b", }, + ContainerEnv: []string{"foo_finder=bar"}, } hostname, err := info.Hostname() @@ -295,6 +298,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) { event["source"] != "stdout" || event["tag"] != "container_image_name/container_name" || event["attrs"].(map[string]interface{})["a"] != "b" || + event["attrs"].(map[string]interface{})["foo_finder"] != "bar" || len(event) != 4 { t.Fatalf("Unexpected event in message %v", event) } diff --git a/daemon/logger/syslog/syslog.go b/daemon/logger/syslog/syslog.go index f765b7d976..b4ec01780e 100644 --- a/daemon/logger/syslog/syslog.go +++ b/daemon/logger/syslog/syslog.go @@ -184,6 +184,7 @@ func ValidateLogOpt(cfg map[string]string) error { for key := range cfg { switch key { case "env": + case "env-regex": case "labels": case "syslog-address": case "syslog-facility":