From 2764d67c3a85cf638e211a1cfb0808e9eba7edb9 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 12 May 2016 11:18:08 -0300 Subject: [PATCH 1/2] added logentries driver Signed-off-by: Carlos Alexandro Becker --- daemon/logdrivers_linux.go | 1 + daemon/logdrivers_windows.go | 1 + daemon/logger/logentries/logentries.go | 94 ++++++++++++++++++++++++++ docs/admin/logging/logentries.md | 42 ++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 daemon/logger/logentries/logentries.go create mode 100644 docs/admin/logging/logentries.md diff --git a/daemon/logdrivers_linux.go b/daemon/logdrivers_linux.go index 89fe49a858..ad343c1e8e 100644 --- a/daemon/logdrivers_linux.go +++ b/daemon/logdrivers_linux.go @@ -9,6 +9,7 @@ import ( _ "github.com/docker/docker/daemon/logger/gelf" _ "github.com/docker/docker/daemon/logger/journald" _ "github.com/docker/docker/daemon/logger/jsonfilelog" + _ "github.com/docker/docker/daemon/logger/logentries" _ "github.com/docker/docker/daemon/logger/splunk" _ "github.com/docker/docker/daemon/logger/syslog" ) diff --git a/daemon/logdrivers_windows.go b/daemon/logdrivers_windows.go index 8337c0f0a8..646046814f 100644 --- a/daemon/logdrivers_windows.go +++ b/daemon/logdrivers_windows.go @@ -6,6 +6,7 @@ import ( _ "github.com/docker/docker/daemon/logger/awslogs" _ "github.com/docker/docker/daemon/logger/etwlogs" _ "github.com/docker/docker/daemon/logger/jsonfilelog" + _ "github.com/docker/docker/daemon/logger/logentries" _ "github.com/docker/docker/daemon/logger/splunk" _ "github.com/docker/docker/daemon/logger/syslog" ) diff --git a/daemon/logger/logentries/logentries.go b/daemon/logger/logentries/logentries.go new file mode 100644 index 0000000000..e794b1ed08 --- /dev/null +++ b/daemon/logger/logentries/logentries.go @@ -0,0 +1,94 @@ +// Package logentries provides the log driver for forwarding server logs +// to logentries endpoints. +package logentries + +import ( + "fmt" + + "github.com/Sirupsen/logrus" + "github.com/bsphere/le_go" + "github.com/docker/docker/daemon/logger" +) + +type logentries struct { + tag string + containerID string + containerName string + writer *le_go.Logger + extra map[string]string +} + +const ( + name = "logentries" + token = "logentries-token" +) + +func init() { + if err := logger.RegisterLogDriver(name, New); err != nil { + logrus.Fatal(err) + } + if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil { + logrus.Fatal(err) + } +} + +// New creates a logentries logger using the configuration passed in on +// the context. The supported context configuration variable is +// logentries-token. +func New(ctx logger.Context) (logger.Logger, error) { + logrus.WithField("container", ctx.ContainerID). + WithField("token", ctx.Config[token]). + Debug("logging driver logentries configured") + + log, err := le_go.Connect(ctx.Config[token]) + if err != nil { + return nil, err + } + return &logentries{ + containerID: ctx.ContainerID, + containerName: ctx.ContainerName, + writer: log, + }, nil +} + +func (f *logentries) Log(msg *logger.Message) error { + data := map[string]string{ + "container_id": f.containerID, + "container_name": f.containerName, + "source": msg.Source, + "log": string(msg.Line), + } + for k, v := range f.extra { + data[k] = v + } + f.writer.Println(f.tag, msg.Timestamp, data) + return nil +} + +func (f *logentries) Close() error { + return f.writer.Close() +} + +func (f *logentries) Name() string { + return name +} + +// ValidateLogOpt looks for logentries specific log option logentries-address. +func ValidateLogOpt(cfg map[string]string) error { + for key := range cfg { + switch key { + case "env": + case "labels": + case "tag": + case key: + default: + return fmt.Errorf("unknown log opt '%s' for logentries log driver", key) + } + } + + if cfg[token] == "" { + return fmt.Errorf("Missing logentries token") + } + + return nil +} diff --git a/docs/admin/logging/logentries.md b/docs/admin/logging/logentries.md new file mode 100644 index 0000000000..6bbee5d544 --- /dev/null +++ b/docs/admin/logging/logentries.md @@ -0,0 +1,42 @@ + + +# Logentries logging driver + +The `logentries` logging driver sends container logs to the Logentries server. + +## Usage + +You can configure the default logging driver by passing the `--log-driver` +option to the Docker daemon: + + docker daemon --log-driver=logentries + +You can set the logging driver for a specific container by using the +`--log-driver` option to `docker run`: + + docker run --log-driver=logentries ... + +## Logentries options + +You can use the `--log-opt NAME=VALUE` flag to specify these additional +Logentries logging driver options: + +| Option | Required | Description | +|-----------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `logentries-token` | required | Logentries token. | + +An example usage would be somethig like: + + docker run --log-driver=logentries \ + --log-opt logentries-token=176FCEBF-4CF5-4EDF-91BC-703796522D20 \ + your/application From 0f4db1cc59faddb0207db269fd660e8e1582b77a Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Wed, 1 Jun 2016 15:54:15 -0300 Subject: [PATCH 2/2] Added logentries dependency to vendor Signed-off-by: Carlos Alexandro Becker --- contrib/completion/bash/docker | 7 +- contrib/completion/zsh/_docker | 4 +- docs/admin/logging/logentries.md | 42 ---- hack/vendor.sh | 3 + .../src/github.com/bsphere/le_go/.gitignore | 23 ++ .../src/github.com/bsphere/le_go/.travis.yml | 4 + vendor/src/github.com/bsphere/le_go/README.md | 37 +++ vendor/src/github.com/bsphere/le_go/le.go | 216 ++++++++++++++++++ 8 files changed, 292 insertions(+), 44 deletions(-) delete mode 100644 docs/admin/logging/logentries.md create mode 100644 vendor/src/github.com/bsphere/le_go/.gitignore create mode 100644 vendor/src/github.com/bsphere/le_go/.travis.yml create mode 100644 vendor/src/github.com/bsphere/le_go/README.md create mode 100644 vendor/src/github.com/bsphere/le_go/le.go diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 11d7b9faf4..0103b25590 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -505,6 +505,7 @@ __docker_complete_log_drivers() { gelf journald json-file + logentries none splunk syslog @@ -519,10 +520,11 @@ __docker_complete_log_options() { local gelf_options="env gelf-address gelf-compression-level gelf-compression-type labels tag" local journald_options="env labels tag" local json_file_options="env labels max-file max-size" + local logentries_options="logentries-token" local syslog_options="env labels syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag" local splunk_options="env labels splunk-caname splunk-capath splunk-format splunk-gzip splunk-gzip-level splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag" - local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options" + local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $logentries_options $json_file_options $syslog_options $splunk_options" case $(__docker_value_of_option --log-driver) in '') @@ -546,6 +548,9 @@ __docker_complete_log_options() { json-file) COMPREPLY=( $( compgen -W "$json_file_options" -S = -- "$cur" ) ) ;; + logentries) + COMPREPLY=( $( compgen -W "$logentries_options" -S = -- "$cur" ) ) + ;; syslog) COMPREPLY=( $( compgen -W "$syslog_options" -S = -- "$cur" ) ) ;; diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 62dc465043..05119515f5 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -219,7 +219,7 @@ __docker_get_log_options() { integer ret=1 local log_driver=${opt_args[--log-driver]:-"all"} - local -a awslogs_options fluentd_options gelf_options journald_options json_file_options syslog_options splunk_options + local -a awslogs_options fluentd_options gelf_options journald_options json_file_options logentries_options syslog_options splunk_options awslogs_options=("awslogs-region" "awslogs-group" "awslogs-stream") fluentd_options=("env" "fluentd-address" "fluentd-async-connect" "fluentd-buffer-limit" "fluentd-retry-wait" "fluentd-max-retries" "labels" "tag") @@ -227,6 +227,7 @@ __docker_get_log_options() { gelf_options=("env" "gelf-address" "gelf-compression-level" "gelf-compression-type" "labels" "tag") journald_options=("env" "labels" "tag") json_file_options=("env" "labels" "max-file" "max-size") + logentries_options=("logentries-token") syslog_options=("env" "labels" "syslog-address" "syslog-facility" "syslog-format" "syslog-tls-ca-cert" "syslog-tls-cert" "syslog-tls-key" "syslog-tls-skip-verify" "tag") splunk_options=("env" "labels" "splunk-caname" "splunk-capath" "splunk-format" "splunk-gzip" "splunk-gzip-level" "splunk-index" "splunk-insecureskipverify" "splunk-source" "splunk-sourcetype" "splunk-token" "splunk-url" "splunk-verify-connection" "tag") @@ -236,6 +237,7 @@ __docker_get_log_options() { [[ $log_driver = (gelf|all) ]] && _describe -t gelf-options "gelf options" gelf_options "$@" && ret=0 [[ $log_driver = (journald|all) ]] && _describe -t journald-options "journald options" journald_options "$@" && ret=0 [[ $log_driver = (json-file|all) ]] && _describe -t json-file-options "json-file options" json_file_options "$@" && ret=0 + [[ $log_driver = (logentries|all) ]] && _describe -t logentries-options "logentries options" logentries_options "$@" && ret=0 [[ $log_driver = (syslog|all) ]] && _describe -t syslog-options "syslog options" syslog_options "$@" && ret=0 [[ $log_driver = (splunk|all) ]] && _describe -t splunk-options "splunk options" splunk_options "$@" && ret=0 diff --git a/docs/admin/logging/logentries.md b/docs/admin/logging/logentries.md deleted file mode 100644 index 6bbee5d544..0000000000 --- a/docs/admin/logging/logentries.md +++ /dev/null @@ -1,42 +0,0 @@ - - -# Logentries logging driver - -The `logentries` logging driver sends container logs to the Logentries server. - -## Usage - -You can configure the default logging driver by passing the `--log-driver` -option to the Docker daemon: - - docker daemon --log-driver=logentries - -You can set the logging driver for a specific container by using the -`--log-driver` option to `docker run`: - - docker run --log-driver=logentries ... - -## Logentries options - -You can use the `--log-opt NAME=VALUE` flag to specify these additional -Logentries logging driver options: - -| Option | Required | Description | -|-----------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `logentries-token` | required | Logentries token. | - -An example usage would be somethig like: - - docker run --log-driver=logentries \ - --log-opt logentries-token=176FCEBF-4CF5-4EDF-91BC-703796522D20 \ - your/application diff --git a/hack/vendor.sh b/hack/vendor.sh index 01c021ea8a..14d0917b33 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -131,6 +131,9 @@ clone git github.com/aws/aws-sdk-go v1.1.30 clone git github.com/go-ini/ini 060d7da055ba6ec5ea7a31f116332fe5efa04ce0 clone git github.com/jmespath/go-jmespath 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 +# logentries +clone git github.com/bsphere/le_go d3308aafe090956bc89a65f0769f58251a1b4f03 + # gcplogs deps clone git golang.org/x/oauth2 2baa8a1b9338cf13d9eeb27696d761155fa480be https://github.com/golang/oauth2.git clone git google.golang.org/api dc6d2353af16e2a2b0ff6986af051d473a4ed468 https://code.googlesource.com/google-api-go-client diff --git a/vendor/src/github.com/bsphere/le_go/.gitignore b/vendor/src/github.com/bsphere/le_go/.gitignore new file mode 100644 index 0000000000..836562412f --- /dev/null +++ b/vendor/src/github.com/bsphere/le_go/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/vendor/src/github.com/bsphere/le_go/.travis.yml b/vendor/src/github.com/bsphere/le_go/.travis.yml new file mode 100644 index 0000000000..aa1b91fe6c --- /dev/null +++ b/vendor/src/github.com/bsphere/le_go/.travis.yml @@ -0,0 +1,4 @@ +language: go + +go: + - 1.4 \ No newline at end of file diff --git a/vendor/src/github.com/bsphere/le_go/README.md b/vendor/src/github.com/bsphere/le_go/README.md new file mode 100644 index 0000000000..9d2ca9a988 --- /dev/null +++ b/vendor/src/github.com/bsphere/le_go/README.md @@ -0,0 +1,37 @@ +le_go +===== + +Golang client library for logentries.com + +It is compatible with http://golang.org/pkg/log/#Logger +and also implements http://golang.org/pkg/io/#Writer + +[![GoDoc](https://godoc.org/github.com/bsphere/le_go?status.png)](https://godoc.org/github.com/bsphere/le_go) + +[![Build Status](https://travis-ci.org/bsphere/le_go.svg)](https://travis-ci.org/bsphere/le_go) + +Usage +----- +Add a new manual TCP token log at [logentries.com](https://logentries.com/quick-start/) and copy the [token](https://logentries.com/doc/input-token/). + +Installation: `go get github.com/bsphere/le_go` + +**Note:** The Logger is blocking, it can be easily run in a goroutine by calling `go le.Println(...)` + +```go +package main + +import "github.com/bsphere/le_go" + +func main() { + le, err := le_go.Connect("XXXX-XXXX-XXXX-XXXX") // replace with token + if err != nil { + panic(err) + } + + defer le.Close() + + le.Println("another test message") +} +``` + diff --git a/vendor/src/github.com/bsphere/le_go/le.go b/vendor/src/github.com/bsphere/le_go/le.go new file mode 100644 index 0000000000..553e4c7054 --- /dev/null +++ b/vendor/src/github.com/bsphere/le_go/le.go @@ -0,0 +1,216 @@ +// Package le_go provides a Golang client library for logging to +// logentries.com over a TCP connection. +// +// it uses an access token for sending log events. +package le_go + +import ( + "crypto/tls" + "fmt" + "net" + "os" + "strings" + "sync" + "time" +) + +// Logger represents a Logentries logger, +// it holds the open TCP connection, access token, prefix and flags. +// +// all Logger operations are thread safe and blocking, +// log operations can be invoked in a non-blocking way by calling them from +// a goroutine. +type Logger struct { + conn net.Conn + flag int + mu sync.Mutex + prefix string + token string + buf []byte +} + +const lineSep = "\n" + +// Connect creates a new Logger instance and opens a TCP connection to +// logentries.com, +// The token can be generated at logentries.com by adding a new log, +// choosing manual configuration and token based TCP connection. +func Connect(token string) (*Logger, error) { + logger := Logger{ + token: token, + } + + if err := logger.openConnection(); err != nil { + return nil, err + } + + return &logger, nil +} + +// Close closes the TCP connection to logentries.com +func (logger *Logger) Close() error { + if logger.conn != nil { + return logger.conn.Close() + } + + return nil +} + +// Opens a TCP connection to logentries.com +func (logger *Logger) openConnection() error { + conn, err := tls.Dial("tcp", "data.logentries.com:443", &tls.Config{}) + if err != nil { + return err + } + logger.conn = conn + return nil +} + +// It returns if the TCP connection to logentries.com is open +func (logger *Logger) isOpenConnection() bool { + if logger.conn == nil { + return false + } + + buf := make([]byte, 1) + + logger.conn.SetReadDeadline(time.Now()) + + _, err := logger.conn.Read(buf) + + switch err.(type) { + case net.Error: + if err.(net.Error).Timeout() == true { + logger.conn.SetReadDeadline(time.Time{}) + + return true + } + } + + return false +} + +// It ensures that the TCP connection to logentries.com is open. +// If the connection is closed, a new one is opened. +func (logger *Logger) ensureOpenConnection() error { + if !logger.isOpenConnection() { + if err := logger.openConnection(); err != nil { + return err + } + } + + return nil +} + +// Fatal is same as Print() but calls to os.Exit(1) +func (logger *Logger) Fatal(v ...interface{}) { + logger.Output(2, fmt.Sprint(v...)) + os.Exit(1) +} + +// Fatalf is same as Printf() but calls to os.Exit(1) +func (logger *Logger) Fatalf(format string, v ...interface{}) { + logger.Output(2, fmt.Sprintf(format, v...)) + os.Exit(1) +} + +// Fatalln is same as Println() but calls to os.Exit(1) +func (logger *Logger) Fatalln(v ...interface{}) { + logger.Output(2, fmt.Sprintln(v...)) + os.Exit(1) +} + +// Flags returns the logger flags +func (logger *Logger) Flags() int { + return logger.flag +} + +// Output does the actual writing to the TCP connection +func (logger *Logger) Output(calldepth int, s string) error { + _, err := logger.Write([]byte(s)) + + return err +} + +// Panic is same as Print() but calls to panic +func (logger *Logger) Panic(v ...interface{}) { + s := fmt.Sprint(v...) + logger.Output(2, s) + panic(s) +} + +// Panicf is same as Printf() but calls to panic +func (logger *Logger) Panicf(format string, v ...interface{}) { + s := fmt.Sprintf(format, v...) + logger.Output(2, s) + panic(s) +} + +// Panicln is same as Println() but calls to panic +func (logger *Logger) Panicln(v ...interface{}) { + s := fmt.Sprintln(v...) + logger.Output(2, s) + panic(s) +} + +// Prefix returns the logger prefix +func (logger *Logger) Prefix() string { + return logger.prefix +} + +// Print logs a message +func (logger *Logger) Print(v ...interface{}) { + logger.Output(2, fmt.Sprint(v...)) +} + +// Printf logs a formatted message +func (logger *Logger) Printf(format string, v ...interface{}) { + logger.Output(2, fmt.Sprintf(format, v...)) +} + +// Println logs a message with a linebreak +func (logger *Logger) Println(v ...interface{}) { + logger.Output(2, fmt.Sprintln(v...)) +} + +// SetFlags sets the logger flags +func (logger *Logger) SetFlags(flag int) { + logger.flag = flag +} + +// SetPrefix sets the logger prefix +func (logger *Logger) SetPrefix(prefix string) { + logger.prefix = prefix +} + +// Write writes a bytes array to the Logentries TCP connection, +// it adds the access token and prefix and also replaces +// line breaks with the unicode \u2028 character +func (logger *Logger) Write(p []byte) (n int, err error) { + if err := logger.ensureOpenConnection(); err != nil { + return 0, err + } + + logger.mu.Lock() + defer logger.mu.Unlock() + + logger.makeBuf(p) + + return logger.conn.Write(logger.buf) +} + +// makeBuf constructs the logger buffer +// it is not safe to be used from within multiple concurrent goroutines +func (logger *Logger) makeBuf(p []byte) { + count := strings.Count(string(p), lineSep) + p = []byte(strings.Replace(string(p), lineSep, "\u2028", count-1)) + + logger.buf = logger.buf[:0] + logger.buf = append(logger.buf, (logger.token + " ")...) + logger.buf = append(logger.buf, (logger.prefix + " ")...) + logger.buf = append(logger.buf, p...) + + if !strings.HasSuffix(string(logger.buf), lineSep) { + logger.buf = append(logger.buf, (lineSep)...) + } +}