Merge pull request #20121 from solganik/master

syslog format

daemon/logger/syslog/syslog.go

@@ -13,6 +13,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	syslog "github.com/RackSec/srslog"
 
@@ -64,6 +65,17 @@ func init() {
 	}
 }
 
+// rsyslog uses appname part of syslog message to fill in an %syslogtag% template
+// attribute in rsyslog.conf. In order to be backward compatible to rfc3164
+// tag will be also used as an appname
+func rfc5424formatterWithAppNameAsTag(p syslog.Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.RFC3339)
+	pid := os.Getpid()
+	msg := fmt.Sprintf("<%d>%d %s %s %s %d %s %s",
+		p, 1, timestamp, hostname, tag, pid, tag, content)
+	return msg
+}
+
 // New creates a syslog logger using the configuration passed in on
 // the context. Supported context configuration variables are
 // syslog-address, syslog-facility, & syslog-tag.
@@ -83,6 +95,11 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		return nil, err
 	}
 
+	syslogFormatter, syslogFramer, err := parseLogFormat(ctx.Config["syslog-format"])
+	if err != nil {
+		return nil, err
+	}
+
 	logTag := path.Base(os.Args[0]) + "/" + tag
 
 	var log *syslog.Writer
@@ -100,6 +117,9 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		return nil, err
 	}
 
+	log.SetFormatter(syslogFormatter)
+	log.SetFramer(syslogFramer)
+
 	return &syslogger{
 		writer: log,
 	}, nil
@@ -165,6 +185,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case "syslog-tls-key":
 		case "syslog-tls-skip-verify":
 		case "tag":
+		case "syslog-format":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for syslog log driver", key)
 		}
@@ -175,6 +196,9 @@ func ValidateLogOpt(cfg map[string]string) error {
 	if _, err := parseFacility(cfg["syslog-facility"]); err != nil {
 		return err
 	}
+	if _, _, err := parseLogFormat(cfg["syslog-format"]); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -207,3 +231,17 @@ func parseTLSConfig(cfg map[string]string) (*tls.Config, error) {
 
 	return tlsconfig.Client(opts)
 }
+
+func parseLogFormat(logFormat string) (syslog.Formatter, syslog.Framer, error) {
+	switch logFormat {
+	case "":
+		return syslog.UnixFormatter, syslog.DefaultFramer, nil
+	case "rfc3164":
+		return syslog.RFC3164Formatter, syslog.DefaultFramer, nil
+	case "rfc5424":
+		return rfc5424formatterWithAppNameAsTag, syslog.RFC5425MessageLengthFramer, nil
+	default:
+		return nil, nil, errors.New("Invalid syslog format")
+	}
+
+}

daemon/logger/syslog/syslog_test.go

@@ -0,0 +1,45 @@
+// +build linux
+
+package syslog
+
+import (
+	syslog "github.com/RackSec/srslog"
+	"reflect"
+	"testing"
+)
+
+func functionMatches(expectedFun interface{}, actualFun interface{}) bool {
+	return reflect.ValueOf(expectedFun).Pointer() == reflect.ValueOf(actualFun).Pointer()
+}
+
+func TestParseLogFormat(t *testing.T) {
+	formatter, framer, err := parseLogFormat("rfc5424")
+	if err != nil || !functionMatches(rfc5424formatterWithAppNameAsTag, formatter) ||
+		!functionMatches(syslog.RFC5425MessageLengthFramer, framer) {
+		t.Fatal("Failed to parse rfc5424 format", err, formatter, framer)
+	}
+
+	formatter, framer, err = parseLogFormat("rfc3164")
+	if err != nil || !functionMatches(syslog.RFC3164Formatter, formatter) ||
+		!functionMatches(syslog.DefaultFramer, framer) {
+		t.Fatal("Failed to parse rfc3164 format", err, formatter, framer)
+	}
+
+	formatter, framer, err = parseLogFormat("")
+	if err != nil || !functionMatches(syslog.UnixFormatter, formatter) ||
+		!functionMatches(syslog.DefaultFramer, framer) {
+		t.Fatal("Failed to parse empty format", err, formatter, framer)
+	}
+
+	formatter, framer, err = parseLogFormat("invalid")
+	if err == nil {
+		t.Fatal("Failed to parse invalid format", err, formatter, framer)
+	}
+}
+
+func TestValidateLogOptEmpty(t *testing.T) {
+	emptyConfig := make(map[string]string)
+	if err := ValidateLogOpt(emptyConfig); err != nil {
+		t.Fatal("Failed to parse empty config", err)
+	}
+}

docs/admin/logging/overview.md

@@ -80,6 +80,7 @@ The following logging options are supported for the `syslog` logging driver:
     --log-opt syslog-tls-key=/etc/ca-certificates/custom/key.pem
     --log-opt syslog-tls-skip-verify=true
     --log-opt tag="mailer"
+    --log-opt syslog-format=[rfc5424|rfc3164] 
 
 `syslog-address` specifies the remote syslog server address where the driver connects to.
 If not specified it defaults to the local unix socket of the running system.
@@ -131,6 +132,11 @@ By default, Docker uses the first 12 characters of the container ID to tag log m
 Refer to the [log tag option documentation](log_tags.md) for customizing
 the log tag format.
 
+`syslog-format` specifies syslog message format to use when logging.
+If not specified it defaults to the local unix syslog format without hostname specification.
+Specify rfc3164 to perform logging in RFC-3164 compatible format. Specify rfc5424 to perform 
+logging in RFC-5424 compatible format
+
 
 ## journald options
 

hack/vendor.sh

@@ -25,7 +25,7 @@ clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://gith
 clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 clone git github.com/docker/go-connections v0.2.0
 clone git github.com/docker/engine-api 9bab0d5b73872e53dfadfa055dcc519e57b09439
-clone git github.com/RackSec/srslog 6eb773f331e46fbba8eecb8e794e635e75fc04de
+clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages

vendor/src/github.com/RackSec/srslog/.travis.yml

@@ -4,10 +4,15 @@ group: edge
 language: go
 go:
 - 1.5
+before_install:
+  - pip install --user codecov
 script:
 - |
   go get ./...
-  go test -v ./...
+  go test -v -coverprofile=coverage.txt -covermode=atomic
+  go vet
+after_success:
+  - codecov
 notifications:
   slack:
     secure: dtDue9gP6CRR1jYjEf6raXXFak3QKGcCFvCf5mfvv5XScdpmc3udwgqc5TdyjC0goaC9OK/4jTcCD30dYZm/u6ux3E9mo3xwMl2xRLHx76p5r9rSQtloH19BDwA2+A+bpDfFQVz05k2YXuTiGSvNMMdwzx+Dr294Sl/z43RFB4+b9/R/6LlFpRW89IwftvpLAFnBy4K/ZcspQzKM+rQfQTL5Kk+iZ/KBsuR/VziDq6MoJ8t43i4ee8vwS06vFBKDbUiZ4FIZpLgc2RAL5qso5aWRKYXL6waXfoKHZWKPe0w4+9IY1rDJxG1jEb7YGgcbLaF9xzPRRs2b2yO/c87FKpkh6PDgYHfLjpgXotCoojZrL4p1x6MI1ldJr3NhARGPxS9r4liB9n6Y5nD+ErXi1IMf55fuUHcPY27Jc0ySeLFeM6cIWJ8OhFejCgGw6a5DnnmJo0PqopsaBDHhadpLejT1+K6bL2iGkT4SLcVNuRGLs+VyuNf1+5XpkWZvy32vquO7SZOngLLBv+GIem+t3fWm0Z9s/0i1uRCQei1iUutlYjoV/LBd35H2rhob4B5phIuJin9kb0zbHf6HnaoN0CtN8r0d8G5CZiInVlG5Xcid5Byb4dddf5U2EJTDuCMVyyiM7tcnfjqw9UbVYNxtYM9SzcqIq+uVqM8pYL9xSec=

vendor/src/github.com/RackSec/srslog/dialer.go

@@ -5,19 +5,49 @@ import (
 	"net"
 )
 
-func (w Writer) getDialer() func() (serverConn, string, error) {
+// dialerFunctionWrapper is a simple object that consists of a dialer function
-	dialers := map[string]func() (serverConn, string, error){
+// and its name. This is primarily for testing, so we can make sure that the
-		"":        w.unixDialer,
+// getDialer method returns the correct dialer function. However, if you ever
-		"tcp+tls": w.tlsDialer,
+// find that you need to check which dialer function you have, this would also
+// be useful for you without having to use reflection.
+type dialerFunctionWrapper struct {
+	Name   string
+	Dialer func() (serverConn, string, error)
+}
+
+// Call the wrapped dialer function and return its return values.
+func (df dialerFunctionWrapper) Call() (serverConn, string, error) {
+	return df.Dialer()
+}
+
+// getDialer returns a "dialer" function that can be called to connect to a
+// syslog server.
+//
+// Each dialer function is responsible for dialing the remote host and returns
+// a serverConn, the hostname (or a default if the Writer has not specified a
+// hostname), and an error in case dialing fails.
+//
+// The reason for separate dialers is that different network types may need
+// to dial their connection differently, yet still provide a net.Conn interface
+// that you can use once they have dialed. Rather than an increasingly long
+// conditional, we have a map of network -> dialer function (with a sane default
+// value), and adding a new network type is as easy as writing the dialer
+// function and adding it to the map.
+func (w *Writer) getDialer() dialerFunctionWrapper {
+	dialers := map[string]dialerFunctionWrapper{
+		"":        dialerFunctionWrapper{"unixDialer", w.unixDialer},
+		"tcp+tls": dialerFunctionWrapper{"tlsDialer", w.tlsDialer},
 	}
 	dialer, ok := dialers[w.network]
 	if !ok {
-		dialer = w.basicDialer
+		dialer = dialerFunctionWrapper{"basicDialer", w.basicDialer}
 	}
 	return dialer
 }
 
-func (w Writer) unixDialer() (serverConn, string, error) {
+// unixDialer uses the unixSyslog method to open a connection to the syslog
+// daemon running on the local machine.
+func (w *Writer) unixDialer() (serverConn, string, error) {
 	sc, err := unixSyslog()
 	hostname := w.hostname
 	if hostname == "" {
@@ -26,7 +56,9 @@ func (w Writer) unixDialer() (serverConn, string, error) {
 	return sc, hostname, err
 }
 
-func (w Writer) tlsDialer() (serverConn, string, error) {
+// tlsDialer connects to TLS over TCP, and is used for the "tcp+tls" network
+// type.
+func (w *Writer) tlsDialer() (serverConn, string, error) {
 	c, err := tls.Dial("tcp", w.raddr, w.tlsConfig)
 	var sc serverConn
 	hostname := w.hostname
@@ -39,7 +71,9 @@ func (w Writer) tlsDialer() (serverConn, string, error) {
 	return sc, hostname, err
 }
 
-func (w Writer) basicDialer() (serverConn, string, error) {
+// basicDialer is the most common dialer for syslog, and supports both TCP and
+// UDP connections.
+func (w *Writer) basicDialer() (serverConn, string, error) {
 	c, err := net.Dial(w.network, w.raddr)
 	var sc serverConn
 	hostname := w.hostname

vendor/src/github.com/RackSec/srslog/formatter.go

@@ -0,0 +1,48 @@
+package srslog
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+// Formatter is a type of function that takes the consituent parts of a
+// syslog message and returns a formatted string. A different Formatter is
+// defined for each different syslog protocol we support.
+type Formatter func(p Priority, hostname, tag, content string) string
+
+// DefaultFormatter is the original format supported by the Go syslog package,
+// and is a non-compliant amalgamation of 3164 and 5424 that is intended to
+// maximize compatibility.
+func DefaultFormatter(p Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.RFC3339)
+	msg := fmt.Sprintf("<%d> %s %s %s[%d]: %s",
+		p, timestamp, hostname, tag, os.Getpid(), content)
+	return msg
+}
+
+// UnixFormatter omits the hostname, because it is only used locally.
+func UnixFormatter(p Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.Stamp)
+	msg := fmt.Sprintf("<%d>%s %s[%d]: %s",
+		p, timestamp, tag, os.Getpid(), content)
+	return msg
+}
+
+// RFC3164Formatter provides an RFC 3164 compliant message.
+func RFC3164Formatter(p Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.Stamp)
+	msg := fmt.Sprintf("<%d> %s %s %s[%d]: %s",
+		p, timestamp, hostname, tag, os.Getpid(), content)
+	return msg
+}
+
+// RFC5424Formatter provides an RFC 5424 compliant message.
+func RFC5424Formatter(p Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.RFC3339)
+	pid := os.Getpid()
+	appName := os.Args[0]
+	msg := fmt.Sprintf("<%d>%d %s %s %s %d %s %s",
+		p, 1, timestamp, hostname, appName, pid, tag, content)
+	return msg
+}

vendor/src/github.com/RackSec/srslog/framer.go

@@ -0,0 +1,24 @@
+package srslog
+
+import (
+	"fmt"
+)
+
+// Framer is a type of function that takes an input string (typically an
+// already-formatted syslog message) and applies "message framing" to it. We
+// have different framers because different versions of the syslog protocol
+// and its transport requirements define different framing behavior.
+type Framer func(in string) string
+
+// DefaultFramer does nothing, since there is no framing to apply. This is
+// the original behavior of the Go syslog package, and is also typically used
+// for UDP syslog.
+func DefaultFramer(in string) string {
+	return in
+}
+
+// RFC5425MessageLengthFramer prepends the message length to the front of the
+// provided message, as defined in RFC 5425.
+func RFC5425MessageLengthFramer(in string) string {
+	return fmt.Sprintf("%d %s", len(in), in)
+}

vendor/src/github.com/RackSec/srslog/net_conn.go

@@ -1,24 +1,30 @@
 package srslog
 
 import (
-	"fmt"
 	"net"
-	"os"
-	"time"
 )
 
+// netConn has an internal net.Conn and adheres to the serverConn interface,
+// allowing us to send syslog messages over the network.
 type netConn struct {
 	conn net.Conn
 }
 
-func (n *netConn) writeString(p Priority, hostname, tag, msg string) error {
+// writeString formats syslog messages using time.RFC3339 and includes the
-	timestamp := time.Now().Format(time.RFC3339)
+// hostname, and sends the message to the connection.
-	_, err := fmt.Fprintf(n.conn, "<%d>%s %s %s[%d]: %s",
+func (n *netConn) writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, msg string) error {
-		p, timestamp, hostname,
+	if framer == nil {
-		tag, os.Getpid(), msg)
+		framer = DefaultFramer
+	}
+	if formatter == nil {
+		formatter = DefaultFormatter
+	}
+	formattedMessage := framer(formatter(p, hostname, tag, msg))
+	_, err := n.conn.Write([]byte(formattedMessage))
 	return err
 }
 
+// close the network connection
 func (n *netConn) close() error {
 	return n.conn.Close()
 }

vendor/src/github.com/RackSec/srslog/srslog.go

@@ -8,14 +8,10 @@ import (
 	"os"
 )
 
-// This interface and the separate syslog_unix.go file exist for
+// This interface allows us to work with both local and network connections,
-// Solaris support as implemented by gccgo.  On Solaris you can not
+// and enables Solaris support (see syslog_unix.go).
-// simply open a TCP connection to the syslog daemon.  The gccgo
-// sources have a syslog_solaris.go file that implements unixSyslog to
-// return a type that satisfies this interface and simply calls the C
-// library syslog function.
 type serverConn interface {
-	writeString(p Priority, hostname, tag, s string) error
+	writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, s string) error
 	close() error
 }
 
@@ -39,11 +35,19 @@ func Dial(network, raddr string, priority Priority, tag string) (*Writer, error)
 // address raddr on the specified network. It uses certPath to load TLS certificates and configure
 // the secure connection.
 func DialWithTLSCertPath(network, raddr string, priority Priority, tag, certPath string) (*Writer, error) {
-	pool := x509.NewCertPool()
 	serverCert, err := ioutil.ReadFile(certPath)
 	if err != nil {
 		return nil, err
 	}
+
+	return DialWithTLSCert(network, raddr, priority, tag, serverCert)
+}
+
+// DialWIthTLSCert establishes a secure connection to a log daemon by connecting to
+// address raddr on the specified network. It uses serverCert to load a TLS certificate
+// and configure the secure connection.
+func DialWithTLSCert(network, raddr string, priority Priority, tag string, serverCert []byte) (*Writer, error) {
+	pool := x509.NewCertPool()
 	pool.AppendCertsFromPEM(serverCert)
 	config := tls.Config{
 		RootCAs: pool,

vendor/src/github.com/RackSec/srslog/srslog_unix.go

@@ -2,15 +2,17 @@ package srslog
 
 import (
 	"errors"
-	"fmt"
+	"io"
 	"net"
-	"os"
-	"time"
 )
 
 // unixSyslog opens a connection to the syslog daemon running on the
-// local machine using a Unix domain socket.
+// local machine using a Unix domain socket. This function exists because of
-
+// Solaris support as implemented by gccgo.  On Solaris you can not
+// simply open a TCP connection to the syslog daemon.  The gccgo
+// sources have a syslog_solaris.go file that implements unixSyslog to
+// return a type that satisfies the serverConn interface and simply calls the C
+// library syslog function.
 func unixSyslog() (conn serverConn, err error) {
 	logTypes := []string{"unixgram", "unix"}
 	logPaths := []string{"/dev/log", "/var/run/syslog", "/var/run/log"}
@@ -27,21 +29,26 @@ func unixSyslog() (conn serverConn, err error) {
 	return nil, errors.New("Unix syslog delivery error")
 }
 
+// localConn adheres to the serverConn interface, allowing us to send syslog
+// messages to the local syslog daemon over a Unix domain socket.
 type localConn struct {
-	conn net.Conn
+	conn io.WriteCloser
 }
 
-func (n *localConn) writeString(p Priority, hostname, tag, msg string) error {
+// writeString formats syslog messages using time.Stamp instead of time.RFC3339,
-	// Compared to the network form at srslog.netConn, the changes are:
+// and omits the hostname (because it is expected to be used locally).
-	//	1. Use time.Stamp instead of time.RFC3339.
+func (n *localConn) writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, msg string) error {
-	//	2. Drop the hostname field from the Fprintf.
+	if framer == nil {
-	timestamp := time.Now().Format(time.Stamp)
+		framer = DefaultFramer
-	_, err := fmt.Fprintf(n.conn, "<%d>%s %s[%d]: %s",
+	}
-		p, timestamp,
+	if formatter == nil {
-		tag, os.Getpid(), msg)
+		formatter = UnixFormatter
+	}
+	_, err := n.conn.Write([]byte(framer(formatter(p, hostname, tag, msg))))
 	return err
 }
 
+// close the (local) network connection
 func (n *localConn) close() error {
 	return n.conn.Close()
 }

vendor/src/github.com/RackSec/srslog/writer.go

@@ -16,6 +16,8 @@ type Writer struct {
 	network   string
 	raddr     string
 	tlsConfig *tls.Config
+	framer    Framer
+	formatter Formatter
 
 	conn serverConn
 }
@@ -32,7 +34,7 @@ func (w *Writer) connect() (err error) {
 	var conn serverConn
 	var hostname string
 	dialer := w.getDialer()
-	conn, hostname, err = dialer()
+	conn, hostname, err = dialer.Call()
 	if err == nil {
 		w.conn = conn
 		w.hostname = hostname
@@ -41,6 +43,16 @@ func (w *Writer) connect() (err error) {
 	return
 }
 
+// SetFormatter changes the formatter function for subsequent messages.
+func (w *Writer) SetFormatter(f Formatter) {
+	w.formatter = f
+}
+
+// SetFramer changes the framer function for subsequent messages.
+func (w *Writer) SetFramer(f Framer) {
+	w.framer = f
+}
+
 // Write sends a log message to the syslog daemon using the default priority
 // passed into `srslog.New` or the `srslog.Dial*` functions.
 func (w *Writer) Write(b []byte) (int, error) {
@@ -133,15 +145,15 @@ func (w *Writer) writeAndRetry(p Priority, s string) (int, error) {
 	return w.write(pr, s)
 }
 
-// write generates and writes a syslog formatted string. The
+// write generates and writes a syslog formatted string. It formats the
-// format is as follows: <PRI>TIMESTAMP HOSTNAME TAG[PID]: MSG
+// message based on the current Formatter and Framer.
 func (w *Writer) write(p Priority, msg string) (int, error) {
 	// ensure it ends in a \n
 	if !strings.HasSuffix(msg, "\n") {
 		msg += "\n"
 	}
 
-	err := w.conn.writeString(p, w.hostname, w.tag, msg)
+	err := w.conn.writeString(w.framer, w.formatter, p, w.hostname, w.tag, msg)
 	if err != nil {
 		return 0, err
 	}