2015-07-21 18:26:52 -04:00
|
|
|
// Package logger defines interfaces that logger drivers implement to
|
|
|
|
// log messages.
|
|
|
|
//
|
|
|
|
// The other half of a logger driver is the implementation of the
|
|
|
|
// factory, which holds the contextual instance information that
|
|
|
|
// allows multiple loggers of the same type to perform different
|
|
|
|
// actions, such as logging to different locations.
|
2015-02-03 18:41:26 -05:00
|
|
|
package logger
|
|
|
|
|
2015-04-09 00:23:30 -04:00
|
|
|
import (
|
|
|
|
"errors"
|
2016-04-08 12:15:08 -04:00
|
|
|
"sort"
|
|
|
|
"strings"
|
2016-06-27 17:14:17 -04:00
|
|
|
"sync"
|
2015-04-09 00:23:30 -04:00
|
|
|
"time"
|
2015-07-03 09:50:06 -04:00
|
|
|
|
2015-12-15 14:49:41 -05:00
|
|
|
"github.com/docker/docker/pkg/jsonlog"
|
2015-04-09 00:23:30 -04:00
|
|
|
)
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// ErrReadLogsNotSupported is returned when the logger does not support reading logs.
|
2015-07-03 09:50:06 -04:00
|
|
|
var ErrReadLogsNotSupported = errors.New("configured logging reader does not support reading")
|
|
|
|
|
|
|
|
const (
|
2015-07-21 18:26:52 -04:00
|
|
|
// TimeFormat is the time format used for timestamps sent to log readers.
|
2015-12-15 14:49:41 -05:00
|
|
|
TimeFormat = jsonlog.RFC3339NanoFixed
|
2015-07-03 09:50:06 -04:00
|
|
|
logWatcherBufferSize = 4096
|
|
|
|
)
|
2015-02-03 18:41:26 -05:00
|
|
|
|
Improve logging of long log lines
This change updates how we handle long lines of output from the
container. The previous logic used a bufio reader to read entire lines
of output from the container through an intermediate BytesPipe, and that
allowed the container to cause dockerd to consume an unconstrained
amount of memory as it attempted to collect a whole line of output, by
outputting data without newlines.
To avoid that, we replace the bufio reader with our own buffering scheme
that handles log lines up to 16k in length, breaking up anything longer
than that into multiple chunks. If we can dispense with noting this
detail properly at the end of output, we can switch from using
ReadBytes() to using ReadLine() instead. We add a field ("Partial") to
the log message structure to flag when we pass data to the log driver
that did not end with a newline.
The Line member of Message structures that we pass to log drivers is now
a slice into data which can be overwritten between calls to the log
driver's Log() method, so drivers which batch up Messages before
processing them need to take additional care: we add a function
(logger.CopyMessage()) that can be used to create a deep copy of a
Message structure, and modify the awslogs driver to use it.
We update the jsonfile log driver to append a "\n" to the data that it
logs to disk only when the Partial flag is false (it previously did so
unconditionally), to make its "logs" output correctly reproduce the data
as we received it.
Likewise, we modify the journald log driver to add a data field with
value CONTAINER_PARTIAL_MESSAGE=true to entries when the Partial flag is
true, and update its "logs" reader to refrain from appending a "\n" to
the data that it retrieves if it does not see this field/value pair (it
also previously did this unconditionally).
Signed-off-by: Nalin Dahyabhai <nalin@redhat.com> (github: nalind)
2016-05-24 14:12:47 -04:00
|
|
|
// Message is datastructure that represents piece of output produced by some
|
|
|
|
// container. The Line member is a slice of an array whose contents can be
|
|
|
|
// changed after a log driver's Log() method returns.
|
2015-02-03 18:41:26 -05:00
|
|
|
type Message struct {
|
2016-05-31 15:46:55 -04:00
|
|
|
Line []byte
|
|
|
|
Source string
|
|
|
|
Timestamp time.Time
|
|
|
|
Attrs LogAttributes
|
Improve logging of long log lines
This change updates how we handle long lines of output from the
container. The previous logic used a bufio reader to read entire lines
of output from the container through an intermediate BytesPipe, and that
allowed the container to cause dockerd to consume an unconstrained
amount of memory as it attempted to collect a whole line of output, by
outputting data without newlines.
To avoid that, we replace the bufio reader with our own buffering scheme
that handles log lines up to 16k in length, breaking up anything longer
than that into multiple chunks. If we can dispense with noting this
detail properly at the end of output, we can switch from using
ReadBytes() to using ReadLine() instead. We add a field ("Partial") to
the log message structure to flag when we pass data to the log driver
that did not end with a newline.
The Line member of Message structures that we pass to log drivers is now
a slice into data which can be overwritten between calls to the log
driver's Log() method, so drivers which batch up Messages before
processing them need to take additional care: we add a function
(logger.CopyMessage()) that can be used to create a deep copy of a
Message structure, and modify the awslogs driver to use it.
We update the jsonfile log driver to append a "\n" to the data that it
logs to disk only when the Partial flag is false (it previously did so
unconditionally), to make its "logs" output correctly reproduce the data
as we received it.
Likewise, we modify the journald log driver to add a data field with
value CONTAINER_PARTIAL_MESSAGE=true to entries when the Partial flag is
true, and update its "logs" reader to refrain from appending a "\n" to
the data that it retrieves if it does not see this field/value pair (it
also previously did this unconditionally).
Signed-off-by: Nalin Dahyabhai <nalin@redhat.com> (github: nalind)
2016-05-24 14:12:47 -04:00
|
|
|
Partial bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// CopyMessage creates a copy of the passed-in Message which will remain
|
|
|
|
// unchanged if the original is changed. Log drivers which buffer Messages
|
|
|
|
// rather than dispatching them during their Log() method should use this
|
|
|
|
// function to obtain a Message whose Line member's contents won't change.
|
|
|
|
func CopyMessage(msg *Message) *Message {
|
|
|
|
m := new(Message)
|
|
|
|
m.Line = make([]byte, len(msg.Line))
|
|
|
|
copy(m.Line, msg.Line)
|
|
|
|
m.Source = msg.Source
|
|
|
|
m.Timestamp = msg.Timestamp
|
|
|
|
m.Partial = msg.Partial
|
|
|
|
m.Attrs = make(LogAttributes)
|
2016-11-24 04:02:48 -05:00
|
|
|
for k, v := range msg.Attrs {
|
Improve logging of long log lines
This change updates how we handle long lines of output from the
container. The previous logic used a bufio reader to read entire lines
of output from the container through an intermediate BytesPipe, and that
allowed the container to cause dockerd to consume an unconstrained
amount of memory as it attempted to collect a whole line of output, by
outputting data without newlines.
To avoid that, we replace the bufio reader with our own buffering scheme
that handles log lines up to 16k in length, breaking up anything longer
than that into multiple chunks. If we can dispense with noting this
detail properly at the end of output, we can switch from using
ReadBytes() to using ReadLine() instead. We add a field ("Partial") to
the log message structure to flag when we pass data to the log driver
that did not end with a newline.
The Line member of Message structures that we pass to log drivers is now
a slice into data which can be overwritten between calls to the log
driver's Log() method, so drivers which batch up Messages before
processing them need to take additional care: we add a function
(logger.CopyMessage()) that can be used to create a deep copy of a
Message structure, and modify the awslogs driver to use it.
We update the jsonfile log driver to append a "\n" to the data that it
logs to disk only when the Partial flag is false (it previously did so
unconditionally), to make its "logs" output correctly reproduce the data
as we received it.
Likewise, we modify the journald log driver to add a data field with
value CONTAINER_PARTIAL_MESSAGE=true to entries when the Partial flag is
true, and update its "logs" reader to refrain from appending a "\n" to
the data that it retrieves if it does not see this field/value pair (it
also previously did this unconditionally).
Signed-off-by: Nalin Dahyabhai <nalin@redhat.com> (github: nalind)
2016-05-24 14:12:47 -04:00
|
|
|
m.Attrs[k] = v
|
|
|
|
}
|
|
|
|
return m
|
2016-04-08 12:15:08 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
// LogAttributes is used to hold the extra attributes available in the log message
|
|
|
|
// Primarily used for converting the map type to string and sorting.
|
|
|
|
type LogAttributes map[string]string
|
|
|
|
type byKey []string
|
|
|
|
|
|
|
|
func (s byKey) Len() int { return len(s) }
|
|
|
|
func (s byKey) Less(i, j int) bool {
|
|
|
|
keyI := strings.Split(s[i], "=")
|
|
|
|
keyJ := strings.Split(s[j], "=")
|
|
|
|
return keyI[0] < keyJ[0]
|
|
|
|
}
|
|
|
|
func (s byKey) Swap(i, j int) {
|
|
|
|
s[i], s[j] = s[j], s[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a LogAttributes) String() string {
|
|
|
|
var ss byKey
|
|
|
|
for k, v := range a {
|
|
|
|
ss = append(ss, k+"="+v)
|
|
|
|
}
|
|
|
|
sort.Sort(ss)
|
|
|
|
return strings.Join(ss, ",")
|
2015-02-03 18:41:26 -05:00
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// Logger is the interface for docker logging drivers.
|
2015-02-03 18:41:26 -05:00
|
|
|
type Logger interface {
|
|
|
|
Log(*Message) error
|
|
|
|
Name() string
|
|
|
|
Close() error
|
2015-06-30 20:40:13 -04:00
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// ReadConfig is the configuration passed into ReadLogs.
|
2015-07-03 09:50:06 -04:00
|
|
|
type ReadConfig struct {
|
|
|
|
Since time.Time
|
|
|
|
Tail int
|
|
|
|
Follow bool
|
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// LogReader is the interface for reading log messages for loggers that support reading.
|
2015-07-03 09:50:06 -04:00
|
|
|
type LogReader interface {
|
|
|
|
// Read logs from underlying logging backend
|
|
|
|
ReadLogs(ReadConfig) *LogWatcher
|
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// LogWatcher is used when consuming logs read from the LogReader interface.
|
2015-07-03 09:50:06 -04:00
|
|
|
type LogWatcher struct {
|
2015-07-21 18:26:52 -04:00
|
|
|
// For sending log messages to a reader.
|
2015-07-03 09:50:06 -04:00
|
|
|
Msg chan *Message
|
2015-07-21 18:26:52 -04:00
|
|
|
// For sending error messages that occur while while reading logs.
|
2015-07-03 09:50:06 -04:00
|
|
|
Err chan error
|
2016-06-27 17:14:17 -04:00
|
|
|
closeOnce sync.Once
|
2015-07-03 09:50:06 -04:00
|
|
|
closeNotifier chan struct{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewLogWatcher returns a new LogWatcher.
|
|
|
|
func NewLogWatcher() *LogWatcher {
|
|
|
|
return &LogWatcher{
|
|
|
|
Msg: make(chan *Message, logWatcherBufferSize),
|
|
|
|
Err: make(chan error, 1),
|
|
|
|
closeNotifier: make(chan struct{}),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// Close notifies the underlying log reader to stop.
|
2015-07-03 09:50:06 -04:00
|
|
|
func (w *LogWatcher) Close() {
|
2015-07-31 16:49:07 -04:00
|
|
|
// only close if not already closed
|
2016-06-27 17:14:17 -04:00
|
|
|
w.closeOnce.Do(func() {
|
2015-07-31 16:49:07 -04:00
|
|
|
close(w.closeNotifier)
|
2016-06-27 17:14:17 -04:00
|
|
|
})
|
2015-07-03 09:50:06 -04:00
|
|
|
}
|
|
|
|
|
2015-07-21 18:26:52 -04:00
|
|
|
// WatchClose returns a channel receiver that receives notification
|
|
|
|
// when the watcher has been closed. This should only be called from
|
|
|
|
// one goroutine.
|
2015-07-03 09:50:06 -04:00
|
|
|
func (w *LogWatcher) WatchClose() <-chan struct{} {
|
|
|
|
return w.closeNotifier
|
2015-02-03 18:41:26 -05:00
|
|
|
}
|