Merge pull request #42291 from angelcar/awslogs-dont-log-messge-discarded-errors
Limit the rate at which logger errors are logged into daemon logs
This commit is contained in:
commit
a8a769f04f
|
@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if logErr := c.dst.Log(msg); logErr != nil {
|
if logErr := c.dst.Log(msg); logErr != nil {
|
||||||
logWritesFailedCount.Inc(1)
|
logDriverError(c.dst.Name(), string(msg.Line), logErr)
|
||||||
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p += q + 1
|
p += q + 1
|
||||||
|
@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
|
||||||
hasMorePartial = true
|
hasMorePartial = true
|
||||||
|
|
||||||
if logErr := c.dst.Log(msg); logErr != nil {
|
if logErr := c.dst.Log(msg); logErr != nil {
|
||||||
logWritesFailedCount.Inc(1)
|
logDriverError(c.dst.Name(), string(msg.Line), logErr)
|
||||||
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
|
|
||||||
}
|
}
|
||||||
p = 0
|
p = 0
|
||||||
n = 0
|
n = 0
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
package logger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Rates based on journald defaults of 10,000 messages in 30s.
|
||||||
|
// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
|
||||||
|
var logErrorLimiter = rate.NewLimiter(333, 333)
|
||||||
|
|
||||||
|
// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
|
||||||
|
// metric.
|
||||||
|
// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
|
||||||
|
// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
|
||||||
|
func logDriverError(loggerName, msgLine string, logErr error) {
|
||||||
|
logWritesFailedCount.Inc(1)
|
||||||
|
if logErrorLimiter.Allow() {
|
||||||
|
logrus.WithError(logErr).
|
||||||
|
WithField("driver", loggerName).
|
||||||
|
WithField("message", msgLine).
|
||||||
|
Errorf("Error writing log message")
|
||||||
|
}
|
||||||
|
}
|
|
@ -4,8 +4,6 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -107,10 +105,7 @@ func (r *RingLogger) Close() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := r.l.Log(msg); err != nil {
|
if err := r.l.Log(msg); err != nil {
|
||||||
logrus.WithField("driver", r.l.Name()).
|
logDriverError(r.l.Name(), string(msg.Line), err)
|
||||||
WithField("container", r.logInfo.ContainerID).
|
|
||||||
WithError(err).
|
|
||||||
Errorf("Error writing log message")
|
|
||||||
logErr = true
|
logErr = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -132,10 +127,7 @@ func (r *RingLogger) run() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := r.l.Log(msg); err != nil {
|
if err := r.l.Log(msg); err != nil {
|
||||||
logrus.WithField("driver", r.l.Name()).
|
logDriverError(r.l.Name(), string(msg.Line), err)
|
||||||
WithField("container", r.logInfo.ContainerID).
|
|
||||||
WithError(err).
|
|
||||||
Errorf("Error writing log message")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue