kotovalexarian-likes-github
/
moby--moby
mirror of https://github.com/moby/moby.git
1
0
Fork 0
Code Releases Activity

Merge pull request #42291 from angelcar/awslogs-dont-log-messge-discarded-errors 

Limit the rate at which logger errors are logged into daemon logs
This commit is contained in:
Brian Goff 2021-05-27 19:33:44 -07:00 committed by GitHub
parent 454abc560d fb5a9ec741
commit a8a769f04f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 28 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
daemon/logger/copier.go
View File

@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
}
if logErr := c.dst.Log(msg); logErr != nil {
logWritesFailedCount.Inc(1)
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
logDriverError(c.dst.Name(), string(msg.Line), logErr)
}
}
p += q + 1
@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
hasMorePartial = true
if logErr := c.dst.Log(msg); logErr != nil {
logWritesFailedCount.Inc(1)
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
logDriverError(c.dst.Name(), string(msg.Line), logErr)
}
p = 0
n = 0

24
daemon/logger/logger_error.go Normal file
View File

@ -0,0 +1,24 @@
package logger
import (
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
// Rates based on journald defaults of 10,000 messages in 30s.
// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
var logErrorLimiter = rate.NewLimiter(333, 333)
// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
// metric.
// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
func logDriverError(loggerName, msgLine string, logErr error) {
logWritesFailedCount.Inc(1)
if logErrorLimiter.Allow() {
logrus.WithError(logErr).
WithField("driver", loggerName).
WithField("message", msgLine).
Errorf("Error writing log message")
}
}

12
daemon/logger/ring.go
View File

@ -4,8 +4,6 @@ import (
"errors"
"sync"
"sync/atomic"
"github.com/sirupsen/logrus"
)
const (
@ -107,10 +105,7 @@ func (r *RingLogger) Close() error {
}
if err := r.l.Log(msg); err != nil {
logrus.WithField("driver", r.l.Name()).
WithField("container", r.logInfo.ContainerID).
WithError(err).
Errorf("Error writing log message")
logDriverError(r.l.Name(), string(msg.Line), err)
logErr = true
}
}
@ -132,10 +127,7 @@ func (r *RingLogger) run() {
return
}
if err := r.l.Log(msg); err != nil {
logrus.WithField("driver", r.l.Name()).
WithField("container", r.logInfo.ContainerID).
WithError(err).
Errorf("Error writing log message")
logDriverError(r.l.Name(), string(msg.Line), err)
}
}
}