From 1acaf2aabe000c6101501af321969df7c0ca5413 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 12 Jul 2019 15:29:57 -0700 Subject: [PATCH] Sleep before restarting event processing This prevents restarting event processing in a tight loop. You can see this with the following steps: ```terminal $ containerd & $ dockerd --containerd=/run/containerd/containerd.sock & $ pkill -9 containerd ``` At this point you will be spammed with logs such as: ``` ERRO[2019-07-12T22:29:37.318761400Z] failed to get event error="rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = \"transport: Error while dialing dial unix /run/containerd/containerd.sock: connect: connection refused\"" module=libcontainerd namespace=plugins.moby ``` Without this change you can quickly end up with gigabytes of log data. Signed-off-by: Brian Goff --- libcontainerd/remote/client.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libcontainerd/remote/client.go b/libcontainerd/remote/client.go index 9c5b98349f..10a8a25c51 100644 --- a/libcontainerd/remote/client.go +++ b/libcontainerd/remote/client.go @@ -703,10 +703,16 @@ func (c *client) processEventStream(ctx context.Context, ns string) { errStatus, ok := status.FromError(err) if !ok || errStatus.Code() != codes.Canceled { c.logger.WithError(err).Error("failed to get event") - go c.processEventStream(ctx, ns) - } else { - c.logger.WithError(ctx.Err()).Info("stopping event stream following graceful shutdown") + + // rate limit + select { + case <-time.After(time.Second): + go c.processEventStream(ctx, ns) + return + case <-ctx.Done(): + } } + c.logger.WithError(ctx.Err()).Info("stopping event stream following graceful shutdown") } return case ev = <-eventStream: