kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

Merge pull request #37758 from dmcgowan/fix-libcontainerd-supervisor

Browse source 
Fix supervisor healthcheck throttling
This commit is contained in:
Tibor Vass 2018-09-04 23:01:28 -07:00 committed by GitHub
commit 53e55db9d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
libcontainerd/supervisor/remote_daemon.go
View file

@ -245,20 +245,26 @@ func (r *remote) monitorDaemon(ctx context.Context) {
}() }()
for { for {
select { if delay != nil {
case <-ctx.Done(): select {
r.logger.Info("stopping healthcheck following graceful shutdown") case <-ctx.Done():
if client != nil { r.logger.Info("stopping healthcheck following graceful shutdown")
client.Close() if client != nil {
client.Close()
}
return
case <-delay:
} }
return
case <-delay:
default:
} }
if r.daemonPid == -1 { if r.daemonPid == -1 {
if r.daemonWaitCh != nil { if r.daemonWaitCh != nil {
<-r.daemonWaitCh select {
case <-ctx.Done():
r.logger.Info("stopping containerd startup following graceful shutdown")
return
case <-r.daemonWaitCh:
}
} }
os.RemoveAll(r.GRPC.Address) os.RemoveAll(r.GRPC.Address)
@ -276,26 +282,28 @@ func (r *remote) monitorDaemon(ctx context.Context) {
} }
} }
tctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) if client != nil {
_, err := client.IsServing(tctx) tctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
cancel() _, err := client.IsServing(tctx)
if err == nil { cancel()
if !started { if err == nil {
close(r.daemonStartCh) if !started {
started = true close(r.daemonStartCh)
started = true
}
transientFailureCount = 0
delay = time.After(500 * time.Millisecond)
continue
} }
transientFailureCount = 0 r.logger.WithError(err).WithField("binary", binaryName).Debug("daemon is not responding")
delay = time.After(500 * time.Millisecond)
continue
}
r.logger.WithError(err).WithField("binary", binaryName).Debug("daemon is not responding") transientFailureCount++
if transientFailureCount < maxConnectionRetryCount || system.IsProcessAlive(r.daemonPid) {
transientFailureCount++ delay = time.After(time.Duration(transientFailureCount) * 200 * time.Millisecond)
if transientFailureCount < maxConnectionRetryCount || system.IsProcessAlive(r.daemonPid) { continue
delay = time.After(time.Duration(transientFailureCount) * 200 * time.Millisecond) }
continue
} }
if system.IsProcessAlive(r.daemonPid) { if system.IsProcessAlive(r.daemonPid) {
@ -304,6 +312,7 @@ func (r *remote) monitorDaemon(ctx context.Context) {
} }
client.Close() client.Close()
client = nil
r.daemonPid = -1 r.daemonPid = -1
delay = nil delay = nil
transientFailureCount = 0 transientFailureCount = 0