From f585f3304264eca7e9950449c248ecb0f4686374 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Mon, 22 May 2017 17:11:56 -0700 Subject: [PATCH] Node failure timeout fix The time to keep a node failed into the failed node list was originally supposed to be 24h. If a node leaves explicitly it will be removed from the list of nodes and put into the leftNodes list. This way the NotifyLeave event won't insert it into the retry list. NOTE: if the event is lost instead the behavior will be the same as a failed node. If a node fails, the NotifyLeave will insert it into the failedNodes list with a reapTime of 24h. This means that the node will be checked for 24h before being completely forgot. The current check time is every 1 second and is done by the reconnectNode function. The failed node list is updated every 2h instead. Signed-off-by: Flavio Crisciani --- libnetwork/networkdb/event_delegate.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libnetwork/networkdb/event_delegate.go b/libnetwork/networkdb/event_delegate.go index 6ae0a32ad1..0b137bc818 100644 --- a/libnetwork/networkdb/event_delegate.go +++ b/libnetwork/networkdb/event_delegate.go @@ -47,7 +47,9 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) { if n, ok := e.nDB.nodes[mn.Name]; ok { delete(e.nDB.nodes, mn.Name) - n.reapTime = reapInterval + // In case of node failure, keep retrying to reconnect every retryInterval (1sec) for nodeReapInterval (24h) + // Explicit leave will have already removed the node from the list of nodes (nDB.nodes) and put it into the leftNodes map + n.reapTime = nodeReapInterval e.nDB.failedNodes[mn.Name] = n } e.nDB.Unlock()