mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
Fix a lb rule race in loadbalancer
When a goroutine which is adding the service and another which is adding just a destination interleave the destination which is dependent on the service may not get added and will result in service working at reduced scale. The fix is to synchronize this with the service mutex. Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>
This commit is contained in:
parent
5c011c3673
commit
4d1a5ce968
1 changed files with 12 additions and 11 deletions
|
@ -56,7 +56,13 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
|
|||
}
|
||||
c.Unlock()
|
||||
|
||||
// Add endpoint IP to special "tasks.svc_name" so that the
|
||||
// applications have access to DNS RR.
|
||||
n.(*network).addSvcRecords("tasks."+name, ip, nil, false)
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
lb, ok := s.loadBalancers[nid]
|
||||
if !ok {
|
||||
// Create a new load balancer if we are seeing this
|
||||
|
@ -89,11 +95,6 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
|
|||
}
|
||||
|
||||
lb.backEnds[eid] = ip
|
||||
s.Unlock()
|
||||
|
||||
// Add endpoint IP to special "tasks.svc_name" so that the
|
||||
// applications have access to DNS RR.
|
||||
n.(*network).addSvcRecords("tasks."+name, ip, nil, false)
|
||||
|
||||
// Add loadbalancer service and backend in all sandboxes in
|
||||
// the network only if vip is valid.
|
||||
|
@ -120,17 +121,18 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
|
|||
}
|
||||
c.Unlock()
|
||||
|
||||
// Delete the special "tasks.svc_name" backend record.
|
||||
n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false)
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
lb, ok := s.loadBalancers[nid]
|
||||
if !ok {
|
||||
s.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the special "tasks.svc_name" backend record.
|
||||
n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false)
|
||||
delete(lb.backEnds, eid)
|
||||
|
||||
if len(lb.backEnds) == 0 {
|
||||
// All the backends for this service have been
|
||||
// removed. Time to remove the load balancer and also
|
||||
|
@ -153,7 +155,6 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
|
|||
// remove the service itself.
|
||||
delete(c.serviceBindings, sid)
|
||||
}
|
||||
s.Unlock()
|
||||
|
||||
// Remove loadbalancer service(if needed) and backend in all
|
||||
// sandboxes in the network only if the vip is valid.
|
||||
|
@ -312,7 +313,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
|
|||
// destination.
|
||||
s.SchedName = ""
|
||||
if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST {
|
||||
logrus.Errorf("Failed to create real server %s for vip %s fwmark %d: %v", ip, vip, fwMark, err)
|
||||
logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sb %s: %v", ip, vip, fwMark, sb.containerID, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue