From 4d1a5ce9681bb56aed2d7be1d23d5af642e429c0 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 7 Jun 2016 17:04:51 -0700 Subject: [PATCH] Fix a lb rule race in loadbalancer When a goroutine which is adding the service and another which is adding just a destination interleave the destination which is dependent on the service may not get added and will result in service working at reduced scale. The fix is to synchronize this with the service mutex. Signed-off-by: Jana Radhakrishnan --- libnetwork/service_linux.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/libnetwork/service_linux.go b/libnetwork/service_linux.go index ac3f775fdf..7c49666ba5 100644 --- a/libnetwork/service_linux.go +++ b/libnetwork/service_linux.go @@ -56,7 +56,13 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i } c.Unlock() + // Add endpoint IP to special "tasks.svc_name" so that the + // applications have access to DNS RR. + n.(*network).addSvcRecords("tasks."+name, ip, nil, false) + s.Lock() + defer s.Unlock() + lb, ok := s.loadBalancers[nid] if !ok { // Create a new load balancer if we are seeing this @@ -89,11 +95,6 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i } lb.backEnds[eid] = ip - s.Unlock() - - // Add endpoint IP to special "tasks.svc_name" so that the - // applications have access to DNS RR. - n.(*network).addSvcRecords("tasks."+name, ip, nil, false) // Add loadbalancer service and backend in all sandboxes in // the network only if vip is valid. @@ -120,17 +121,18 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in } c.Unlock() + // Delete the special "tasks.svc_name" backend record. + n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false) + s.Lock() + defer s.Unlock() + lb, ok := s.loadBalancers[nid] if !ok { - s.Unlock() return nil } - // Delete the special "tasks.svc_name" backend record. - n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false) delete(lb.backEnds, eid) - if len(lb.backEnds) == 0 { // All the backends for this service have been // removed. Time to remove the load balancer and also @@ -153,7 +155,6 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in // remove the service itself. delete(c.serviceBindings, sid) } - s.Unlock() // Remove loadbalancer service(if needed) and backend in all // sandboxes in the network only if the vip is valid. @@ -312,7 +313,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P // destination. s.SchedName = "" if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST { - logrus.Errorf("Failed to create real server %s for vip %s fwmark %d: %v", ip, vip, fwMark, err) + logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sb %s: %v", ip, vip, fwMark, sb.containerID, err) } }