Merge pull request #30690 from aboch/vnd

Vendoring libnetwork @802895f

integration-cli/docker_cli_network_unix_test.go

@@ -1641,16 +1641,17 @@ func (s *DockerSuite) TestDockerNetworkInternalMode(c *check.C) {
 	c.Assert(err, check.IsNil)
 }
 
-// Test for special characters in network names. only [a-zA-Z0-9][a-zA-Z0-9_.-] are
+// Test for #21401
-// valid characters
 func (s *DockerNetworkSuite) TestDockerNetworkCreateDeleteSpecialCharacters(c *check.C) {
-	_, _, err := dockerCmdWithError("network", "create", "test@#$")
+	dockerCmd(c, "network", "create", "test@#$")
-	c.Assert(err, check.NotNil)
+	assertNwIsAvailable(c, "test@#$")
+	dockerCmd(c, "network", "rm", "test@#$")
+	assertNwNotAvailable(c, "test@#$")
 
-	dockerCmd(c, "network", "create", "test-1_0.net")
+	dockerCmd(c, "network", "create", "kiwl$%^")
-	assertNwIsAvailable(c, "test-1_0.net")
+	assertNwIsAvailable(c, "kiwl$%^")
-	dockerCmd(c, "network", "rm", "test-1_0.net")
+	dockerCmd(c, "network", "rm", "kiwl$%^")
-	assertNwNotAvailable(c, "test-1_0.net")
+	assertNwNotAvailable(c, "kiwl$%^")
 }
 
 func (s *DockerDaemonSuite) TestDaemonRestartRestoreBridgeNetwork(t *check.C) {

vendor.conf

@@ -22,7 +22,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
 github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-github.com/docker/libnetwork ca62711acec77034e0a670188628e26025e1482d
+github.com/docker/libnetwork 802895f7f342253a60f1666565e84707851a94f2
 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

vendor/github.com/docker/libnetwork/agent.go

@@ -3,6 +3,7 @@ package libnetwork
 //go:generate protoc -I.:Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. agent.proto
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -285,6 +286,7 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr st
 	}
 
 	ch, cancel := nDB.Watch("endpoint_table", "", "")
+	nodeCh, cancel := nDB.Watch(networkdb.NodeTable, "", "")
 
 	c.Lock()
 	c.agent = &agent{
@@ -297,6 +299,7 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr st
 	c.Unlock()
 
 	go c.handleTableEvents(ch, c.handleEpTableEvent)
+	go c.handleTableEvents(nodeCh, c.handleNodeTableEvent)
 
 	drvEnc := discoverapi.DriverEncryptionConfig{}
 	keys, tags = c.getKeys(subsysIPSec)
@@ -634,6 +637,31 @@ func (n *network) handleDriverTableEvent(ev events.Event) {
 	d.EventNotify(etype, n.ID(), tname, key, value)
 }
 
+func (c *controller) handleNodeTableEvent(ev events.Event) {
+	var (
+		value    []byte
+		isAdd    bool
+		nodeAddr networkdb.NodeAddr
+	)
+	switch event := ev.(type) {
+	case networkdb.CreateEvent:
+		value = event.Value
+		isAdd = true
+	case networkdb.DeleteEvent:
+		value = event.Value
+	case networkdb.UpdateEvent:
+		logrus.Errorf("Unexpected update node table event = %#v", event)
+	}
+
+	err := json.Unmarshal(value, &nodeAddr)
+	if err != nil {
+		logrus.Errorf("Error unmarshalling node table event %v", err)
+		return
+	}
+	c.processNodeDiscovery([]net.IP{nodeAddr.Addr}, isAdd)
+
+}
+
 func (c *controller) handleEpTableEvent(ev events.Event) {
 	var (
 		nid   string

vendor/github.com/docker/libnetwork/config/config.go

@@ -1,8 +1,6 @@
 package config
 
 import (
-	"fmt"
-	"regexp"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -17,12 +15,6 @@ import (
 	"github.com/docker/libnetwork/osl"
 )
 
-// restrictedNameRegex represents the regular expression which regulates the allowed network or endpoint names.
-const restrictedNameRegex = `^[\w]+[\w-. ]*[\w]+$`
-
-// RestrictedNamePattern is a regular expression to validate names against the collection of restricted characters.
-var restrictedNamePattern = regexp.MustCompile(restrictedNameRegex)
-
 // Config encapsulates configurations of various Libnetwork components
 type Config struct {
 	Daemon          DaemonCfg
@@ -240,12 +232,12 @@ func (c *Config) ProcessOptions(options ...Option) {
 	}
 }
 
-// ValidateName validates configuration objects supported by libnetwork
+// IsValidName validates configuration objects supported by libnetwork
-func ValidateName(name string) error {
+func IsValidName(name string) bool {
-	if !restrictedNamePattern.MatchString(name) {
+	if strings.TrimSpace(name) == "" {
-		return fmt.Errorf("%q includes invalid characters, resource name has to conform to %q", name, restrictedNameRegex)
+		return false
 	}
-	return nil
+	return true
 }
 
 // OptionLocalKVProvider function returns an option setter for kvstore provider

vendor/github.com/docker/libnetwork/controller.go

@@ -567,6 +567,12 @@ func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capabil
 	if c.cfg != nil {
 		addr := strings.Split(c.cfg.Cluster.Address, ":")
 		self = net.ParseIP(addr[0])
+		// if external kvstore is not configured, try swarm-mode config
+		if self == nil {
+			if agent := c.getAgent(); agent != nil {
+				self = net.ParseIP(agent.advertiseAddr)
+			}
+		}
 	}
 
 	if d == nil || cap.DataScope != datastore.GlobalScope || nodes == nil {
@@ -647,8 +653,8 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...
 		}
 	}
 
-	if err := config.ValidateName(name); err != nil {
+	if !config.IsValidName(name) {
-		return nil, ErrInvalidName(err.Error())
+		return nil, ErrInvalidName(name)
 	}
 
 	if id == "" {

vendor/github.com/docker/libnetwork/datastore/datastore.go

@@ -40,6 +40,8 @@ type DataStore interface {
 	// key. The caller must pass a KVObject of the same type as
 	// the objects that need to be listed
 	List(string, KVObject) ([]KVObject, error)
+	// Map returns a Map of KVObjects
+	Map(key string, kvObject KVObject) (map[string]KVObject, error)
 	// Scope returns the scope of the store
 	Scope() string
 	// KVStore returns access to the KV Store
@@ -512,23 +514,34 @@ func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
 		return ds.cache.list(kvObject)
 	}
 
+	var kvol []KVObject
+	cb := func(key string, val KVObject) {
+		kvol = append(kvol, val)
+	}
+	err := ds.iterateKVPairsFromStore(key, kvObject, cb)
+	if err != nil {
+		return nil, err
+	}
+	return kvol, nil
+}
+
+func (ds *datastore) iterateKVPairsFromStore(key string, kvObject KVObject, callback func(string, KVObject)) error {
 	// Bail out right away if the kvObject does not implement KVConstructor
 	ctor, ok := kvObject.(KVConstructor)
 	if !ok {
-		return nil, fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
+		return fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
 	}
 
 	// Make sure the parent key exists
 	if err := ds.ensureParent(key); err != nil {
-		return nil, err
+		return err
 	}
 
 	kvList, err := ds.store.List(key)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	var kvol []KVObject
 	for _, kvPair := range kvList {
 		if len(kvPair.Value) == 0 {
 			continue
@@ -536,16 +549,33 @@ func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
 
 		dstO := ctor.New()
 		if err := dstO.SetValue(kvPair.Value); err != nil {
-			return nil, err
+			return err
 		}
 
 		// Make sure the object has a correct view of the DB index in
 		// case we need to modify it and update the DB.
 		dstO.SetIndex(kvPair.LastIndex)
-
+		callback(kvPair.Key, dstO)
-		kvol = append(kvol, dstO)
 	}
 
+	return nil
+}
+
+func (ds *datastore) Map(key string, kvObject KVObject) (map[string]KVObject, error) {
+	if ds.sequential {
+		ds.Lock()
+		defer ds.Unlock()
+	}
+
+	kvol := make(map[string]KVObject)
+	cb := func(key string, val KVObject) {
+		// Trim the leading & trailing "/" to make it consistent across all stores
+		kvol[strings.Trim(key, "/")] = val
+	}
+	err := ds.iterateKVPairsFromStore(key, kvObject, cb)
+	if err != nil {
+		return nil, err
+	}
 	return kvol, nil
 }
 

vendor/github.com/docker/libnetwork/drivers/bridge/resolvconf.go

@@ -1,67 +0,0 @@
-package bridge
-
-import (
-	"bytes"
-	"io/ioutil"
-	"regexp"
-)
-
-const (
-	ipv4NumBlock = `(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`
-	ipv4Address  = `(` + ipv4NumBlock + `\.){3}` + ipv4NumBlock
-
-	// This is not an IPv6 address verifier as it will accept a super-set of IPv6, and also
-	// will *not match* IPv4-Embedded IPv6 Addresses (RFC6052), but that and other variants
-	// -- e.g. other link-local types -- either won't work in containers or are unnecessary.
-	// For readability and sufficiency for Docker purposes this seemed more reasonable than a
-	// 1000+ character regexp with exact and complete IPv6 validation
-	ipv6Address = `([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{0,4})`
-)
-
-var nsRegexp = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `)|(` + ipv6Address + `))\s*$`)
-
-func readResolvConf() ([]byte, error) {
-	resolv, err := ioutil.ReadFile("/etc/resolv.conf")
-	if err != nil {
-		return nil, err
-	}
-	return resolv, nil
-}
-
-// getLines parses input into lines and strips away comments.
-func getLines(input []byte, commentMarker []byte) [][]byte {
-	lines := bytes.Split(input, []byte("\n"))
-	var output [][]byte
-	for _, currentLine := range lines {
-		var commentIndex = bytes.Index(currentLine, commentMarker)
-		if commentIndex == -1 {
-			output = append(output, currentLine)
-		} else {
-			output = append(output, currentLine[:commentIndex])
-		}
-	}
-	return output
-}
-
-// GetNameserversAsCIDR returns nameservers (if any) listed in
-// /etc/resolv.conf as CIDR blocks (e.g., "1.2.3.4/32")
-// This function's output is intended for net.ParseCIDR
-func getNameserversAsCIDR(resolvConf []byte) []string {
-	nameservers := []string{}
-	for _, nameserver := range getNameservers(resolvConf) {
-		nameservers = append(nameservers, nameserver+"/32")
-	}
-	return nameservers
-}
-
-// GetNameservers returns nameservers (if any) listed in /etc/resolv.conf
-func getNameservers(resolvConf []byte) []string {
-	nameservers := []string{}
-	for _, line := range getLines(resolvConf, []byte("#")) {
-		var ns = nsRegexp.FindSubmatch(line)
-		if len(ns) > 0 {
-			nameservers = append(nameservers, string(ns[1]))
-		}
-	}
-	return nameservers
-}

vendor/github.com/docker/libnetwork/endpoint_info.go

@@ -181,6 +181,9 @@ type tableEntry struct {
 }
 
 func (ep *endpoint) Info() EndpointInfo {
+	if ep.sandboxID != "" {
+		return ep
+	}
 	n, err := ep.getNetworkFromStore()
 	if err != nil {
 		return nil

vendor/github.com/docker/libnetwork/error.go

@@ -69,7 +69,7 @@ func (ii ErrInvalidID) Error() string {
 func (ii ErrInvalidID) BadRequest() {}
 
 // ErrInvalidName is returned when a query-by-name or resource create method is
-// invoked with an invalid name parameter
+// invoked with an empty name parameter
 type ErrInvalidName string
 
 func (in ErrInvalidName) Error() string {

vendor/github.com/docker/libnetwork/ipvs/ipvs.go

@@ -42,6 +42,7 @@ type Destination struct {
 // Handle provides a namespace specific ipvs handle to program ipvs
 // rules.
 type Handle struct {
+	seq  uint32
 	sock *nl.NetlinkSocket
 }
 
@@ -82,6 +83,11 @@ func (i *Handle) NewService(s *Service) error {
 	return i.doCmd(s, nil, ipvsCmdNewService)
 }
 
+// IsServicePresent queries for the ipvs service in the passed handle.
+func (i *Handle) IsServicePresent(s *Service) bool {
+	return nil == i.doCmd(s, nil, ipvsCmdGetService)
+}
+
 // UpdateService updates an already existing service in the passed
 // handle.
 func (i *Handle) UpdateService(s *Service) error {

vendor/github.com/docker/libnetwork/ipvs/netlink.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
@@ -118,6 +119,7 @@ func fillDestinaton(d *Destination) nl.NetlinkRequestData {
 
 func (i *Handle) doCmd(s *Service, d *Destination, cmd uint8) error {
 	req := newIPVSRequest(cmd)
+	req.Seq = atomic.AddUint32(&i.seq, 1)
 	req.AddData(fillService(s))
 
 	if d != nil {
@@ -206,7 +208,7 @@ done:
 		}
 		for _, m := range msgs {
 			if m.Header.Seq != req.Seq {
-				return nil, fmt.Errorf("Wrong Seq nr %d, expected %d", m.Header.Seq, req.Seq)
+				continue
 			}
 			if m.Header.Pid != pid {
 				return nil, fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid)

vendor/github.com/docker/libnetwork/network.go

@@ -879,9 +879,8 @@ func (n *network) addEndpoint(ep *endpoint) error {
 
 func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error) {
 	var err error
-
+	if !config.IsValidName(name) {
-	if err = config.ValidateName(name); err != nil {
+		return nil, ErrInvalidName(name)
-		return nil, ErrInvalidName(err.Error())
 	}
 
 	if _, err = n.EndpointByName(name); err == nil {

vendor/github.com/docker/libnetwork/networkdb/event_delegate.go

@@ -1,12 +1,28 @@
 package networkdb
 
-import "github.com/hashicorp/memberlist"
+import (
+	"encoding/json"
+	"net"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/hashicorp/memberlist"
+)
 
 type eventDelegate struct {
 	nDB *NetworkDB
 }
 
+func (e *eventDelegate) broadcastNodeEvent(addr net.IP, op opType) {
+	value, err := json.Marshal(&NodeAddr{addr})
+	if err == nil {
+		e.nDB.broadcaster.Write(makeEvent(op, NodeTable, "", "", value))
+	} else {
+		logrus.Errorf("Error marshalling node broadcast event %s", addr.String())
+	}
+}
+
 func (e *eventDelegate) NotifyJoin(mn *memberlist.Node) {
+	e.broadcastNodeEvent(mn.Addr, opCreate)
 	e.nDB.Lock()
 	// In case the node is rejoining after a failure or leave,
 	// wait until an explicit join message arrives before adding
@@ -24,6 +40,7 @@ func (e *eventDelegate) NotifyJoin(mn *memberlist.Node) {
 }
 
 func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) {
+	e.broadcastNodeEvent(mn.Addr, opDelete)
 	e.nDB.deleteNodeTableEntries(mn.Name)
 	e.nDB.deleteNetworkEntriesForNode(mn.Name)
 	e.nDB.Lock()

vendor/github.com/docker/libnetwork/networkdb/watch.go

@@ -1,6 +1,10 @@
 package networkdb
 
-import "github.com/docker/go-events"
+import (
+	"net"
+
+	"github.com/docker/go-events"
+)
 
 type opType uint8
 
@@ -17,6 +21,14 @@ type event struct {
 	Value     []byte
 }
 
+// NodeTable represents table event for node join and leave
+const NodeTable = "NodeTable"
+
+// NodeAddr represents the value carried for node event in NodeTable
+type NodeAddr struct {
+	Addr net.IP
+}
+
 // CreateEvent generates a table entry create event to the watchers
 type CreateEvent event
 

vendor/github.com/docker/libnetwork/service_common.go

@@ -61,11 +61,6 @@ func (c *controller) cleanupServiceBindings(cleanupNID string) {
 }
 
 func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, ingressPorts []*PortConfig, aliases []string, ip net.IP) error {
-	var (
-		s          *service
-		addService bool
-	)
-
 	n, err := c.NetworkByID(nid)
 	if err != nil {
 		return err
@@ -123,11 +118,6 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
 		fwMarkCtrMu.Unlock()
 
 		s.loadBalancers[nid] = lb
-
-		// Since we just created this load balancer make sure
-		// we add a new service service in IPVS rules.
-		addService = true
-
 	}
 
 	lb.backEnds[eid] = ip
@@ -135,7 +125,7 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
 	// Add loadbalancer service and backend in all sandboxes in
 	// the network only if vip is valid.
 	if len(vip) != 0 {
-		n.(*network).addLBBackend(ip, vip, lb.fwMark, ingressPorts, addService)
+		n.(*network).addLBBackend(ip, vip, lb.fwMark, ingressPorts)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/service_linux.go

@@ -68,7 +68,7 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
 
 	if n.ingress {
 		if err := addRedirectRules(sb.Key(), eIP, ep.ingressPorts); err != nil {
-			logrus.Errorf("Failed to add redirect rules for ep %s: %v", ep.Name(), err)
+			logrus.Errorf("Failed to add redirect rules for ep %s (%s): %v", ep.Name(), ep.ID()[0:7], err)
 		}
 	}
 
@@ -97,20 +97,16 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
 		}
 
 		lb.service.Lock()
-		addService := true
 		for _, ip := range lb.backEnds {
-			sb.addLBBackend(ip, lb.vip, lb.fwMark, lb.service.ingressPorts,
+			sb.addLBBackend(ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress)
-				eIP, gwIP, addService, n.ingress)
-			addService = false
 		}
 		lb.service.Unlock()
 	}
 }
 
 // Add loadbalancer backend to all sandboxes which has a connection to
-// this network. If needed add the service as well, as specified by
+// this network. If needed add the service as well.
-// the addService bool.
+func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig) {
-func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, addService bool) {
 	n.WalkEndpoints(func(e Endpoint) bool {
 		ep := e.(*endpoint)
 		if sb, ok := ep.getSandbox(); ok {
@@ -123,7 +119,7 @@ func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 				gwIP = ep.Iface().Address().IP
 			}
 
-			sb.addLBBackend(ip, vip, fwMark, ingressPorts, ep.Iface().Address(), gwIP, addService, n.ingress)
+			sb.addLBBackend(ip, vip, fwMark, ingressPorts, ep.Iface().Address(), gwIP, n.ingress)
 		}
 
 		return false
@@ -154,7 +150,7 @@ func (n *network) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Por
 }
 
 // Add loadbalancer backend into one connected sandbox.
-func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, addService bool, isIngressNetwork bool) {
+func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, isIngressNetwork bool) {
 	if sb.osSbox == nil {
 		return
 	}
@@ -165,7 +161,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 
 	i, err := ipvs.New(sb.Key())
 	if err != nil {
-		logrus.Errorf("Failed to create an ipvs handle for sbox %s: %v", sb.Key(), err)
+		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb addition: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
 		return
 	}
 	defer i.Close()
@@ -176,7 +172,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 		SchedName:     ipvs.RoundRobin,
 	}
 
-	if addService {
+	if !i.IsServicePresent(s) {
 		var filteredPorts []*PortConfig
 		if sb.ingress {
 			filteredPorts = filterPortConfigs(ingressPorts, false)
@@ -186,14 +182,14 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 			}
 		}
 
-		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v", vip, fwMark, ingressPorts)
+		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %s (%s)", vip, fwMark, ingressPorts, sb.ID()[0:7], sb.ContainerID()[0:7])
 		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, false); err != nil {
-			logrus.Errorf("Failed to add firewall mark rule in sbox %s: %v", sb.Key(), err)
+			logrus.Errorf("Failed to add firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
 			return
 		}
 
-		if err := i.NewService(s); err != nil {
+		if err := i.NewService(s); err != nil && err != syscall.EEXIST {
-			logrus.Errorf("Failed to create a new service for vip %s fwmark %d: %v", vip, fwMark, err)
+			logrus.Errorf("Failed to create a new service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
 			return
 		}
 	}
@@ -208,7 +204,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 	// destination.
 	s.SchedName = ""
 	if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST {
-		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sb %s: %v", ip, vip, fwMark, sb.containerID, err)
+		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
 	}
 }
 
@@ -224,7 +220,7 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 
 	i, err := ipvs.New(sb.Key())
 	if err != nil {
-		logrus.Errorf("Failed to create an ipvs handle for sbox %s: %v", sb.Key(), err)
+		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb removal: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
 		return
 	}
 	defer i.Close()
@@ -240,14 +236,14 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 		Weight:        1,
 	}
 
-	if err := i.DelDestination(s, d); err != nil {
+	if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {
-		logrus.Infof("Failed to delete real server %s for vip %s fwmark %d: %v", ip, vip, fwMark, err)
+		logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
 	}
 
 	if rmService {
 		s.SchedName = ipvs.RoundRobin
-		if err := i.DelService(s); err != nil {
+		if err := i.DelService(s); err != nil && err != syscall.ENOENT {
-			logrus.Errorf("Failed to delete service for vip %s fwmark %d: %v", vip, fwMark, err)
+			logrus.Errorf("Failed to delete service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
 		}
 
 		var filteredPorts []*PortConfig
@@ -259,7 +255,7 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 		}
 
 		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, true); err != nil {
-			logrus.Errorf("Failed to delete firewall mark rule in sbox %s: %v", sb.Key(), err)
+			logrus.Errorf("Failed to delete firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
 		}
 	}
 }

vendor/github.com/docker/libnetwork/service_windows.go

@@ -2,7 +2,7 @@ package libnetwork
 
 import "net"
 
-func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, addService bool) {
+func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig) {
 }
 
 func (n *network) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, rmService bool) {

vendor/github.com/docker/libnetwork/store.go

@@ -2,6 +2,7 @@ package libnetwork
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libkv/store/boltdb"
@@ -152,21 +153,24 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 			continue
 		}
 
+		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
+		if err != nil {
+			if err != datastore.ErrKeyNotFound {
+				logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
+			}
+		}
+
 		for _, kvo := range kvol {
 			n := kvo.(*network)
 			n.Lock()
 			n.ctrlr = c
-			n.Unlock()
-
 			ec := &endpointCnt{n: n}
-			err = store.GetObject(datastore.Key(ec.Key()...), ec)
+			// Trim the leading & trailing "/" to make it consistent across all stores
-			if err != nil && !n.inDelete {
+			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
-				logrus.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
+				ec = val.(*endpointCnt)
-				continue
+				ec.n = n
+				n.epCnt = ec
 			}
-
-			n.Lock()
-			n.epCnt = ec
 			n.scope = store.Scope()
 			n.Unlock()
 			nl = append(nl, n)