Merge pull request #16 from thaJeztah/18.06-backport-scalable-lb

[18.06] Improve scalability of the Linux load balancing

daemon/network.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -230,9 +229,7 @@ func (daemon *Daemon) releaseIngress(id string) {
 		return
 	}
 
-	daemon.deleteLoadBalancerSandbox(n)
+	if err := n.Delete(libnetwork.NetworkDeleteOptionRemoveLB); err != nil {
-
-	if err := n.Delete(); err != nil {
 		logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
 		return
 	}
@@ -349,7 +346,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		nwOptions = append(nwOptions, libnetwork.NetworkOptionConfigFrom(create.ConfigFrom.Network))
 	}
 
-	if agent && driver == "overlay" && (create.Ingress || runtime.GOOS == "windows") {
+	if agent && driver == "overlay" {
 		nodeIP, exists := daemon.GetAttachmentStore().GetIPForNetwork(id)
 		if !exists {
 			return nil, fmt.Errorf("Failed to find a load balancer IP to use for network: %v", id)
@@ -512,37 +509,6 @@ func (daemon *Daemon) DeleteNetwork(networkID string) error {
 	return daemon.deleteNetwork(n, false)
 }
 
-func (daemon *Daemon) deleteLoadBalancerSandbox(n libnetwork.Network) {
-	controller := daemon.netController
-
-	//The only endpoint left should be the LB endpoint (nw.Name() + "-endpoint")
-	endpoints := n.Endpoints()
-	if len(endpoints) == 1 {
-		sandboxName := n.Name() + "-sbox"
-
-		info := endpoints[0].Info()
-		if info != nil {
-			sb := info.Sandbox()
-			if sb != nil {
-				if err := sb.DisableService(); err != nil {
-					logrus.Warnf("Failed to disable service on sandbox %s: %v", sandboxName, err)
-					//Ignore error and attempt to delete the load balancer endpoint
-				}
-			}
-		}
-
-		if err := endpoints[0].Delete(true); err != nil {
-			logrus.Warnf("Failed to delete endpoint %s (%s) in %s: %v", endpoints[0].Name(), endpoints[0].ID(), sandboxName, err)
-			//Ignore error and attempt to delete the sandbox.
-		}
-
-		if err := controller.SandboxDestroy(sandboxName); err != nil {
-			logrus.Warnf("Failed to delete %s sandbox: %v", sandboxName, err)
-			//Ignore error and attempt to delete the network.
-		}
-	}
-}
-
 func (daemon *Daemon) deleteNetwork(nw libnetwork.Network, dynamic bool) error {
 	if runconfig.IsPreDefinedNetwork(nw.Name()) && !dynamic {
 		err := fmt.Errorf("%s is a pre-defined network and cannot be removed", nw.Name())

hack/dockerfile/install/proxy.installer

@@ -3,7 +3,7 @@
 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
 # updating the binary version, consider updating github.com/docker/libnetwork
 # in vendor.conf accordingly
-LIBNETWORK_COMMIT=430c00a6a6b3dfdd774f21e1abd4ad6b0216c629
+LIBNETWORK_COMMIT=3ac297bc7fd0afec9051bbb47024c9bc1d75bf5b
 
 install_proxy() {
 	case "$1" in

vendor.conf

@@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b
 #get libnetwork packages
 
 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
-github.com/docker/libnetwork 430c00a6a6b3dfdd774f21e1abd4ad6b0216c629
+github.com/docker/libnetwork 3ac297bc7fd0afec9051bbb47024c9bc1d75bf5b
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

vendor/github.com/docker/libnetwork/controller.go

@@ -871,7 +871,7 @@ addToStore:
 		}
 	}()
 
-	if len(network.loadBalancerIP) != 0 {
+	if network.hasLoadBalancerEndpoint() {
 		if err = network.createLoadBalancerSandbox(); err != nil {
 			return nil, err
 		}
@@ -1144,6 +1144,11 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S
 		}
 	}
 
+	if sb.osSbox != nil {
+		// Apply operating specific knobs on the load balancer sandbox
+		sb.osSbox.ApplyOSTweaks(sb.oslTypes)
+	}
+
 	c.Lock()
 	c.sandboxes[sb.id] = sb
 	c.Unlock()

vendor/github.com/docker/libnetwork/default_gateway.go

@@ -49,9 +49,11 @@ func (sb *sandbox) setupDefaultGW() error {
 
 	createOptions := []EndpointOption{CreateOptionAnonymous()}
 
-	eplen := gwEPlen
+	var gwName string
-	if len(sb.containerID) < gwEPlen {
+	if len(sb.containerID) <= gwEPlen {
-		eplen = len(sb.containerID)
+		gwName = "gateway_" + sb.containerID
+	} else {
+		gwName = "gateway_" + sb.id[:gwEPlen]
 	}
 
 	sbLabels := sb.Labels()
@@ -69,7 +71,7 @@ func (sb *sandbox) setupDefaultGW() error {
 		createOptions = append(createOptions, epOption)
 	}
 
-	newEp, err := n.CreateEndpoint("gateway_"+sb.containerID[0:eplen], createOptions...)
+	newEp, err := n.CreateEndpoint(gwName, createOptions...)
 	if err != nil {
 		return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err)
 	}

vendor/github.com/docker/libnetwork/diagnostic/types.go

@@ -120,3 +120,13 @@ type TablePeersResult struct {
 	TableObj
 	Elements []PeerEntryObj `json:"entries"`
 }
+
+// NetworkStatsResult network db stats related to entries and queue len for a network
+type NetworkStatsResult struct {
+	Entries  int `json:"entries"`
+	QueueLen int `jsoin:"qlen"`
+}
+
+func (n *NetworkStatsResult) String() string {
+	return fmt.Sprintf("entries: %d, qlen: %d\n", n.Entries, n.QueueLen)
+}

vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -614,9 +614,7 @@ func (d *driver) checkConflict(config *networkConfiguration) error {
 	return nil
 }
 
-func (d *driver) createNetwork(config *networkConfiguration) error {
+func (d *driver) createNetwork(config *networkConfiguration) (err error) {
-	var err error
-
 	defer osl.InitOSContext()()
 
 	networkList := d.getNetworks()
@@ -775,7 +773,7 @@ func (d *driver) deleteNetwork(nid string) error {
 		}
 
 		if err := d.storeDelete(ep); err != nil {
-			logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+			logrus.Warnf("Failed to remove bridge endpoint %.7s from store: %v", ep.id, err)
 		}
 	}
 
@@ -1050,7 +1048,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 
 	if err = d.storeUpdate(endpoint); err != nil {
-		return fmt.Errorf("failed to save bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+		return fmt.Errorf("failed to save bridge endpoint %.7s to store: %v", endpoint.id, err)
 	}
 
 	return nil
@@ -1116,7 +1114,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	}
 
 	if err := d.storeDelete(ep); err != nil {
-		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+		logrus.Warnf("Failed to remove bridge endpoint %.7s from store: %v", ep.id, err)
 	}
 
 	return nil
@@ -1290,7 +1288,7 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
 	}()
 
 	if err = d.storeUpdate(endpoint); err != nil {
-		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+		return fmt.Errorf("failed to update bridge endpoint %.7s to store: %v", endpoint.id, err)
 	}
 
 	if !network.config.EnableICC {
@@ -1332,7 +1330,7 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
 	clearEndpointConnections(d.nlh, endpoint)
 
 	if err = d.storeUpdate(endpoint); err != nil {
-		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+		return fmt.Errorf("failed to update bridge endpoint %.7s to store: %v", endpoint.id, err)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go

@@ -62,7 +62,7 @@ func (d *driver) populateNetworks() error {
 		if err = d.createNetwork(ncfg); err != nil {
 			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)
 		}
-		logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
+		logrus.Debugf("Network (%.7s) restored", ncfg.ID)
 	}
 
 	return nil
@@ -82,16 +82,16 @@ func (d *driver) populateEndpoints() error {
 		ep := kvo.(*bridgeEndpoint)
 		n, ok := d.networks[ep.nid]
 		if !ok {
-			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Network (%.7s) not found for restored bridge endpoint (%.7s)", ep.nid, ep.id)
-			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.id[0:7])
+			logrus.Debugf("Deleting stale bridge endpoint (%.7s) from store", ep.id)
 			if err := d.storeDelete(ep); err != nil {
-				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.id[0:7])
+				logrus.Debugf("Failed to delete stale bridge endpoint (%.7s) from store", ep.id)
 			}
 			continue
 		}
 		n.endpoints[ep.id] = ep
 		n.restorePortAllocations(ep)
-		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+		logrus.Debugf("Endpoint (%.7s) restored to network (%.7s)", ep.id, ep.nid)
 	}
 
 	return nil
@@ -382,7 +382,7 @@ func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
 	ep.extConnConfig.PortBindings = ep.portMapping
 	_, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy)
 	if err != nil {
-		logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err)
+		logrus.Warnf("Failed to reserve existing port mapping for endpoint %.7s:%v", ep.id, err)
 	}
 	ep.extConnConfig.PortBindings = tmp
 }

vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go

@@ -53,7 +53,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 
 	if err := d.storeUpdate(ep); err != nil {
-		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to save ipvlan endpoint %.7s to store: %v", ep.id, err)
 	}
 
 	n.addEndpoint(ep)
@@ -82,7 +82,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	}
 
 	if err := d.storeDelete(ep); err != nil {
-		logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
+		logrus.Warnf("Failed to remove ipvlan endpoint %.7s from store: %v", ep.id, err)
 	}
 	n.deleteEndpoint(ep.id)
 	return nil

vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go

@@ -117,7 +117,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return err
 	}
 	if err = d.storeUpdate(ep); err != nil {
-		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to save ipvlan endpoint %.7s to store: %v", ep.id, err)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_network.go

@@ -156,7 +156,7 @@ func (d *driver) DeleteNetwork(nid string) error {
 		}
 
 		if err := d.storeDelete(ep); err != nil {
-			logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
+			logrus.Warnf("Failed to remove ipvlan endpoint %.7s from store: %v", ep.id, err)
 		}
 	}
 	// delete the *network

vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go

@@ -95,15 +95,15 @@ func (d *driver) populateEndpoints() error {
 		ep := kvo.(*endpoint)
 		n, ok := d.networks[ep.nid]
 		if !ok {
-			logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Network (%.7s) not found for restored ipvlan endpoint (%.7s)", ep.nid, ep.id)
-			logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.id[0:7])
+			logrus.Debugf("Deleting stale ipvlan endpoint (%.7s) from store", ep.id)
 			if err := d.storeDelete(ep); err != nil {
-				logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.id[0:7])
+				logrus.Debugf("Failed to delete stale ipvlan endpoint (%.7s) from store", ep.id)
 			}
 			continue
 		}
 		n.endpoints[ep.id] = ep
-		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+		logrus.Debugf("Endpoint (%.7s) restored to network (%.7s)", ep.id, ep.nid)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go

@@ -58,7 +58,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 
 	if err := d.storeUpdate(ep); err != nil {
-		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to save macvlan endpoint %.7s to store: %v", ep.id, err)
 	}
 
 	n.addEndpoint(ep)
@@ -87,7 +87,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	}
 
 	if err := d.storeDelete(ep); err != nil {
-		logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
+		logrus.Warnf("Failed to remove macvlan endpoint %.7s from store: %v", ep.id, err)
 	}
 
 	n.deleteEndpoint(ep.id)

vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go

@@ -78,7 +78,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return err
 	}
 	if err := d.storeUpdate(ep); err != nil {
-		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to save macvlan endpoint %.7s to store: %v", ep.id, err)
 	}
 	return nil
 }

vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_network.go

@@ -160,7 +160,7 @@ func (d *driver) DeleteNetwork(nid string) error {
 		}
 
 		if err := d.storeDelete(ep); err != nil {
-			logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
+			logrus.Warnf("Failed to remove macvlan endpoint %.7s from store: %v", ep.id, err)
 		}
 	}
 	// delete the *network

vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go

@@ -95,15 +95,15 @@ func (d *driver) populateEndpoints() error {
 		ep := kvo.(*endpoint)
 		n, ok := d.networks[ep.nid]
 		if !ok {
-			logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Network (%.7s) not found for restored macvlan endpoint (%.7s)", ep.nid, ep.id)
-			logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.id[0:7])
+			logrus.Debugf("Deleting stale macvlan endpoint (%.7s) from store", ep.id)
 			if err := d.storeDelete(ep); err != nil {
-				logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.id[0:7])
+				logrus.Debugf("Failed to delete stale macvlan endpoint (%.7s) from store", ep.id)
 			}
 			continue
 		}
 		n.endpoints[ep.id] = ep
-		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+		logrus.Debugf("Endpoint (%.7s) restored to network (%.7s)", ep.id, ep.nid)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go

@@ -78,7 +78,7 @@ func (e *encrMap) String() string {
 }
 
 func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal, add bool) error {
-	logrus.Debugf("checkEncryption(%s, %v, %d, %t)", nid[0:7], rIP, vxlanID, isLocal)
+	logrus.Debugf("checkEncryption(%.7s, %v, %d, %t)", nid, rIP, vxlanID, isLocal)
 
 	n := d.network(nid)
 	if n == nil || !n.secure {
@@ -101,7 +101,7 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal
 			}
 			return false
 		}); err != nil {
-			logrus.Warnf("Failed to retrieve list of participating nodes in overlay network %s: %v", nid[0:5], err)
+			logrus.Warnf("Failed to retrieve list of participating nodes in overlay network %.5s: %v", nid, err)
 		}
 	default:
 		if len(d.network(nid).endpoints) > 0 {

vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go

@@ -69,7 +69,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	ep.ifName = containerIfName
 
 	if err = d.writeEndpointToStore(ep); err != nil {
-		return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to update overlay endpoint %.7s to local data store: %v", ep.id, err)
 	}
 
 	// Set the container interface and its peer MTU to 1450 to allow

vendor/github.com/docker/libnetwork/drivers/overlay/ostweaks_linux.go

@@ -1,72 +1,23 @@
 package overlay
 
 import (
-	"io/ioutil"
-	"path"
 	"strconv"
-	"strings"
 
-	"github.com/sirupsen/logrus"
+	"github.com/docker/libnetwork/osl/kernel"
 )
 
-type conditionalCheck func(val1, val2 string) bool
+var ovConfig = map[string]*kernel.OSValue{
-
-type osValue struct {
-	value   string
-	checkFn conditionalCheck
-}
-
-var osConfig = map[string]osValue{
 	"net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher},
 	"net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher},
 	"net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher},
 }
 
-func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
-	if check == nil || check(val1, val2) {
-		return true
-	}
-	return false
-}
-
 func checkHigher(val1, val2 string) bool {
 	val1Int, _ := strconv.ParseInt(val1, 10, 32)
 	val2Int, _ := strconv.ParseInt(val2, 10, 32)
 	return val1Int < val2Int
 }
 
-// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
-// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
-func writeSystemProperty(key, value string) error {
-	keyPath := strings.Replace(key, ".", "/", -1)
-	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
-}
-
-func readSystemProperty(key string) (string, error) {
-	keyPath := strings.Replace(key, ".", "/", -1)
-	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
-	if err != nil {
-		return "", err
-	}
-	return string(value), nil
-}
-
 func applyOStweaks() {
-	for k, v := range osConfig {
+	kernel.ApplyOSTweaks(ovConfig)
-		// read the existing property from disk
-		oldv, err := readSystemProperty(k)
-		if err != nil {
-			logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err)
-			continue
-		}
-
-		if propertyIsValid(oldv, v.value, v.checkFn) {
-			// write new prop value to disk
-			if err := writeSystemProperty(k, v.value); err != nil {
-				logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err)
-				continue
-			}
-			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv)
-		}
-	}
 }

vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go

@@ -90,7 +90,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	n.addEndpoint(ep)
 
 	if err := d.writeEndpointToStore(ep); err != nil {
-		return fmt.Errorf("failed to update overlay endpoint %s to local store: %v", ep.id[0:7], err)
+		return fmt.Errorf("failed to update overlay endpoint %.7s to local store: %v", ep.id, err)
 	}
 
 	return nil
@@ -116,7 +116,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	n.deleteEndpoint(eid)
 
 	if err := d.deleteEndpointFromStore(ep); err != nil {
-		logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
+		logrus.Warnf("Failed to delete overlay endpoint %.7s from local store: %v", ep.id, err)
 	}
 
 	if ep.ifName == "" {

vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -274,7 +274,7 @@ func (d *driver) DeleteNetwork(nid string) error {
 		}
 
 		if err := d.deleteEndpointFromStore(ep); err != nil {
-			logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
+			logrus.Warnf("Failed to delete overlay endpoint %.7s from local store: %v", ep.id, err)
 		}
 	}
 	// flush the peerDB entries

vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go

@@ -137,10 +137,10 @@ func (d *driver) restoreEndpoints() error {
 		ep := kvo.(*endpoint)
 		n := d.network(ep.nid)
 		if n == nil {
-			logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Network (%.7s) not found for restored endpoint (%.7s)", ep.nid, ep.id)
-			logrus.Debugf("Deleting stale overlay endpoint (%s) from store", ep.id[0:7])
+			logrus.Debugf("Deleting stale overlay endpoint (%.7s) from store", ep.id)
 			if err := d.deleteEndpointFromStore(ep); err != nil {
-				logrus.Debugf("Failed to delete stale overlay endpoint (%s) from store", ep.id[0:7])
+				logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from store", ep.id)
 			}
 			continue
 		}

vendor/github.com/docker/libnetwork/drivers/overlay/overlay.pb.go

@@ -1,12 +1,11 @@
-// Code generated by protoc-gen-gogo.
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: overlay.proto
+// source: drivers/overlay/overlay.proto
-// DO NOT EDIT!
 
 /*
 	Package overlay is a generated protocol buffer package.
 
 	It is generated from these files:
-		overlay.proto
+		drivers/overlay/overlay.proto
 
 	It has these top-level messages:
 		PeerRecord
@@ -19,9 +18,6 @@ import math "math"
 import _ "github.com/gogo/protobuf/gogoproto"
 
 import strings "strings"
-import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
-import sort "sort"
-import strconv "strconv"
 import reflect "reflect"
 
 import io "io"
@@ -33,7 +29,9 @@ var _ = math.Inf
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the proto package it is being compiled against.
-const _ = proto.GoGoProtoPackageIsVersion1
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 
 // PeerRecord defines the information corresponding to a peer
 // container in the overlay network.
@@ -54,6 +52,27 @@ func (m *PeerRecord) Reset()                    { *m = PeerRecord{} }
 func (*PeerRecord) ProtoMessage()               {}
 func (*PeerRecord) Descriptor() ([]byte, []int) { return fileDescriptorOverlay, []int{0} }
 
+func (m *PeerRecord) GetEndpointIP() string {
+	if m != nil {
+		return m.EndpointIP
+	}
+	return ""
+}
+
+func (m *PeerRecord) GetEndpointMAC() string {
+	if m != nil {
+		return m.EndpointMAC
+	}
+	return ""
+}
+
+func (m *PeerRecord) GetTunnelEndpointIP() string {
+	if m != nil {
+		return m.TunnelEndpointIP
+	}
+	return ""
+}
+
 func init() {
 	proto.RegisterType((*PeerRecord)(nil), "overlay.PeerRecord")
 }
@@ -77,84 +96,49 @@ func valueToGoStringOverlay(v interface{}, typ string) string {
 	pv := reflect.Indirect(rv).Interface()
 	return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
 }
-func extensionToGoStringOverlay(e map[int32]github_com_gogo_protobuf_proto.Extension) string {
+func (m *PeerRecord) Marshal() (dAtA []byte, err error) {
-	if e == nil {
-		return "nil"
-	}
-	s := "map[int32]proto.Extension{"
-	keys := make([]int, 0, len(e))
-	for k := range e {
-		keys = append(keys, int(k))
-	}
-	sort.Ints(keys)
-	ss := []string{}
-	for _, k := range keys {
-		ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString())
-	}
-	s += strings.Join(ss, ",") + "}"
-	return s
-}
-func (m *PeerRecord) Marshal() (data []byte, err error) {
 	size := m.Size()
-	data = make([]byte, size)
+	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(data)
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
-	return data[:n], nil
+	return dAtA[:n], nil
 }
 
-func (m *PeerRecord) MarshalTo(data []byte) (int, error) {
+func (m *PeerRecord) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
 	if len(m.EndpointIP) > 0 {
-		data[i] = 0xa
+		dAtA[i] = 0xa
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointIP)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointIP)))
-		i += copy(data[i:], m.EndpointIP)
+		i += copy(dAtA[i:], m.EndpointIP)
 	}
 	if len(m.EndpointMAC) > 0 {
-		data[i] = 0x12
+		dAtA[i] = 0x12
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointMAC)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointMAC)))
-		i += copy(data[i:], m.EndpointMAC)
+		i += copy(dAtA[i:], m.EndpointMAC)
 	}
 	if len(m.TunnelEndpointIP) > 0 {
-		data[i] = 0x1a
+		dAtA[i] = 0x1a
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.TunnelEndpointIP)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.TunnelEndpointIP)))
-		i += copy(data[i:], m.TunnelEndpointIP)
+		i += copy(dAtA[i:], m.TunnelEndpointIP)
 	}
 	return i, nil
 }
 
-func encodeFixed64Overlay(data []byte, offset int, v uint64) int {
+func encodeVarintOverlay(dAtA []byte, offset int, v uint64) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	data[offset+4] = uint8(v >> 32)
-	data[offset+5] = uint8(v >> 40)
-	data[offset+6] = uint8(v >> 48)
-	data[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Overlay(data []byte, offset int, v uint32) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
-func encodeVarintOverlay(data []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
-		data[offset] = uint8(v&0x7f | 0x80)
+		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
-	data[offset] = uint8(v)
+	dAtA[offset] = uint8(v)
 	return offset + 1
 }
 func (m *PeerRecord) Size() (n int) {
@@ -208,8 +192,8 @@ func valueToStringOverlay(v interface{}) string {
 	pv := reflect.Indirect(rv).Interface()
 	return fmt.Sprintf("*%v", pv)
 }
-func (m *PeerRecord) Unmarshal(data []byte) error {
+func (m *PeerRecord) Unmarshal(dAtA []byte) error {
-	l := len(data)
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		preIndex := iNdEx
@@ -221,7 +205,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -249,7 +233,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -264,7 +248,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.EndpointIP = string(data[iNdEx:postIndex])
+			m.EndpointIP = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 2:
 			if wireType != 2 {
@@ -278,7 +262,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -293,7 +277,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.EndpointMAC = string(data[iNdEx:postIndex])
+			m.EndpointMAC = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 3:
 			if wireType != 2 {
@@ -307,7 +291,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -322,11 +306,11 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.TunnelEndpointIP = string(data[iNdEx:postIndex])
+			m.TunnelEndpointIP = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skipOverlay(data[iNdEx:])
+			skippy, err := skipOverlay(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
@@ -345,8 +329,8 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func skipOverlay(data []byte) (n int, err error) {
+func skipOverlay(dAtA []byte) (n int, err error) {
-	l := len(data)
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		var wire uint64
@@ -357,7 +341,7 @@ func skipOverlay(data []byte) (n int, err error) {
 			if iNdEx >= l {
 				return 0, io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -375,7 +359,7 @@ func skipOverlay(data []byte) (n int, err error) {
 					return 0, io.ErrUnexpectedEOF
 				}
 				iNdEx++
-				if data[iNdEx-1] < 0x80 {
+				if dAtA[iNdEx-1] < 0x80 {
 					break
 				}
 			}
@@ -392,7 +376,7 @@ func skipOverlay(data []byte) (n int, err error) {
 				if iNdEx >= l {
 					return 0, io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				length |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -415,7 +399,7 @@ func skipOverlay(data []byte) (n int, err error) {
 					if iNdEx >= l {
 						return 0, io.ErrUnexpectedEOF
 					}
-					b := data[iNdEx]
+					b := dAtA[iNdEx]
 					iNdEx++
 					innerWire |= (uint64(b) & 0x7F) << shift
 					if b < 0x80 {
@@ -426,7 +410,7 @@ func skipOverlay(data []byte) (n int, err error) {
 				if innerWireType == 4 {
 					break
 				}
-				next, err := skipOverlay(data[start:])
+				next, err := skipOverlay(dAtA[start:])
 				if err != nil {
 					return 0, err
 				}
@@ -450,19 +434,22 @@ var (
 	ErrIntOverflowOverlay   = fmt.Errorf("proto: integer overflow")
 )
 
+func init() { proto.RegisterFile("drivers/overlay/overlay.proto", fileDescriptorOverlay) }
+
 var fileDescriptorOverlay = []byte{
-	// 195 bytes of a gzipped FileDescriptorProto
+	// 212 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x2f, 0x4b, 0x2d,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x29, 0xca, 0x2c,
-	0xca, 0x49, 0xac, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0xa5, 0x44, 0xd2,
+	0x4b, 0x2d, 0x2a, 0xd6, 0xcf, 0x2f, 0x4b, 0x2d, 0xca, 0x49, 0xac, 0x84, 0xd1, 0x7a, 0x05, 0x45,
-	0xf3, 0xd3, 0xf3, 0xc1, 0x62, 0xfa, 0x20, 0x16, 0x44, 0x5a, 0x69, 0x2b, 0x23, 0x17, 0x57, 0x40,
+	0xf9, 0x25, 0xf9, 0x42, 0xec, 0x50, 0xae, 0x94, 0x48, 0x7a, 0x7e, 0x7a, 0x3e, 0x58, 0x4c, 0x1f,
-	0x6a, 0x6a, 0x51, 0x50, 0x6a, 0x72, 0x7e, 0x51, 0x8a, 0x90, 0x3e, 0x17, 0x77, 0x6a, 0x5e, 0x4a,
+	0xc4, 0x82, 0x48, 0x2b, 0x6d, 0x65, 0xe4, 0xe2, 0x0a, 0x48, 0x4d, 0x2d, 0x0a, 0x4a, 0x4d, 0xce,
-	0x41, 0x7e, 0x66, 0x5e, 0x49, 0x7c, 0x66, 0x81, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0xa7, 0x13, 0xdf,
+	0x2f, 0x4a, 0x11, 0xd2, 0xe7, 0xe2, 0x4e, 0xcd, 0x4b, 0x29, 0xc8, 0xcf, 0xcc, 0x2b, 0x89, 0xcf,
-	0xa3, 0x7b, 0xf2, 0x5c, 0xae, 0x50, 0x61, 0xcf, 0x80, 0x20, 0x2e, 0x98, 0x12, 0xcf, 0x02, 0x21,
+	0x2c, 0x90, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x74, 0xe2, 0x7b, 0x74, 0x4f, 0x9e, 0xcb, 0x15, 0x2a,
-	0x23, 0x2e, 0x1e, 0xb8, 0x86, 0xdc, 0xc4, 0x64, 0x09, 0x26, 0xb0, 0x0e, 0x7e, 0xa0, 0x0e, 0x6e,
+	0xec, 0x19, 0x10, 0xc4, 0x05, 0x53, 0xe2, 0x59, 0x20, 0x64, 0xc4, 0xc5, 0x03, 0xd7, 0x90, 0x9b,
-	0x98, 0x0e, 0x5f, 0x47, 0xe7, 0x20, 0xb8, 0xa9, 0xbe, 0x89, 0xc9, 0x42, 0x4e, 0x5c, 0x42, 0x25,
+	0x98, 0x2c, 0xc1, 0x04, 0xd6, 0xc1, 0xff, 0xe8, 0x9e, 0x3c, 0x37, 0x4c, 0x87, 0xaf, 0xa3, 0x73,
-	0xa5, 0x79, 0x79, 0xa9, 0x39, 0xf1, 0xc8, 0x76, 0x31, 0x83, 0x75, 0x8a, 0x00, 0x75, 0x0a, 0x84,
+	0x10, 0xdc, 0x54, 0xdf, 0xc4, 0x64, 0x21, 0x27, 0x2e, 0xa1, 0x92, 0xd2, 0xbc, 0xbc, 0xd4, 0x9c,
-	0x80, 0x65, 0x91, 0x6c, 0x14, 0x28, 0x41, 0x15, 0x29, 0x70, 0x92, 0xb8, 0xf1, 0x50, 0x8e, 0xe1,
+	0x78, 0x64, 0xbb, 0x98, 0xc1, 0x3a, 0x45, 0x1e, 0xdd, 0x93, 0x17, 0x08, 0x01, 0xcb, 0x22, 0xd9,
-	0xc3, 0x43, 0x39, 0xc6, 0x86, 0x47, 0x72, 0x8c, 0x27, 0x80, 0xf8, 0x02, 0x10, 0x3f, 0x00, 0xe2,
+	0x28, 0x50, 0x82, 0x2a, 0x52, 0xe0, 0x24, 0x71, 0xe3, 0xa1, 0x1c, 0xc3, 0x87, 0x87, 0x72, 0x8c,
-	0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbf, 0xd7, 0x7d, 0x7d, 0x08,
+	0x0d, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39,
-	0x01, 0x00, 0x00,
+	0xc6, 0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x48, 0x07, 0xf6, 0xf3,
+	0x18, 0x01, 0x00, 0x00,
 }

vendor/github.com/docker/libnetwork/drivers/remote/driver.go

@@ -1,16 +1,17 @@
 package remote
 
 import (
-	"errors"
 	"fmt"
 	"net"
 
+	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/discoverapi"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/drivers/remote/api"
 	"github.com/docker/libnetwork/types"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -49,7 +50,11 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		handleFunc = pg.Handle
 		activePlugins := pg.GetAllManagedPluginsByCap(driverapi.NetworkPluginEndpointType)
 		for _, ap := range activePlugins {
-			newPluginHandler(ap.Name(), ap.Client())
+			client, err := getPluginClient(ap)
+			if err != nil {
+				return err
+			}
+			newPluginHandler(ap.Name(), client)
 		}
 	}
 	handleFunc(driverapi.NetworkPluginEndpointType, newPluginHandler)
@@ -57,6 +62,28 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	return nil
 }
 
+func getPluginClient(p plugingetter.CompatPlugin) (*plugins.Client, error) {
+	if v1, ok := p.(plugingetter.PluginWithV1Client); ok {
+		return v1.Client(), nil
+	}
+
+	pa, ok := p.(plugingetter.PluginAddr)
+	if !ok {
+		return nil, errors.Errorf("unknown plugin type %T", p)
+	}
+
+	if pa.Protocol() != plugins.ProtocolSchemeHTTPV1 {
+		return nil, errors.Errorf("unsupported plugin protocol %s", pa.Protocol())
+	}
+
+	addr := pa.Addr()
+	client, err := plugins.NewClientWithTimeout(addr.Network()+"://"+addr.String(), nil, pa.Timeout())
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating plugin client")
+	}
+	return client, nil
+}
+
 // Get capability from client
 func (d *driver) getCapabilities() (*driverapi.Capability, error) {
 	var capResp api.GetCapabilityResponse

vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_endpoint_windows.go

@@ -80,7 +80,7 @@ func (n *network) removeEndpointWithAddress(addr *net.IPNet) {
 		_, err := hcsshim.HNSEndpointRequest("DELETE", networkEndpoint.profileID, "")
 
 		if err != nil {
-			logrus.Debugf("Failed to delete stale overlay endpoint (%s) from hns", networkEndpoint.id[0:7])
+			logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from hns", networkEndpoint.id)
 		}
 	}
 }

vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay.pb.go

@@ -1,12 +1,11 @@
-// Code generated by protoc-gen-gogo.
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: overlay.proto
+// source: drivers/windows/overlay/overlay.proto
-// DO NOT EDIT!
 
 /*
 	Package overlay is a generated protocol buffer package.
 
 	It is generated from these files:
-		overlay.proto
+		drivers/windows/overlay/overlay.proto
 
 	It has these top-level messages:
 		PeerRecord
@@ -19,9 +18,6 @@ import math "math"
 import _ "github.com/gogo/protobuf/gogoproto"
 
 import strings "strings"
-import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
-import sort "sort"
-import strconv "strconv"
 import reflect "reflect"
 
 import io "io"
@@ -33,7 +29,9 @@ var _ = math.Inf
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the proto package it is being compiled against.
-const _ = proto.GoGoProtoPackageIsVersion1
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 
 // PeerRecord defines the information corresponding to a peer
 // container in the overlay network.
@@ -54,6 +52,27 @@ func (m *PeerRecord) Reset()                    { *m = PeerRecord{} }
 func (*PeerRecord) ProtoMessage()               {}
 func (*PeerRecord) Descriptor() ([]byte, []int) { return fileDescriptorOverlay, []int{0} }
 
+func (m *PeerRecord) GetEndpointIP() string {
+	if m != nil {
+		return m.EndpointIP
+	}
+	return ""
+}
+
+func (m *PeerRecord) GetEndpointMAC() string {
+	if m != nil {
+		return m.EndpointMAC
+	}
+	return ""
+}
+
+func (m *PeerRecord) GetTunnelEndpointIP() string {
+	if m != nil {
+		return m.TunnelEndpointIP
+	}
+	return ""
+}
+
 func init() {
 	proto.RegisterType((*PeerRecord)(nil), "overlay.PeerRecord")
 }
@@ -77,84 +96,49 @@ func valueToGoStringOverlay(v interface{}, typ string) string {
 	pv := reflect.Indirect(rv).Interface()
 	return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
 }
-func extensionToGoStringOverlay(e map[int32]github_com_gogo_protobuf_proto.Extension) string {
+func (m *PeerRecord) Marshal() (dAtA []byte, err error) {
-	if e == nil {
-		return "nil"
-	}
-	s := "map[int32]proto.Extension{"
-	keys := make([]int, 0, len(e))
-	for k := range e {
-		keys = append(keys, int(k))
-	}
-	sort.Ints(keys)
-	ss := []string{}
-	for _, k := range keys {
-		ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString())
-	}
-	s += strings.Join(ss, ",") + "}"
-	return s
-}
-func (m *PeerRecord) Marshal() (data []byte, err error) {
 	size := m.Size()
-	data = make([]byte, size)
+	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(data)
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
-	return data[:n], nil
+	return dAtA[:n], nil
 }
 
-func (m *PeerRecord) MarshalTo(data []byte) (int, error) {
+func (m *PeerRecord) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
 	if len(m.EndpointIP) > 0 {
-		data[i] = 0xa
+		dAtA[i] = 0xa
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointIP)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointIP)))
-		i += copy(data[i:], m.EndpointIP)
+		i += copy(dAtA[i:], m.EndpointIP)
 	}
 	if len(m.EndpointMAC) > 0 {
-		data[i] = 0x12
+		dAtA[i] = 0x12
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointMAC)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointMAC)))
-		i += copy(data[i:], m.EndpointMAC)
+		i += copy(dAtA[i:], m.EndpointMAC)
 	}
 	if len(m.TunnelEndpointIP) > 0 {
-		data[i] = 0x1a
+		dAtA[i] = 0x1a
 		i++
-		i = encodeVarintOverlay(data, i, uint64(len(m.TunnelEndpointIP)))
+		i = encodeVarintOverlay(dAtA, i, uint64(len(m.TunnelEndpointIP)))
-		i += copy(data[i:], m.TunnelEndpointIP)
+		i += copy(dAtA[i:], m.TunnelEndpointIP)
 	}
 	return i, nil
 }
 
-func encodeFixed64Overlay(data []byte, offset int, v uint64) int {
+func encodeVarintOverlay(dAtA []byte, offset int, v uint64) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	data[offset+4] = uint8(v >> 32)
-	data[offset+5] = uint8(v >> 40)
-	data[offset+6] = uint8(v >> 48)
-	data[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Overlay(data []byte, offset int, v uint32) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
-func encodeVarintOverlay(data []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
-		data[offset] = uint8(v&0x7f | 0x80)
+		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
-	data[offset] = uint8(v)
+	dAtA[offset] = uint8(v)
 	return offset + 1
 }
 func (m *PeerRecord) Size() (n int) {
@@ -208,8 +192,8 @@ func valueToStringOverlay(v interface{}) string {
 	pv := reflect.Indirect(rv).Interface()
 	return fmt.Sprintf("*%v", pv)
 }
-func (m *PeerRecord) Unmarshal(data []byte) error {
+func (m *PeerRecord) Unmarshal(dAtA []byte) error {
-	l := len(data)
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		preIndex := iNdEx
@@ -221,7 +205,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -249,7 +233,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -264,7 +248,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.EndpointIP = string(data[iNdEx:postIndex])
+			m.EndpointIP = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 2:
 			if wireType != 2 {
@@ -278,7 +262,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -293,7 +277,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.EndpointMAC = string(data[iNdEx:postIndex])
+			m.EndpointMAC = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 3:
 			if wireType != 2 {
@@ -307,7 +291,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -322,11 +306,11 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.TunnelEndpointIP = string(data[iNdEx:postIndex])
+			m.TunnelEndpointIP = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skipOverlay(data[iNdEx:])
+			skippy, err := skipOverlay(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
@@ -345,8 +329,8 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func skipOverlay(data []byte) (n int, err error) {
+func skipOverlay(dAtA []byte) (n int, err error) {
-	l := len(data)
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		var wire uint64
@@ -357,7 +341,7 @@ func skipOverlay(data []byte) (n int, err error) {
 			if iNdEx >= l {
 				return 0, io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -375,7 +359,7 @@ func skipOverlay(data []byte) (n int, err error) {
 					return 0, io.ErrUnexpectedEOF
 				}
 				iNdEx++
-				if data[iNdEx-1] < 0x80 {
+				if dAtA[iNdEx-1] < 0x80 {
 					break
 				}
 			}
@@ -392,7 +376,7 @@ func skipOverlay(data []byte) (n int, err error) {
 				if iNdEx >= l {
 					return 0, io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				length |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -415,7 +399,7 @@ func skipOverlay(data []byte) (n int, err error) {
 					if iNdEx >= l {
 						return 0, io.ErrUnexpectedEOF
 					}
-					b := data[iNdEx]
+					b := dAtA[iNdEx]
 					iNdEx++
 					innerWire |= (uint64(b) & 0x7F) << shift
 					if b < 0x80 {
@@ -426,7 +410,7 @@ func skipOverlay(data []byte) (n int, err error) {
 				if innerWireType == 4 {
 					break
 				}
-				next, err := skipOverlay(data[start:])
+				next, err := skipOverlay(dAtA[start:])
 				if err != nil {
 					return 0, err
 				}
@@ -450,19 +434,22 @@ var (
 	ErrIntOverflowOverlay   = fmt.Errorf("proto: integer overflow")
 )
 
+func init() { proto.RegisterFile("drivers/windows/overlay/overlay.proto", fileDescriptorOverlay) }
+
 var fileDescriptorOverlay = []byte{
-	// 195 bytes of a gzipped FileDescriptorProto
+	// 220 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x2f, 0x4b, 0x2d,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4d, 0x29, 0xca, 0x2c,
-	0xca, 0x49, 0xac, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0xa5, 0x44, 0xd2,
+	0x4b, 0x2d, 0x2a, 0xd6, 0x2f, 0xcf, 0xcc, 0x4b, 0xc9, 0x2f, 0x2f, 0xd6, 0xcf, 0x2f, 0x4b, 0x2d,
-	0xf3, 0xd3, 0xf3, 0xc1, 0x62, 0xfa, 0x20, 0x16, 0x44, 0x5a, 0x69, 0x2b, 0x23, 0x17, 0x57, 0x40,
+	0xca, 0x49, 0xac, 0x84, 0xd1, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xec, 0x50, 0xae, 0x94,
-	0x6a, 0x6a, 0x51, 0x50, 0x6a, 0x72, 0x7e, 0x51, 0x8a, 0x90, 0x3e, 0x17, 0x77, 0x6a, 0x5e, 0x4a,
+	0x48, 0x7a, 0x7e, 0x7a, 0x3e, 0x58, 0x4c, 0x1f, 0xc4, 0x82, 0x48, 0x2b, 0x6d, 0x65, 0xe4, 0xe2,
-	0x41, 0x7e, 0x66, 0x5e, 0x49, 0x7c, 0x66, 0x81, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0xa7, 0x13, 0xdf,
+	0x0a, 0x48, 0x4d, 0x2d, 0x0a, 0x4a, 0x4d, 0xce, 0x2f, 0x4a, 0x11, 0xd2, 0xe7, 0xe2, 0x4e, 0xcd,
-	0xa3, 0x7b, 0xf2, 0x5c, 0xae, 0x50, 0x61, 0xcf, 0x80, 0x20, 0x2e, 0x98, 0x12, 0xcf, 0x02, 0x21,
+	0x4b, 0x29, 0xc8, 0xcf, 0xcc, 0x2b, 0x89, 0xcf, 0x2c, 0x90, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x74,
-	0x23, 0x2e, 0x1e, 0xb8, 0x86, 0xdc, 0xc4, 0x64, 0x09, 0x26, 0xb0, 0x0e, 0x7e, 0xa0, 0x0e, 0x6e,
+	0xe2, 0x7b, 0x74, 0x4f, 0x9e, 0xcb, 0x15, 0x2a, 0xec, 0x19, 0x10, 0xc4, 0x05, 0x53, 0xe2, 0x59,
-	0x98, 0x0e, 0x5f, 0x47, 0xe7, 0x20, 0xb8, 0xa9, 0xbe, 0x89, 0xc9, 0x42, 0x4e, 0x5c, 0x42, 0x25,
+	0x20, 0x64, 0xc4, 0xc5, 0x03, 0xd7, 0x90, 0x9b, 0x98, 0x2c, 0xc1, 0x04, 0xd6, 0xc1, 0xff, 0xe8,
-	0xa5, 0x79, 0x79, 0xa9, 0x39, 0xf1, 0xc8, 0x76, 0x31, 0x83, 0x75, 0x8a, 0x00, 0x75, 0x0a, 0x84,
+	0x9e, 0x3c, 0x37, 0x4c, 0x87, 0xaf, 0xa3, 0x73, 0x10, 0xdc, 0x54, 0xdf, 0xc4, 0x64, 0x21, 0x27,
-	0x80, 0x65, 0x91, 0x6c, 0x14, 0x28, 0x41, 0x15, 0x29, 0x70, 0x92, 0xb8, 0xf1, 0x50, 0x8e, 0xe1,
+	0x2e, 0xa1, 0x92, 0xd2, 0xbc, 0xbc, 0xd4, 0x9c, 0x78, 0x64, 0xbb, 0x98, 0xc1, 0x3a, 0x45, 0x1e,
-	0xc3, 0x43, 0x39, 0xc6, 0x86, 0x47, 0x72, 0x8c, 0x27, 0x80, 0xf8, 0x02, 0x10, 0x3f, 0x00, 0xe2,
+	0xdd, 0x93, 0x17, 0x08, 0x01, 0xcb, 0x22, 0xd9, 0x28, 0x50, 0x82, 0x2a, 0x52, 0xe0, 0x24, 0x71,
-	0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbf, 0xd7, 0x7d, 0x7d, 0x08,
+	0xe3, 0xa1, 0x1c, 0xc3, 0x87, 0x87, 0x72, 0x8c, 0x0d, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63,
-	0x01, 0x00, 0x00,
+	0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01,
+	0x00, 0x00, 0xff, 0xff, 0xc0, 0x48, 0xd1, 0xc0, 0x20, 0x01, 0x00, 0x00,
 }

vendor/github.com/docker/libnetwork/drivers/windows/windows.go

@@ -415,7 +415,7 @@ func (d *driver) DeleteNetwork(nid string) error {
 	// delele endpoints belong to this network
 	for _, ep := range n.endpoints {
 		if err := d.storeDelete(ep); err != nil {
-			logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+			logrus.Warnf("Failed to remove bridge endpoint %.7s from store: %v", ep.id, err)
 		}
 	}
 
@@ -704,7 +704,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 
 	if err = d.storeUpdate(endpoint); err != nil {
-		logrus.Errorf("Failed to save endpoint %s to store: %v", endpoint.id[0:7], err)
+		logrus.Errorf("Failed to save endpoint %.7s to store: %v", endpoint.id, err)
 	}
 
 	return nil
@@ -731,7 +731,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	}
 
 	if err := d.storeDelete(ep); err != nil {
-		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+		logrus.Warnf("Failed to remove bridge endpoint %.7s from store: %v", ep.id, err)
 	}
 	return nil
 }

vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go

@@ -64,7 +64,7 @@ func (d *driver) populateNetworks() error {
 		if err = d.createNetwork(ncfg); err != nil {
 			logrus.Warnf("could not create windows network for id %s hnsid %s while booting up from persistent state: %v", ncfg.ID, ncfg.HnsID, err)
 		}
-		logrus.Debugf("Network  %v (%s) restored", d.name, ncfg.ID[0:7])
+		logrus.Debugf("Network  %v (%.7s) restored", d.name, ncfg.ID)
 	}
 
 	return nil
@@ -87,15 +87,15 @@ func (d *driver) populateEndpoints() error {
 		}
 		n, ok := d.networks[ep.nid]
 		if !ok {
-			logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Network (%.7s) not found for restored endpoint (%.7s)", ep.nid, ep.id)
-			logrus.Debugf("Deleting stale endpoint (%s) from store", ep.id[0:7])
+			logrus.Debugf("Deleting stale endpoint (%.7s) from store", ep.id)
 			if err := d.storeDelete(ep); err != nil {
-				logrus.Debugf("Failed to delete stale endpoint (%s) from store", ep.id[0:7])
+				logrus.Debugf("Failed to delete stale endpoint (%.7s) from store", ep.id)
 			}
 			continue
 		}
 		n.endpoints[ep.id] = ep
-		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+		logrus.Debugf("Endpoint (%.7s) restored to network (%.7s)", ep.id, ep.nid)
 	}
 
 	return nil

vendor/github.com/docker/libnetwork/endpoint.go

@@ -540,6 +540,12 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
 		}
 	}()
 
+	// Load balancing endpoints should never have a default gateway nor
+	// should they alter the status of a network's default gateway
+	if ep.loadBalancer && !sb.ingress {
+		return nil
+	}
+
 	if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
 		return sb.setupDefaultGW()
 	}

vendor/github.com/docker/libnetwork/endpoint_info.go

@@ -49,6 +49,9 @@ type InterfaceInfo interface {
 
 	// LinkLocalAddresses returns the list of link-local (IPv4/IPv6) addresses assigned to the endpoint.
 	LinkLocalAddresses() []*net.IPNet
+
+	// SrcName returns the name of the interface w/in the container
+	SrcName() string
 }
 
 type endpointInterface struct {
@@ -272,6 +275,10 @@ func (epi *endpointInterface) LinkLocalAddresses() []*net.IPNet {
 	return epi.llAddrs
 }
 
+func (epi *endpointInterface) SrcName() string {
+	return epi.srcName
+}
+
 func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
 	epi.srcName = srcName
 	epi.dstPrefix = dstPrefix

vendor/github.com/docker/libnetwork/ipam/allocator.go

@@ -203,6 +203,10 @@ func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) {
 }
 
 // RequestPool returns an address pool along with its unique id.
+// addressSpace must be a valid address space name and must not be the empty string.
+// If pool is the empty string then the default predefined pool for addressSpace will be used, otherwise pool must be a valid IP address and length in CIDR notation.
+// If subPool is not empty, it must be a valid IP address and length in CIDR notation which is a sub-range of pool.
+// subPool must be empty if pool is empty.
 func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) {
 	logrus.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6)
 
@@ -283,8 +287,8 @@ retry:
 	return remove()
 }
 
-// Given the address space, returns the local or global PoolConfig based on the
+// Given the address space, returns the local or global PoolConfig based on whether the
-// address space is local or global. AddressSpace locality is being registered with IPAM out of band.
+// address space is local or global. AddressSpace locality is registered with IPAM out of band.
 func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) {
 	a.Lock()
 	defer a.Unlock()
@@ -295,6 +299,8 @@ func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) {
 	return aSpace, nil
 }
 
+// parsePoolRequest parses and validates a request to create a new pool under addressSpace and returns
+// a SubnetKey, network and range describing the request.
 func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) {
 	var (
 		nw  *net.IPNet

vendor/github.com/docker/libnetwork/ipam/structures.go

@@ -257,6 +257,7 @@ func (aSpace *addrSpace) New() datastore.KVObject {
 	}
 }
 
+// updatePoolDBOnAdd returns a closure which will add the subnet k to the address space when executed.
 func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange, pdf bool) (func() error, error) {
 	aSpace.Lock()
 	defer aSpace.Unlock()
@@ -281,7 +282,7 @@ func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *Addr
 		return func() error { return aSpace.alloc.insertBitMask(k, nw) }, nil
 	}
 
-	// This is a new non-master pool
+	// This is a new non-master pool (subPool)
 	p := &PoolData{
 		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
 		Pool:      nw,

vendor/github.com/docker/libnetwork/ipams/remote/remote.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/libnetwork/discoverapi"
 	"github.com/docker/libnetwork/ipamapi"
 	"github.com/docker/libnetwork/ipams/remote/api"
 	"github.com/docker/libnetwork/types"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -52,13 +54,39 @@ func Init(cb ipamapi.Callback, l, g interface{}) error {
 		handleFunc = pg.Handle
 		activePlugins := pg.GetAllManagedPluginsByCap(ipamapi.PluginEndpointType)
 		for _, ap := range activePlugins {
-			newPluginHandler(ap.Name(), ap.Client())
+			client, err := getPluginClient(ap)
+			if err != nil {
+				return err
+			}
+			newPluginHandler(ap.Name(), client)
 		}
 	}
 	handleFunc(ipamapi.PluginEndpointType, newPluginHandler)
 	return nil
 }
 
+func getPluginClient(p plugingetter.CompatPlugin) (*plugins.Client, error) {
+	if v1, ok := p.(plugingetter.PluginWithV1Client); ok {
+		return v1.Client(), nil
+	}
+
+	pa, ok := p.(plugingetter.PluginAddr)
+	if !ok {
+		return nil, errors.Errorf("unknown plugin type %T", p)
+	}
+
+	if pa.Protocol() != plugins.ProtocolSchemeHTTPV1 {
+		return nil, errors.Errorf("unsupported plugin protocol %s", pa.Protocol())
+	}
+
+	addr := pa.Addr()
+	client, err := plugins.NewClientWithTimeout(addr.Network()+"://"+addr.String(), nil, pa.Timeout())
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating plugin client")
+	}
+	return client, nil
+}
+
 func (a *allocator) call(methodName string, arg interface{}, retVal PluginResponse) error {
 	method := ipamapi.PluginEndpointType + "." + methodName
 	err := a.endpoint.Call(method, arg, retVal)

vendor/github.com/docker/libnetwork/network.go

@@ -40,7 +40,7 @@ type Network interface {
 	CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error)
 
 	// Delete the network.
-	Delete() error
+	Delete(options ...NetworkDeleteOption) error
 
 	// Endpoints returns the list of Endpoint(s) in this network.
 	Endpoints() []Endpoint
@@ -875,6 +875,28 @@ func (n *network) processOptions(options ...NetworkOption) {
 	}
 }
 
+type networkDeleteParams struct {
+	rmLBEndpoint bool
+}
+
+// NetworkDeleteOption is a type for optional parameters to pass to the
+// network.Delete() function.
+type NetworkDeleteOption func(p *networkDeleteParams)
+
+// NetworkDeleteOptionRemoveLB informs a network.Delete() operation that should
+// remove the load balancer endpoint for this network.  Note that the Delete()
+// method will automatically remove a load balancing endpoint for most networks
+// when the network is otherwise empty.  However, this does not occur for some
+// networks.  In particular, networks marked as ingress (which are supposed to
+// be more permanent than other overlay networks) won't automatically remove
+// the LB endpoint on Delete().  This method allows for explicit removal of
+// such networks provided there are no other endpoints present in the network.
+// If the network still has non-LB endpoints present, Delete() will not
+// remove the LB endpoint and will return an error.
+func NetworkDeleteOptionRemoveLB(p *networkDeleteParams) {
+	p.rmLBEndpoint = true
+}
+
 func (n *network) resolveDriver(name string, load bool) (driverapi.Driver, *driverapi.Capability, error) {
 	c := n.getController()
 
@@ -938,11 +960,23 @@ func (n *network) driver(load bool) (driverapi.Driver, error) {
 	return d, nil
 }
 
-func (n *network) Delete() error {
+func (n *network) Delete(options ...NetworkDeleteOption) error {
-	return n.delete(false)
+	var params networkDeleteParams
+	for _, opt := range options {
+		opt(&params)
+	}
+	return n.delete(false, params.rmLBEndpoint)
 }
 
-func (n *network) delete(force bool) error {
+// This function gets called in 3 ways:
+//  * Delete() -- (false, false)
+//      remove if endpoint count == 0 or endpoint count == 1 and
+//      there is a load balancer IP
+//  * Delete(libnetwork.NetworkDeleteOptionRemoveLB) -- (false, true)
+//      remove load balancer and network if endpoint count == 1
+//  * controller.networkCleanup() -- (true, true)
+//      remove the network no matter what
+func (n *network) delete(force bool, rmLBEndpoint bool) error {
 	n.Lock()
 	c := n.ctrlr
 	name := n.name
@@ -957,10 +991,32 @@ func (n *network) delete(force bool) error {
 		return &UnknownNetworkError{name: name, id: id}
 	}
 
-	if len(n.loadBalancerIP) != 0 {
+	// Only remove ingress on force removal or explicit LB endpoint removal
-		endpoints := n.Endpoints()
+	if n.ingress && !force && !rmLBEndpoint {
-		if force || (len(endpoints) == 1 && !n.ingress) {
+		return &ActiveEndpointsError{name: n.name, id: n.id}
-			n.deleteLoadBalancerSandbox()
+	}
+
+	// Check that the network is empty
+	var emptyCount uint64
+	if n.hasLoadBalancerEndpoint() {
+		emptyCount = 1
+	}
+	if !force && n.getEpCnt().EndpointCnt() > emptyCount {
+		if n.configOnly {
+			return types.ForbiddenErrorf("configuration network %q is in use", n.Name())
+		}
+		return &ActiveEndpointsError{name: n.name, id: n.id}
+	}
+
+	if n.hasLoadBalancerEndpoint() {
+		// If we got to this point, then the following must hold:
+		//  * force is true OR endpoint count == 1
+		if err := n.deleteLoadBalancerSandbox(); err != nil {
+			if !force {
+				return err
+			}
+			// continue deletion when force is true even on error
+			logrus.Warnf("Error deleting load balancer sandbox: %v", err)
 		}
 		//Reload the network from the store to update the epcnt.
 		n, err = c.getNetworkFromStore(id)
@@ -969,12 +1025,10 @@ func (n *network) delete(force bool) error {
 		}
 	}
 
-	if !force && n.getEpCnt().EndpointCnt() != 0 {
+	// Up to this point, errors that we returned were recoverable.
-		if n.configOnly {
+	// From here on, any errors leave us in an inconsistent state.
-			return types.ForbiddenErrorf("configuration network %q is in use", n.Name())
+	// This is unfortunate, but there isn't a safe way to
-		}
+	// reconstitute a load-balancer endpoint after removing it.
-		return &ActiveEndpointsError{name: n.name, id: n.id}
-	}
 
 	// Mark the network for deletion
 	n.inDelete = true
@@ -1023,9 +1077,6 @@ func (n *network) delete(force bool) error {
 	// Cleanup the service discovery for this network
 	c.cleanupServiceDiscovery(n.ID())
 
-	// Cleanup the load balancer
-	c.cleanupServiceBindings(n.ID())
-
 removeFromStore:
 	// deleteFromStore performs an atomic delete operation and the
 	// network.epCnt will help prevent any possible
@@ -1339,7 +1390,7 @@ func (n *network) addSvcRecords(eID, name, serviceID string, epIP, epIPv6 net.IP
 		return
 	}
 
-	logrus.Debugf("%s (%s).addSvcRecords(%s, %s, %s, %t) %s sid:%s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID)
+	logrus.Debugf("%s (%.7s).addSvcRecords(%s, %s, %s, %t) %s sid:%s", eID, n.ID(), name, epIP, epIPv6, ipMapUpdate, method, serviceID)
 
 	c := n.getController()
 	c.Lock()
@@ -1375,7 +1426,7 @@ func (n *network) deleteSvcRecords(eID, name, serviceID string, epIP net.IP, epI
 		return
 	}
 
-	logrus.Debugf("%s (%s).deleteSvcRecords(%s, %s, %s, %t) %s sid:%s ", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID)
+	logrus.Debugf("%s (%.7s).deleteSvcRecords(%s, %s, %s, %t) %s sid:%s ", eID, n.ID(), name, epIP, epIPv6, ipMapUpdate, method, serviceID)
 
 	c := n.getController()
 	c.Lock()
@@ -1877,6 +1928,10 @@ func (n *network) hasSpecialDriver() bool {
 	return n.Type() == "host" || n.Type() == "null"
 }
 
+func (n *network) hasLoadBalancerEndpoint() bool {
+	return len(n.loadBalancerIP) != 0
+}
+
 func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) {
 	var ipv6Miss bool
 
@@ -2056,9 +2111,22 @@ func (c *controller) getConfigNetwork(name string) (*network, error) {
 	return n.(*network), nil
 }
 
-func (n *network) createLoadBalancerSandbox() error {
+func (n *network) lbSandboxName() string {
-	sandboxName := n.name + "-sbox"
+	name := "lb-" + n.name
-	sbOptions := []SandboxOption{}
+	if n.ingress {
+		name = n.name + "-sbox"
+	}
+	return name
+}
+
+func (n *network) lbEndpointName() string {
+	return n.name + "-endpoint"
+}
+
+func (n *network) createLoadBalancerSandbox() (retErr error) {
+	sandboxName := n.lbSandboxName()
+	// Mark the sandbox to be a load balancer
+	sbOptions := []SandboxOption{OptionLoadBalancer()}
 	if n.ingress {
 		sbOptions = append(sbOptions, OptionIngress())
 	}
@@ -2067,26 +2135,30 @@ func (n *network) createLoadBalancerSandbox() error {
 		return err
 	}
 	defer func() {
-		if err != nil {
+		if retErr != nil {
 			if e := n.ctrlr.SandboxDestroy(sandboxName); e != nil {
-				logrus.Warnf("could not delete sandbox %s on failure on failure (%v): %v", sandboxName, err, e)
+				logrus.Warnf("could not delete sandbox %s on failure on failure (%v): %v", sandboxName, retErr, e)
 			}
 		}
 	}()
 
-	endpointName := n.name + "-endpoint"
+	endpointName := n.lbEndpointName()
 	epOptions := []EndpointOption{
 		CreateOptionIpam(n.loadBalancerIP, nil, nil, nil),
 		CreateOptionLoadBalancer(),
 	}
+	if n.hasLoadBalancerEndpoint() && !n.ingress {
+		// Mark LB endpoints as anonymous so they don't show up in DNS
+		epOptions = append(epOptions, CreateOptionAnonymous())
+	}
 	ep, err := n.createEndpoint(endpointName, epOptions...)
 	if err != nil {
 		return err
 	}
 	defer func() {
-		if err != nil {
+		if retErr != nil {
 			if e := ep.Delete(true); e != nil {
-				logrus.Warnf("could not delete endpoint %s on failure on failure (%v): %v", endpointName, err, e)
+				logrus.Warnf("could not delete endpoint %s on failure on failure (%v): %v", endpointName, retErr, e)
 			}
 		}
 	}()
@@ -2094,17 +2166,18 @@ func (n *network) createLoadBalancerSandbox() error {
 	if err := ep.Join(sb, nil); err != nil {
 		return err
 	}
+
 	return sb.EnableService()
 }
 
-func (n *network) deleteLoadBalancerSandbox() {
+func (n *network) deleteLoadBalancerSandbox() error {
 	n.Lock()
 	c := n.ctrlr
 	name := n.name
 	n.Unlock()
 
-	endpointName := name + "-endpoint"
+	sandboxName := n.lbSandboxName()
-	sandboxName := name + "-sbox"
+	endpointName := n.lbEndpointName()
 
 	endpoint, err := n.EndpointByName(endpointName)
 	if err != nil {
@@ -2129,6 +2202,7 @@ func (n *network) deleteLoadBalancerSandbox() {
 	}
 
 	if err := c.SandboxDestroy(sandboxName); err != nil {
-		logrus.Warnf("Failed to delete %s sandbox: %v", sandboxName, err)
+		return fmt.Errorf("Failed to delete %s sandbox: %v", sandboxName, err)
 	}
+	return nil
 }

vendor/github.com/docker/libnetwork/networkdb/broadcast.go

@@ -110,7 +110,6 @@ type tableEventMessage struct {
 	tname string
 	key   string
 	msg   []byte
-	node  string
 }
 
 func (m *tableEventMessage) Invalidates(other memberlist.Broadcast) bool {
@@ -168,7 +167,6 @@ func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname st
 		id:    nid,
 		tname: tname,
 		key:   key,
-		node:  nDB.config.NodeID,
 	})
 	return nil
 }

vendor/github.com/docker/libnetwork/networkdb/cluster.go

@@ -24,6 +24,9 @@ const (
 	retryInterval         = 1 * time.Second
 	nodeReapInterval      = 24 * time.Hour
 	nodeReapPeriod        = 2 * time.Hour
+	// considering a cluster with > 20 nodes and a drain speed of 100 msg/s
+	// the following is roughly 1 minute
+	maxQueueLenBroadcastOnSync = 500
 )
 
 type logWriter struct{}
@@ -52,7 +55,7 @@ func (l *logWriter) Write(p []byte) (int, error) {
 
 // SetKey adds a new key to the key ring
 func (nDB *NetworkDB) SetKey(key []byte) {
-	logrus.Debugf("Adding key %s", hex.EncodeToString(key)[0:5])
+	logrus.Debugf("Adding key %.5s", hex.EncodeToString(key))
 	nDB.Lock()
 	defer nDB.Unlock()
 	for _, dbKey := range nDB.config.Keys {
@@ -69,7 +72,7 @@ func (nDB *NetworkDB) SetKey(key []byte) {
 // SetPrimaryKey sets the given key as the primary key. This should have
 // been added apriori through SetKey
 func (nDB *NetworkDB) SetPrimaryKey(key []byte) {
-	logrus.Debugf("Primary Key %s", hex.EncodeToString(key)[0:5])
+	logrus.Debugf("Primary Key %.5s", hex.EncodeToString(key))
 	nDB.RLock()
 	defer nDB.RUnlock()
 	for _, dbKey := range nDB.config.Keys {
@@ -85,7 +88,7 @@ func (nDB *NetworkDB) SetPrimaryKey(key []byte) {
 // RemoveKey removes a key from the key ring. The key being removed
 // can't be the primary key
 func (nDB *NetworkDB) RemoveKey(key []byte) {
-	logrus.Debugf("Remove Key %s", hex.EncodeToString(key)[0:5])
+	logrus.Debugf("Remove Key %.5s", hex.EncodeToString(key))
 	nDB.Lock()
 	defer nDB.Unlock()
 	for i, dbKey := range nDB.config.Keys {
@@ -123,7 +126,7 @@ func (nDB *NetworkDB) clusterInit() error {
 	var err error
 	if len(nDB.config.Keys) > 0 {
 		for i, key := range nDB.config.Keys {
-			logrus.Debugf("Encryption key %d: %s", i+1, hex.EncodeToString(key)[0:5])
+			logrus.Debugf("Encryption key %d: %.5s", i+1, hex.EncodeToString(key))
 		}
 		nDB.keyring, err = memberlist.NewKeyring(nDB.config.Keys, nDB.config.Keys[0])
 		if err != nil {
@@ -285,18 +288,35 @@ func (nDB *NetworkDB) rejoinClusterBootStrap() {
 		return
 	}
 
+	myself, _ := nDB.nodes[nDB.config.NodeID]
 	bootStrapIPs := make([]string, 0, len(nDB.bootStrapIP))
 	for _, bootIP := range nDB.bootStrapIP {
-		for _, node := range nDB.nodes {
+		// botostrap IPs are usually IP:port from the Join
-			if node.Addr.Equal(bootIP) {
+		var bootstrapIP net.IP
-				// One of the bootstrap nodes is part of the cluster, return
+		ipStr, _, err := net.SplitHostPort(bootIP)
-				nDB.RUnlock()
+		if err != nil {
-				return
+			// try to parse it as an IP with port
-			}
+			// Note this seems to be the case for swarm that do not specify any port
+			ipStr = bootIP
+		}
+		bootstrapIP = net.ParseIP(ipStr)
+		if bootstrapIP != nil {
+			for _, node := range nDB.nodes {
+				if node.Addr.Equal(bootstrapIP) && !node.Addr.Equal(myself.Addr) {
+					// One of the bootstrap nodes (and not myself) is part of the cluster, return
+					nDB.RUnlock()
+					return
+				}
+			}
+			bootStrapIPs = append(bootStrapIPs, bootIP)
 		}
-		bootStrapIPs = append(bootStrapIPs, bootIP.String())
 	}
 	nDB.RUnlock()
+	if len(bootStrapIPs) == 0 {
+		// this will also avoid to call the Join with an empty list erasing the current bootstrap ip list
+		logrus.Debug("rejoinClusterBootStrap did not find any valid IP")
+		return
+	}
 	// None of the bootStrap nodes are in the cluster, call memberlist join
 	logrus.Debugf("rejoinClusterBootStrap, calling cluster join with bootStrap %v", bootStrapIPs)
 	ctx, cancel := context.WithTimeout(nDB.ctx, rejoinClusterDuration)
@@ -555,6 +575,7 @@ func (nDB *NetworkDB) bulkSync(nodes []string, all bool) ([]string, error) {
 
 	var err error
 	var networks []string
+	var success bool
 	for _, node := range nodes {
 		if node == nDB.config.NodeID {
 			continue
@@ -562,21 +583,25 @@ func (nDB *NetworkDB) bulkSync(nodes []string, all bool) ([]string, error) {
 		logrus.Debugf("%v(%v): Initiating bulk sync with node %v", nDB.config.Hostname, nDB.config.NodeID, node)
 		networks = nDB.findCommonNetworks(node)
 		err = nDB.bulkSyncNode(networks, node, true)
-		// if its periodic bulksync stop after the first successful sync
-		if !all && err == nil {
-			break
-		}
 		if err != nil {
 			err = fmt.Errorf("bulk sync to node %s failed: %v", node, err)
 			logrus.Warn(err.Error())
+		} else {
+			// bulk sync succeeded
+			success = true
+			// if its periodic bulksync stop after the first successful sync
+			if !all {
+				break
+			}
 		}
 	}
 
-	if err != nil {
+	if success {
-		return nil, err
+		// if at least one node sync succeeded
+		return networks, nil
 	}
 
-	return networks, nil
+	return nil, err
 }
 
 // Bulk sync all the table entries belonging to a set of networks to a

vendor/github.com/docker/libnetwork/networkdb/delegate.go

@@ -142,7 +142,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
 	return true
 }
 
-func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
+func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent, isBulkSync bool) bool {
 	// Update our local clock if the received messages has newer time.
 	nDB.tableClock.Witness(tEvent.LTime)
 
@@ -175,6 +175,14 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 			nDB.Unlock()
 			return false
 		}
+	} else if tEvent.Type == TableEventTypeDelete && !isBulkSync {
+		nDB.Unlock()
+		// We don't know the entry, the entry is being deleted and the message is an async message
+		// In this case the safest approach is to ignore it, it is possible that the queue grew so much to
+		// exceed the garbage collection time (the residual reap time that is in the message is not being
+		// updated, to avoid inserting too many messages in the queue).
+		// Instead the messages coming from TCP bulk sync are safe with the latest value for the garbage collection time
+		return false
 	}
 
 	e = &entry{
@@ -197,11 +205,17 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 	nDB.Unlock()
 
 	if err != nil && tEvent.Type == TableEventTypeDelete {
-		// If it is a delete event and we did not have a state for it, don't propagate to the application
+		// Again we don't know the entry but this is coming from a TCP sync so the message body is up to date.
+		// We had saved the state so to speed up convergence and be able to avoid accepting create events.
+		// Now we will rebroadcast the message if 2 conditions are met:
+		// 1) we had already synced this network (during the network join)
+		// 2) the residual reapTime is higher than 1/6 of the total reapTime.
 		// If the residual reapTime is lower or equal to 1/6 of the total reapTime don't bother broadcasting it around
-		// most likely the cluster is already aware of it, if not who will sync with this node will catch the state too.
+		// most likely the cluster is already aware of it
-		// This also avoids that deletion of entries close to their garbage collection ends up circuling around forever
+		// This also reduce the possibility that deletion of entries close to their garbage collection ends up circuling around
-		return e.reapTime > nDB.config.reapEntryInterval/6
+		// forever
+		//logrus.Infof("exiting on delete not knowing the obj with rebroadcast:%t", network.inSync)
+		return network.inSync && e.reapTime > nDB.config.reapEntryInterval/6
 	}
 
 	var op opType
@@ -215,7 +229,7 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 	}
 
 	nDB.broadcaster.Write(makeEvent(op, tEvent.TableName, tEvent.NetworkID, tEvent.Key, tEvent.Value))
-	return true
+	return network.inSync
 }
 
 func (nDB *NetworkDB) handleCompound(buf []byte, isBulkSync bool) {
@@ -244,7 +258,7 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
 		return
 	}
 
-	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast {
+	if rebroadcast := nDB.handleTableEvent(&tEvent, isBulkSync); rebroadcast {
 		var err error
 		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)
 		if err != nil {
@@ -261,12 +275,16 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
 			return
 		}
 
+		// if the queue is over the threshold, avoid distributing information coming from TCP sync
+		if isBulkSync && n.tableBroadcasts.NumQueued() > maxQueueLenBroadcastOnSync {
+			return
+		}
+
 		n.tableBroadcasts.QueueBroadcast(&tableEventMessage{
 			msg:   buf,
 			id:    tEvent.NetworkID,
 			tname: tEvent.TableName,
 			key:   tEvent.Key,
-			node:  tEvent.NodeName,
 		})
 	}
 }

vendor/github.com/docker/libnetwork/networkdb/networkdb.go

@@ -5,7 +5,6 @@ package networkdb
 import (
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"strings"
 	"sync"
@@ -96,7 +95,7 @@ type NetworkDB struct {
 
 	// bootStrapIP is the list of IPs that can be used to bootstrap
 	// the gossip.
-	bootStrapIP []net.IP
+	bootStrapIP []string
 
 	// lastStatsTimestamp is the last timestamp when the stats got printed
 	lastStatsTimestamp time.Time
@@ -131,6 +130,9 @@ type network struct {
 	// Lamport time for the latest state of the entry.
 	ltime serf.LamportTime
 
+	// Gets set to true after the first bulk sync happens
+	inSync bool
+
 	// Node leave is in progress.
 	leaving bool
 
@@ -268,10 +270,8 @@ func New(c *Config) (*NetworkDB, error) {
 // instances passed by the caller in the form of addr:port
 func (nDB *NetworkDB) Join(members []string) error {
 	nDB.Lock()
-	nDB.bootStrapIP = make([]net.IP, 0, len(members))
+	nDB.bootStrapIP = append([]string(nil), members...)
-	for _, m := range members {
+	logrus.Infof("The new bootstrap node list is:%v", nDB.bootStrapIP)
-		nDB.bootStrapIP = append(nDB.bootStrapIP, net.ParseIP(m))
-	}
 	nDB.Unlock()
 	return nDB.clusterJoin(members)
 }
@@ -619,6 +619,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 	}
 	nDB.addNetworkNode(nid, nDB.config.NodeID)
 	networkNodes := nDB.networkNodes[nid]
+	n = nodeNetworks[nid]
 	nDB.Unlock()
 
 	if err := nDB.sendNetworkEvent(nid, NetworkEventTypeJoin, ltime); err != nil {
@@ -630,6 +631,12 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 		logrus.Errorf("Error bulk syncing while joining network %s: %v", nid, err)
 	}
 
+	// Mark the network as being synced
+	// note this is a best effort, we are not checking the result of the bulk sync
+	nDB.Lock()
+	n.inSync = true
+	nDB.Unlock()
+
 	return nil
 }
 

vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go

@@ -1,11 +1,11 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: networkdb.proto
+// source: networkdb/networkdb.proto
 
 /*
 	Package networkdb is a generated protocol buffer package.
 
 	It is generated from these files:
-		networkdb.proto
+		networkdb/networkdb.proto
 
 	It has these top-level messages:
 		GossipMessage
@@ -476,7 +476,7 @@ func (m *CompoundMessage) GetMessages() []*CompoundMessage_SimpleMessage {
 type CompoundMessage_SimpleMessage struct {
 	// Bytestring payload of a message constructed using
 	// other message type definitions.
-	Payload []byte `protobuf:"bytes,1,opt,name=Payload,proto3" json:"Payload,omitempty"`
+	Payload []byte `protobuf:"bytes,1,opt,name=Payload,json=payload,proto3" json:"Payload,omitempty"`
 }
 
 func (m *CompoundMessage_SimpleMessage) Reset()      { *m = CompoundMessage_SimpleMessage{} }
@@ -997,24 +997,6 @@ func (m *CompoundMessage_SimpleMessage) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Networkdb(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Networkdb(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintNetworkdb(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -2666,68 +2648,68 @@ var (
 	ErrIntOverflowNetworkdb   = fmt.Errorf("proto: integer overflow")
 )
 
-func init() { proto.RegisterFile("networkdb.proto", fileDescriptorNetworkdb) }
+func init() { proto.RegisterFile("networkdb/networkdb.proto", fileDescriptorNetworkdb) }
 
 var fileDescriptorNetworkdb = []byte{
-	// 953 bytes of a gzipped FileDescriptorProto
+	// 955 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x96, 0xcd, 0x6e, 0xe3, 0x54,
-	0x14, 0xc7, 0x7b, 0xf3, 0xd5, 0xe4, 0x34, 0xa5, 0xe6, 0x4e, 0x67, 0xc6, 0xe3, 0x81, 0xc4, 0x98,
+	0x14, 0xc7, 0x7b, 0xf3, 0xd1, 0x26, 0xa7, 0x29, 0x35, 0x77, 0x3a, 0x53, 0xd7, 0x03, 0x89, 0x31,
-	0x99, 0x2a, 0x53, 0x41, 0x8a, 0x3a, 0x4f, 0xd0, 0x24, 0x16, 0x64, 0x26, 0xe3, 0x44, 0x6e, 0x52,
+	0x33, 0x55, 0xa6, 0x82, 0x14, 0x75, 0x9e, 0xa0, 0x49, 0x2c, 0xc8, 0x4c, 0xc6, 0x89, 0xdc, 0xa4,
-	0xc4, 0x2a, 0xba, 0xad, 0x2f, 0xa9, 0x55, 0xc7, 0xb6, 0x6c, 0x27, 0x28, 0x2b, 0x10, 0xab, 0x51,
+	0x88, 0x55, 0x74, 0x5b, 0x5f, 0x52, 0xab, 0x8e, 0x6d, 0xd9, 0x4e, 0x50, 0x56, 0x20, 0x56, 0xa3,
-	0x16, 0xbc, 0x41, 0x56, 0xc3, 0x9a, 0x07, 0x40, 0x2c, 0x59, 0xcc, 0x82, 0x05, 0xec, 0x10, 0x8b,
+	0x2c, 0x78, 0x83, 0xac, 0x86, 0x35, 0x0f, 0x80, 0x58, 0xb2, 0x98, 0x05, 0x0b, 0xd8, 0x21, 0x16,
-	0x88, 0xe6, 0x09, 0x78, 0x04, 0xe4, 0x6b, 0x3b, 0xb9, 0x49, 0xab, 0x91, 0x10, 0x23, 0xc1, 0x26,
+	0x11, 0xcd, 0x13, 0xf0, 0x08, 0xc8, 0xd7, 0x76, 0x72, 0x93, 0x56, 0x23, 0x21, 0x46, 0x82, 0x4d,
-	0xb9, 0x1f, 0xbf, 0x1c, 0x9f, 0xf3, 0xf7, 0xff, 0xdc, 0x1b, 0xd8, 0xb3, 0x69, 0xf0, 0x95, 0xe3,
+	0x72, 0x3f, 0x7e, 0x39, 0x3e, 0xe7, 0xef, 0xff, 0xb9, 0x37, 0x70, 0x60, 0xd3, 0xe0, 0x2b, 0xc7,
-	0x5d, 0x19, 0xe7, 0x55, 0xd7, 0x73, 0x02, 0x07, 0x17, 0x96, 0x0b, 0xd2, 0xfe, 0xc0, 0x19, 0x38,
+	0xbb, 0x36, 0x2e, 0x8e, 0x17, 0xa3, 0x8a, 0xeb, 0x39, 0x81, 0x83, 0xf3, 0x8b, 0x05, 0x69, 0xaf,
-	0x6c, 0xf5, 0x28, 0x1c, 0x45, 0x80, 0xd2, 0x86, 0xdd, 0x4f, 0x1d, 0xdf, 0x37, 0xdd, 0x17, 0xd4,
+	0xef, 0xf4, 0x1d, 0xb6, 0x7a, 0x1c, 0x8e, 0x22, 0x40, 0x69, 0xc1, 0xce, 0xa7, 0x8e, 0xef, 0x9b,
-	0xf7, 0xc9, 0x80, 0xe2, 0x43, 0xc8, 0x04, 0x13, 0x97, 0x8a, 0x48, 0x46, 0x95, 0x77, 0x8e, 0xef,
+	0xee, 0x0b, 0xea, 0xfb, 0xa4, 0x4f, 0xf1, 0x11, 0x64, 0x82, 0xb1, 0x4b, 0x45, 0x24, 0xa3, 0xf2,
-	0x55, 0x57, 0x11, 0x63, 0xa2, 0x3b, 0x71, 0xa9, 0xce, 0x18, 0x8c, 0x21, 0x63, 0x90, 0x80, 0x88,
+	0x3b, 0x27, 0x0f, 0x2a, 0xcb, 0x88, 0x31, 0xd1, 0x19, 0xbb, 0x54, 0x67, 0x0c, 0xc6, 0x90, 0x31,
-	0x29, 0x19, 0x55, 0x8a, 0x3a, 0x1b, 0x2b, 0xaf, 0x52, 0x50, 0xd0, 0x1c, 0x83, 0xaa, 0x63, 0x6a,
+	0x48, 0x40, 0xc4, 0x94, 0x8c, 0xca, 0x05, 0x9d, 0x8d, 0x95, 0x57, 0x29, 0xc8, 0x6b, 0x8e, 0x41,
-	0x07, 0xf8, 0xe3, 0xb5, 0x68, 0x0f, 0xb8, 0x68, 0x4b, 0xa6, 0xca, 0x05, 0x6c, 0x42, 0xce, 0xea,
+	0xd5, 0x11, 0xb5, 0x03, 0xfc, 0xf1, 0x4a, 0xb4, 0x03, 0x2e, 0xda, 0x82, 0xa9, 0x70, 0x01, 0x1b,
-	0x07, 0xe6, 0x90, 0xb2, 0x90, 0x99, 0xda, 0xf1, 0xeb, 0x79, 0x79, 0xeb, 0x8f, 0x79, 0xf9, 0x70,
+	0xb0, 0x69, 0xf5, 0x02, 0x73, 0x40, 0x59, 0xc8, 0x4c, 0xf5, 0xe4, 0xf5, 0xac, 0xb4, 0xf1, 0xc7,
-	0x60, 0x06, 0x97, 0xa3, 0xf3, 0xea, 0x85, 0x33, 0x3c, 0xba, 0x24, 0xfe, 0xa5, 0x79, 0xe1, 0x78,
+	0xac, 0x74, 0xd4, 0x37, 0x83, 0xab, 0xe1, 0x45, 0xe5, 0xd2, 0x19, 0x1c, 0x5f, 0x11, 0xff, 0xca,
-	0xee, 0x91, 0x4f, 0xbd, 0x2f, 0xd9, 0x47, 0xb5, 0x45, 0x86, 0xae, 0xe3, 0x05, 0x5d, 0x73, 0x48,
+	0xbc, 0x74, 0x3c, 0xf7, 0xd8, 0xa7, 0xde, 0x97, 0xec, 0xa3, 0xd2, 0x24, 0x03, 0xd7, 0xf1, 0x82,
-	0xf5, 0xac, 0x15, 0x7e, 0xe1, 0x87, 0x50, 0xb0, 0x1d, 0x83, 0xf6, 0x6d, 0x32, 0xa4, 0x62, 0x5a,
+	0x8e, 0x39, 0xa0, 0x7a, 0xd6, 0x0a, 0xbf, 0xf0, 0x43, 0xc8, 0xdb, 0x8e, 0x41, 0x7b, 0x36, 0x19,
-	0x46, 0x95, 0x82, 0x9e, 0x0f, 0x17, 0x34, 0x32, 0xa4, 0xca, 0xd7, 0x90, 0x09, 0x9f, 0x8a, 0x1f,
+	0x50, 0x31, 0x2d, 0xa3, 0x72, 0x5e, 0xcf, 0x85, 0x0b, 0x1a, 0x19, 0x50, 0xe5, 0x6b, 0xc8, 0x84,
-	0xc3, 0x76, 0x53, 0x3b, 0x3b, 0x69, 0x35, 0x1b, 0xc2, 0x96, 0x24, 0x4e, 0x67, 0xf2, 0xfe, 0x32,
+	0x4f, 0xc5, 0x8f, 0x61, 0xab, 0xa1, 0x9d, 0x9f, 0x36, 0x1b, 0x75, 0x61, 0x43, 0x12, 0x27, 0x53,
-	0xad, 0x70, 0xbf, 0x69, 0x8f, 0x89, 0x65, 0x1a, 0xb8, 0x0c, 0x99, 0x67, 0xed, 0xa6, 0x26, 0x20,
+	0x79, 0x6f, 0x91, 0x56, 0xb8, 0xdf, 0xb0, 0x47, 0xc4, 0x32, 0x0d, 0x5c, 0x82, 0xcc, 0xb3, 0x56,
-	0xe9, 0xee, 0x74, 0x26, 0xbf, 0xbb, 0xc6, 0x3c, 0x73, 0x4c, 0x1b, 0x7f, 0x00, 0xd9, 0x96, 0x7a,
+	0x43, 0x13, 0x90, 0x74, 0x7f, 0x32, 0x95, 0xdf, 0x5d, 0x61, 0x9e, 0x39, 0xa6, 0x8d, 0x3f, 0x80,
-	0x72, 0xa6, 0x0a, 0x29, 0xe9, 0xde, 0x74, 0x26, 0xe3, 0x35, 0xa2, 0x45, 0xc9, 0x98, 0x4a, 0xc5,
+	0x6c, 0x53, 0x3d, 0x3d, 0x57, 0x85, 0x94, 0xf4, 0x60, 0x32, 0x95, 0xf1, 0x0a, 0xd1, 0xa4, 0x64,
-	0x97, 0xaf, 0x4a, 0x5b, 0x3f, 0x7e, 0x5f, 0x62, 0x0f, 0x56, 0xae, 0x53, 0x50, 0xd4, 0x22, 0x2d,
+	0x44, 0xa5, 0xc2, 0xcb, 0x57, 0xc5, 0x8d, 0x1f, 0xbf, 0x2f, 0xb2, 0x07, 0x2b, 0x37, 0x29, 0x28,
-	0x22, 0xa1, 0x3e, 0x59, 0x13, 0xea, 0x3d, 0x5e, 0x28, 0x0e, 0xfb, 0x0f, 0xb4, 0xc2, 0x1f, 0x01,
+	0x68, 0x91, 0x16, 0x91, 0x50, 0x9f, 0xac, 0x08, 0xf5, 0x1e, 0x2f, 0x14, 0x87, 0xfd, 0x07, 0x5a,
-	0xc4, 0xc9, 0xf4, 0x4d, 0x43, 0xcc, 0x84, 0xbb, 0xb5, 0xdd, 0xc5, 0xbc, 0x5c, 0x88, 0x13, 0x6b,
+	0xe1, 0x8f, 0x00, 0xe2, 0x64, 0x7a, 0xa6, 0x21, 0x66, 0xc2, 0xdd, 0xea, 0xce, 0x7c, 0x56, 0xca,
-	0x36, 0xf4, 0xc4, 0x65, 0x4d, 0x43, 0x79, 0x89, 0x62, 0x69, 0x2b, 0xbc, 0xb4, 0x0f, 0xa7, 0x33,
+	0xc7, 0x89, 0x35, 0xea, 0x7a, 0xe2, 0xb2, 0x86, 0xa1, 0xbc, 0x44, 0xb1, 0xb4, 0x65, 0x5e, 0xda,
-	0xf9, 0x3e, 0x5f, 0x08, 0xaf, 0xae, 0xb2, 0x54, 0x37, 0x7a, 0x03, 0x1b, 0x18, 0x13, 0xf8, 0xd1,
+	0x87, 0x93, 0xa9, 0xbc, 0xcf, 0x17, 0xc2, 0xab, 0xab, 0x2c, 0xd4, 0x8d, 0xde, 0xc0, 0x1a, 0xc6,
-	0x4a, 0xe0, 0x07, 0xd3, 0x99, 0x7c, 0x77, 0x13, 0xba, 0x4d, 0xe3, 0x5f, 0xd0, 0x4a, 0x63, 0x3b,
+	0x04, 0x7e, 0xb4, 0x14, 0xf8, 0x60, 0x32, 0x95, 0xef, 0xaf, 0x43, 0x77, 0x69, 0xfc, 0x0b, 0x5a,
-	0xf0, 0x26, 0x1b, 0x95, 0xa0, 0x37, 0x57, 0xf2, 0x36, 0xf5, 0x7d, 0x72, 0x43, 0xdf, 0x5a, 0x71,
+	0x6a, 0x6c, 0x07, 0xde, 0x78, 0xad, 0x12, 0xf4, 0xe6, 0x4a, 0xde, 0xa6, 0xbe, 0x4f, 0x6e, 0xe9,
-	0x31, 0x2f, 0xe7, 0xb5, 0x58, 0x63, 0x4e, 0x6d, 0x11, 0xb6, 0x2d, 0x4a, 0xc6, 0xa6, 0x3d, 0x60,
+	0x5b, 0x2d, 0xcc, 0x67, 0xa5, 0x9c, 0x16, 0x6b, 0xcc, 0xa9, 0x2d, 0xc2, 0x96, 0x45, 0xc9, 0xc8,
-	0x52, 0xe7, 0xf5, 0x64, 0xaa, 0xfc, 0x84, 0x60, 0x2f, 0x4e, 0xb4, 0x33, 0xf2, 0x2f, 0x3b, 0x23,
+	0xb4, 0xfb, 0x4c, 0xea, 0x9c, 0x9e, 0x4c, 0x95, 0x9f, 0x10, 0xec, 0xc6, 0x89, 0xb6, 0x87, 0xfe,
-	0xcb, 0xe2, 0x72, 0x44, 0xff, 0x36, 0xc7, 0xa7, 0x90, 0x8f, 0x6b, 0xf7, 0xc5, 0x94, 0x9c, 0xae,
+	0x55, 0x7b, 0x68, 0x59, 0x5c, 0x8e, 0xe8, 0xdf, 0xe6, 0xf8, 0x14, 0x72, 0x71, 0xed, 0xbe, 0x98,
-	0xec, 0x1c, 0xdf, 0xbf, 0xc5, 0x84, 0xa1, 0x8e, 0xfa, 0x12, 0xfc, 0x07, 0x85, 0x29, 0xdf, 0x65,
+	0x92, 0xd3, 0xe5, 0xed, 0x93, 0xfd, 0x3b, 0x4c, 0x18, 0xea, 0xa8, 0x2f, 0xc0, 0x7f, 0x50, 0x98,
-	0x00, 0xba, 0xe4, 0xdc, 0x8a, 0x0f, 0x86, 0xea, 0x9a, 0xdf, 0x25, 0xee, 0x51, 0x2b, 0xe8, 0x7f,
+	0xf2, 0x5d, 0x06, 0xa0, 0x43, 0x2e, 0xac, 0xf8, 0x60, 0xa8, 0xac, 0xf8, 0x5d, 0xe2, 0x1e, 0xb5,
-	0xef, 0x76, 0xfc, 0x3e, 0x40, 0x10, 0xa6, 0x1b, 0xc5, 0xca, 0xb2, 0x58, 0x05, 0xb6, 0xc2, 0x82,
+	0x84, 0xfe, 0xf7, 0x6e, 0xc7, 0xef, 0x03, 0x04, 0x61, 0xba, 0x51, 0xac, 0x2c, 0x8b, 0x95, 0x67,
-	0x09, 0x90, 0xbe, 0xa2, 0x13, 0x31, 0xc7, 0xd6, 0xc3, 0x21, 0xde, 0x87, 0xec, 0x98, 0x58, 0x23,
+	0x2b, 0x2c, 0x98, 0x00, 0xe9, 0x6b, 0x3a, 0x16, 0x37, 0xd9, 0x7a, 0x38, 0xc4, 0x7b, 0x90, 0x1d,
-	0x2a, 0x6e, 0xb3, 0x23, 0x33, 0x9a, 0xe0, 0x1a, 0x60, 0x8f, 0xfa, 0xa6, 0x31, 0x22, 0x56, 0xdf,
+	0x11, 0x6b, 0x48, 0xc5, 0x2d, 0x76, 0x64, 0x46, 0x13, 0x5c, 0x05, 0xec, 0x51, 0xdf, 0x34, 0x86,
-	0xa3, 0xc4, 0x8d, 0x0a, 0xcd, 0xcb, 0xa8, 0x92, 0xad, 0xed, 0x2f, 0xe6, 0x65, 0x41, 0x8f, 0x77,
+	0xc4, 0xea, 0x79, 0x94, 0xb8, 0x51, 0xa1, 0x39, 0x19, 0x95, 0xb3, 0xd5, 0xbd, 0xf9, 0xac, 0x24,
-	0x75, 0x4a, 0x5c, 0x56, 0x8a, 0xe0, 0x6d, 0xac, 0x28, 0x3f, 0x24, 0x8d, 0x77, 0xc0, 0x37, 0x1e,
+	0xe8, 0xf1, 0xae, 0x4e, 0x89, 0xcb, 0x4a, 0x11, 0xbc, 0xb5, 0x15, 0xe5, 0x87, 0xa4, 0xf1, 0x0e,
-	0x6b, 0x96, 0x95, 0xa2, 0x7c, 0xdb, 0x3d, 0x82, 0x5c, 0x5d, 0x57, 0x4f, 0xba, 0x6a, 0xd2, 0x78,
+	0xf9, 0xc6, 0x63, 0xcd, 0xb2, 0x54, 0x94, 0x6f, 0xbb, 0x47, 0xb0, 0x59, 0xd3, 0xd5, 0xd3, 0x8e,
-	0xeb, 0x58, 0xdd, 0xa3, 0x24, 0xa0, 0x21, 0xd5, 0xeb, 0x34, 0x42, 0x2a, 0x75, 0x1b, 0xd5, 0x73,
+	0x9a, 0x34, 0xde, 0x2a, 0x56, 0xf3, 0x28, 0x09, 0x68, 0x48, 0x75, 0xdb, 0xf5, 0x90, 0x4a, 0xdd,
-	0x8d, 0x98, 0x6a, 0xa8, 0x2d, 0xb5, 0xab, 0x0a, 0xe9, 0xdb, 0xa8, 0x06, 0xb5, 0x68, 0xb0, 0xd9,
+	0x45, 0x75, 0x5d, 0x23, 0xa6, 0xea, 0x6a, 0x53, 0xed, 0xa8, 0x42, 0xfa, 0x2e, 0xaa, 0x4e, 0x2d,
-	0x9e, 0xbf, 0x21, 0xd8, 0xab, 0x8d, 0xac, 0xab, 0xd3, 0x89, 0x7d, 0x91, 0x5c, 0x3e, 0x6f, 0xd1,
+	0x1a, 0xac, 0xb7, 0xe7, 0x6f, 0x08, 0x76, 0xab, 0x43, 0xeb, 0xfa, 0x6c, 0x6c, 0x5f, 0x26, 0x97,
-	0xcf, 0x32, 0xec, 0x8c, 0x6c, 0xdf, 0xb1, 0xcc, 0x0b, 0x33, 0xa0, 0x06, 0x73, 0x4d, 0x5e, 0xe7,
+	0xcf, 0x5b, 0xf4, 0xb3, 0x0c, 0xdb, 0x43, 0xdb, 0x77, 0x2c, 0xf3, 0xd2, 0x0c, 0xa8, 0xc1, 0x5c,
-	0x97, 0xde, 0xec, 0x03, 0x89, 0x6b, 0x87, 0x8c, 0x9c, 0x66, 0x7b, 0x89, 0xeb, 0x45, 0xd8, 0x76,
+	0x93, 0xd3, 0xf9, 0xa5, 0x37, 0xfb, 0x40, 0xe2, 0xda, 0x21, 0x23, 0xa7, 0xd9, 0x5e, 0xe2, 0x7a,
-	0xc9, 0xc4, 0x72, 0x88, 0xc1, 0x5e, 0x79, 0x51, 0x4f, 0xa6, 0xca, 0xb7, 0x08, 0xf6, 0xea, 0xce,
+	0x11, 0xb6, 0x5c, 0x32, 0xb6, 0x1c, 0x62, 0xb0, 0x57, 0x5e, 0xd0, 0x93, 0xa9, 0xf2, 0x2d, 0x82,
-	0xd0, 0x75, 0x46, 0xb6, 0x91, 0xd4, 0xd4, 0x80, 0xfc, 0x30, 0x1a, 0xfa, 0x22, 0x62, 0x8d, 0x55,
+	0xdd, 0x9a, 0x33, 0x70, 0x9d, 0xa1, 0x6d, 0x24, 0x35, 0xd5, 0x21, 0x37, 0x88, 0x86, 0xbe, 0x88,
-	0xe1, 0xdc, 0xbe, 0x41, 0x57, 0x4f, 0xcd, 0xa1, 0x6b, 0xd1, 0x78, 0xa6, 0x2f, 0x7f, 0x29, 0x3d,
+	0x58, 0x63, 0x95, 0x39, 0xb7, 0xaf, 0xd1, 0x95, 0x33, 0x73, 0xe0, 0x5a, 0x34, 0x9e, 0xe9, 0x8b,
-	0x81, 0xdd, 0xb5, 0xad, 0x30, 0x89, 0x4e, 0x9c, 0x04, 0x8a, 0x92, 0x88, 0xa7, 0x87, 0x3f, 0xa7,
+	0x5f, 0x4a, 0x4f, 0x60, 0x67, 0x65, 0x2b, 0x4c, 0xa2, 0x1d, 0x27, 0x81, 0x56, 0x92, 0x38, 0xfa,
-	0x60, 0x87, 0xbb, 0xab, 0xf1, 0x87, 0xbc, 0x21, 0xd8, 0xf5, 0xc4, 0xed, 0x26, 0x6e, 0xa8, 0xc2,
+	0x39, 0x05, 0xdb, 0xdc, 0x5d, 0x8d, 0x3f, 0xe4, 0x0d, 0xc1, 0xae, 0x27, 0x6e, 0x37, 0x71, 0x43,
-	0xae, 0xa6, 0x76, 0x3f, 0x6f, 0xeb, 0xcf, 0xfb, 0xea, 0x99, 0xaa, 0x75, 0x05, 0x14, 0x1d, 0xda,
+	0x05, 0x76, 0x34, 0xb5, 0xf3, 0x79, 0x4b, 0x7f, 0xde, 0x53, 0xcf, 0x55, 0xad, 0x23, 0xa0, 0xe8,
-	0x1c, 0xba, 0x76, 0x5f, 0x1d, 0xc2, 0x4e, 0xf7, 0xa4, 0xd6, 0x52, 0x63, 0x3a, 0x3e, 0x96, 0x39,
+	0xd0, 0xe6, 0xd0, 0x95, 0xfb, 0xea, 0x08, 0xb6, 0x3b, 0xa7, 0xd5, 0xa6, 0x1a, 0xd3, 0xf1, 0xb1,
-	0x9a, 0xeb, 0xf5, 0x03, 0x28, 0x74, 0x7a, 0xa7, 0x9f, 0xf5, 0x3b, 0xbd, 0x56, 0x4b, 0x48, 0x4b,
+	0xcc, 0xd1, 0x5c, 0xaf, 0x1f, 0x42, 0xbe, 0xdd, 0x3d, 0xfb, 0xac, 0xd7, 0xee, 0x36, 0x9b, 0x42,
-	0xf7, 0xa7, 0x33, 0xf9, 0x0e, 0x47, 0x2e, 0x4f, 0xb3, 0x03, 0x28, 0xd4, 0x7a, 0xad, 0xe7, 0xfd,
+	0x5a, 0xda, 0x9f, 0x4c, 0xe5, 0x7b, 0x1c, 0xb9, 0x38, 0xcd, 0x0e, 0x21, 0x5f, 0xed, 0x36, 0x9f,
-	0xd3, 0x2f, 0xb4, 0xba, 0x90, 0xb9, 0xc1, 0x25, 0x66, 0xc1, 0x8f, 0x21, 0x5f, 0x6f, 0xbf, 0xe8,
+	0xf7, 0xce, 0xbe, 0xd0, 0x6a, 0x42, 0xe6, 0x16, 0x97, 0x98, 0x05, 0x3f, 0x86, 0x5c, 0xad, 0xf5,
-	0xb4, 0x7b, 0x5a, 0x43, 0xc8, 0xde, 0xc0, 0x12, 0x45, 0x71, 0x05, 0x40, 0x6b, 0x37, 0x92, 0x0c,
+	0xa2, 0xdd, 0xea, 0x6a, 0x75, 0x21, 0x7b, 0x0b, 0x4b, 0x14, 0xc5, 0x65, 0x00, 0xad, 0x55, 0x4f,
-	0x73, 0x91, 0x31, 0xf9, 0x7a, 0x92, 0x4b, 0x5a, 0xba, 0x13, 0x1b, 0x93, 0x97, 0xad, 0x26, 0xfe,
+	0x32, 0xdc, 0x8c, 0x8c, 0xc9, 0xd7, 0x93, 0x5c, 0xd2, 0xd2, 0xbd, 0xd8, 0x98, 0xbc, 0x6c, 0x55,
-	0x7e, 0x5d, 0xda, 0xfa, 0xeb, 0xba, 0x84, 0xbe, 0x59, 0x94, 0xd0, 0xeb, 0x45, 0x09, 0xfd, 0xba,
+	0xf1, 0xf7, 0x9b, 0xe2, 0xc6, 0x5f, 0x37, 0x45, 0xf4, 0xcd, 0xbc, 0x88, 0x5e, 0xcf, 0x8b, 0xe8,
-	0x28, 0xa1, 0x3f, 0x17, 0x25, 0x74, 0x9e, 0x63, 0x7f, 0x9d, 0x9e, 0xfe, 0x1d, 0x00, 0x00, 0xff,
+	0xd7, 0x79, 0x11, 0xfd, 0x39, 0x2f, 0xa2, 0x8b, 0x4d, 0xf6, 0xd7, 0xe9, 0xe9, 0xdf, 0x01, 0x00,
-	0xff, 0x92, 0x82, 0xdb, 0x1a, 0x6e, 0x09, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x02, 0x9d, 0x53, 0x72, 0x78, 0x09, 0x00, 0x00,
 }

vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnostic.go

@@ -28,6 +28,7 @@ var NetDbPaths2Func = map[string]diagnostic.HTTPHandlerFunc{
 	"/deleteentry":  dbDeleteEntry,
 	"/getentry":     dbGetEntry,
 	"/gettable":     dbGetTable,
+	"/networkstats": dbNetworkStats,
 }
 
 func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) {
@@ -411,3 +412,41 @@ func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) {
 	}
 	diagnostic.HTTPReply(w, diagnostic.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
 }
+
+func dbNetworkStats(ctx interface{}, w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	diagnostic.DebugHTTPForm(r)
+	_, json := diagnostic.ParseHTTPFormOptions(r)
+
+	// audit logs
+	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
+	log.Info("network stats")
+
+	if len(r.Form["nid"]) < 1 {
+		rsp := diagnostic.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path))
+		log.Error("network stats failed, wrong input")
+		diagnostic.HTTPReply(w, rsp, json)
+		return
+	}
+
+	nDB, ok := ctx.(*NetworkDB)
+	if ok {
+		nDB.RLock()
+		networks := nDB.networks[nDB.config.NodeID]
+		network, ok := networks[r.Form["nid"][0]]
+
+		entries := -1
+		qLen := -1
+		if ok {
+			entries = network.entriesNumber
+			qLen = network.tableBroadcasts.NumQueued()
+		}
+		nDB.RUnlock()
+
+		rsp := diagnostic.CommandSucceed(&diagnostic.NetworkStatsResult{Entries: entries, QueueLen: qLen})
+		log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network stats done")
+		diagnostic.HTTPReply(w, rsp, json)
+		return
+	}
+	diagnostic.HTTPReply(w, diagnostic.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
+}

vendor/github.com/docker/libnetwork/osl/kernel/knobs.go

@@ -0,0 +1,16 @@
+package kernel
+
+type conditionalCheck func(val1, val2 string) bool
+
+// OSValue represents a tuple, value defired, check function when to apply the value
+type OSValue struct {
+	Value   string
+	CheckFn conditionalCheck
+}
+
+func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
+	if check == nil || check(val1, val2) {
+		return true
+	}
+	return false
+}

vendor/github.com/docker/libnetwork/osl/kernel/knobs_linux.go

@@ -0,0 +1,47 @@
+package kernel
+
+import (
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
+// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
+func writeSystemProperty(key, value string) error {
+	keyPath := strings.Replace(key, ".", "/", -1)
+	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
+}
+
+// readSystemProperty reads the value from the path under /proc/sys and returns it
+func readSystemProperty(key string) (string, error) {
+	keyPath := strings.Replace(key, ".", "/", -1)
+	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(value)), nil
+}
+
+// ApplyOSTweaks applies the configuration values passed as arguments
+func ApplyOSTweaks(osConfig map[string]*OSValue) {
+	for k, v := range osConfig {
+		// read the existing property from disk
+		oldv, err := readSystemProperty(k)
+		if err != nil {
+			logrus.WithError(err).Errorf("error reading the kernel parameter %s", k)
+			continue
+		}
+
+		if propertyIsValid(oldv, v.Value, v.CheckFn) {
+			// write new prop value to disk
+			if err := writeSystemProperty(k, v.Value); err != nil {
+				logrus.WithError(err).Errorf("error setting the kernel parameter %s = %s, (leaving as %s)", k, v.Value, oldv)
+				continue
+			}
+			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.Value, oldv)
+		}
+	}
+}

vendor/github.com/docker/libnetwork/osl/kernel/knobs_unsupported.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package kernel
+
+// ApplyOSTweaks applies the configuration values passed as arguments
+func ApplyOSTweaks(osConfig map[string]*OSValue) {
+}

vendor/github.com/docker/libnetwork/osl/namespace_linux.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/libnetwork/ns"
+	"github.com/docker/libnetwork/osl/kernel"
 	"github.com/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -29,13 +30,18 @@ func init() {
 }
 
 var (
-	once             sync.Once
+	once               sync.Once
-	garbagePathMap   = make(map[string]bool)
+	garbagePathMap     = make(map[string]bool)
-	gpmLock          sync.Mutex
+	gpmLock            sync.Mutex
-	gpmWg            sync.WaitGroup
+	gpmWg              sync.WaitGroup
-	gpmCleanupPeriod = 60 * time.Second
+	gpmCleanupPeriod   = 60 * time.Second
-	gpmChan          = make(chan chan struct{})
+	gpmChan            = make(chan chan struct{})
-	prefix           = defaultPrefix
+	prefix             = defaultPrefix
+	loadBalancerConfig = map[string]*kernel.OSValue{
+		// expires connection from the IPVS connection table when the backend is not available
+		// more info: https://github.com/torvalds/linux/blob/master/Documentation/networking/ipvs-sysctl.txt#L126:1
+		"net.ipv4.vs.expire_nodest_conn": {"1", nil},
+	}
 )
 
 // The networkNamespace type is the linux implementation of the Sandbox
@@ -358,16 +364,20 @@ func (n *networkNamespace) loopbackUp() error {
 	return n.nlHandle.LinkSetUp(iface)
 }
 
-func (n *networkNamespace) AddLoopbackAliasIP(ip *net.IPNet) error {
+func (n *networkNamespace) GetLoopbackIfaceName() string {
-	iface, err := n.nlHandle.LinkByName("lo")
+	return "lo"
+}
+
+func (n *networkNamespace) AddAliasIP(ifName string, ip *net.IPNet) error {
+	iface, err := n.nlHandle.LinkByName(ifName)
 	if err != nil {
 		return err
 	}
 	return n.nlHandle.AddrAdd(iface, &netlink.Addr{IPNet: ip})
 }
 
-func (n *networkNamespace) RemoveLoopbackAliasIP(ip *net.IPNet) error {
+func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error {
-	iface, err := n.nlHandle.LinkByName("lo")
+	iface, err := n.nlHandle.LinkByName(ifName)
 	if err != nil {
 		return err
 	}
@@ -626,3 +636,13 @@ func setIPv6(path, iface string, enable bool) error {
 	}
 	return nil
 }
+
+// ApplyOSTweaks applies linux configs on the sandbox
+func (n *networkNamespace) ApplyOSTweaks(types []SandboxType) {
+	for _, t := range types {
+		switch t {
+		case SandboxTypeLoadBalancer:
+			kernel.ApplyOSTweaks(loadBalancerConfig)
+		}
+	}
+}

vendor/github.com/docker/libnetwork/osl/sandbox.go

@@ -7,6 +7,16 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
+// SandboxType specify the time of the sandbox, this can be used to apply special configs
+type SandboxType int
+
+const (
+	// SandboxTypeIngress indicates that the sandbox is for the ingress
+	SandboxTypeIngress = iota
+	// SandboxTypeLoadBalancer indicates that the sandbox is a load balancer
+	SandboxTypeLoadBalancer = iota
+)
+
 // Sandbox represents a network sandbox, identified by a specific key.  It
 // holds a list of Interfaces, routes etc, and more can be added dynamically.
 type Sandbox interface {
@@ -32,11 +42,14 @@ type Sandbox interface {
 	// Unset the previously set default IPv6 gateway in the sandbox
 	UnsetGatewayIPv6() error
 
-	// AddLoopbackAliasIP adds the passed IP address to the sandbox loopback interface
+	// GetLoopbackIfaceName returns the name of the loopback interface
-	AddLoopbackAliasIP(ip *net.IPNet) error
+	GetLoopbackIfaceName() string
 
-	// RemoveLoopbackAliasIP removes the passed IP address from the sandbox loopback interface
+	// AddAliasIP adds the passed IP address to the named interface
-	RemoveLoopbackAliasIP(ip *net.IPNet) error
+	AddAliasIP(ifName string, ip *net.IPNet) error
+
+	// RemoveAliasIP removes the passed IP address from the named interface
+	RemoveAliasIP(ifName string, ip *net.IPNet) error
 
 	// Add a static route to the sandbox.
 	AddStaticRoute(*types.StaticRoute) error
@@ -67,6 +80,9 @@ type Sandbox interface {
 
 	// restore sandbox
 	Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
+
+	// ApplyOSTweaks applies operating system specific knobs on the sandbox
+	ApplyOSTweaks([]SandboxType)
 }
 
 // NeighborOptionSetter interface defines the option setter methods for interface options

vendor/github.com/docker/libnetwork/sandbox.go

@@ -83,6 +83,7 @@ type sandbox struct {
 	inDelete           bool
 	ingress            bool
 	ndotsSet           bool
+	oslTypes           []osl.SandboxType // slice of properties of this sandbox
 	sync.Mutex
 	// This mutex is used to serialize service related operation for an endpoint
 	// The lock is here because the endpoint is saved into the store so is not unique
@@ -740,15 +741,8 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) {
 
 	ep.Lock()
 	joinInfo := ep.joinInfo
-	vip := ep.virtualIP
 	ep.Unlock()
 
-	if len(vip) != 0 {
-		if err := osSbox.RemoveLoopbackAliasIP(&net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}); err != nil {
-			logrus.Warnf("Remove virtual IP %v failed: %v", vip, err)
-		}
-	}
-
 	if joinInfo == nil {
 		return
 	}
@@ -861,13 +855,6 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 		}
 	}
 
-	if len(ep.virtualIP) != 0 {
-		err := sb.osSbox.AddLoopbackAliasIP(&net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)})
-		if err != nil {
-			return fmt.Errorf("failed to add virtual IP %v: %v", ep.virtualIP, err)
-		}
-	}
-
 	if joinInfo != nil {
 		// Set up non-interface routes.
 		for _, r := range joinInfo.StaticRoutes {
@@ -893,7 +880,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 	// information including gateway and other routes so that
 	// loadbalancers are populated all the network state is in
 	// place in the sandbox.
-	sb.populateLoadbalancers(ep)
+	sb.populateLoadBalancers(ep)
 
 	// Only update the store if we did not come here as part of
 	// sandbox delete. If we came here as part of delete then do
@@ -1176,6 +1163,15 @@ func OptionPortMapping(portBindings []types.PortBinding) SandboxOption {
 func OptionIngress() SandboxOption {
 	return func(sb *sandbox) {
 		sb.ingress = true
+		sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress)
+	}
+}
+
+// OptionLoadBalancer function returns an option setter for marking a
+// sandbox as a load balancer sandbox.
+func OptionLoadBalancer() SandboxOption {
+	return func(sb *sandbox) {
+		sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer)
 	}
 }
 

vendor/github.com/docker/libnetwork/sandbox_dns_unix.go

@@ -369,11 +369,13 @@ dnsOpt:
 						return fmt.Errorf("invalid ndots option %v", option)
 					}
 					if num, err := strconv.Atoi(parts[1]); err != nil {
-						return fmt.Errorf("invalid number for ndots option %v", option)
+						return fmt.Errorf("invalid number for ndots option: %v", parts[1])
-					} else if num > 0 {
+					} else if num >= 0 {
 						// if the user sets ndots, use the user setting
 						sb.ndotsSet = true
 						break dnsOpt
+					} else {
+						return fmt.Errorf("invalid number for ndots option: %v", num)
 					}
 				}
 			}

vendor/github.com/docker/libnetwork/sandbox_store.go

@@ -244,7 +244,7 @@ func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 		}
 		sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
 		if err != nil {
-			logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
+			logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %.7s%s: %v", sb.ID(), msg, err)
 			continue
 		}
 

vendor/github.com/docker/libnetwork/service_common.go

@@ -225,6 +225,13 @@ func makeServiceCleanupFunc(c *controller, s *service, nID, eID string, vip net.
 func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases, taskAliases []string, ip net.IP, method string) error {
 	var addService bool
 
+	// Failure to lock the network ID on add can result in racing
+	// racing against network deletion resulting in inconsistent
+	// state in the c.serviceBindings map and it's sub-maps. Also,
+	// always lock network ID before services to avoid deadlock.
+	c.networkLocker.Lock(nID)
+	defer c.networkLocker.Unlock(nID)
+
 	n, err := c.NetworkByID(nID)
 	if err != nil {
 		return err
@@ -289,11 +296,8 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
 		logrus.Warnf("addServiceBinding %s possible transient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr)
 	}
 
-	// Add loadbalancer service and backend in all sandboxes in
+	// Add loadbalancer service and backend to the network
-	// the network only if vip is valid.
+	n.(*network).addLBBackend(ip, lb)
-	if len(vip) != 0 {
-		n.(*network).addLBBackend(ip, vip, lb, ingressPorts)
-	}
 
 	// Add the appropriate name resolutions
 	c.addEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, addService, "addServiceBinding")
@@ -307,11 +311,6 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
 
 	var rmService bool
 
-	n, err := c.NetworkByID(nID)
-	if err != nil {
-		return err
-	}
-
 	skey := serviceKey{
 		id:    svcID,
 		ports: portConfigs(ingressPorts).String(),
@@ -368,8 +367,15 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
 
 	// Remove loadbalancer service(if needed) and backend in all
 	// sandboxes in the network only if the vip is valid.
-	if len(vip) != 0 && entries == 0 {
+	if entries == 0 {
-		n.(*network).rmLBBackend(ip, vip, lb, ingressPorts, rmService, fullRemove)
+		// The network may well have been deleted before the last
+		// of the service bindings.  That's ok, because removing
+		// the network sandbox implicitly removes the backend
+		// service bindings.
+		n, err := c.NetworkByID(nID)
+		if err == nil {
+			n.(*network).rmLBBackend(ip, lb, rmService, fullRemove)
+		}
 	}
 
 	// Delete the name resolutions

vendor/github.com/docker/libnetwork/service_linux.go

@@ -30,40 +30,9 @@ func init() {
 	reexec.Register("redirecter", redirecter)
 }
 
-// Get all loadbalancers on this network that is currently discovered
-// on this node.
-func (n *network) connectedLoadbalancers() []*loadBalancer {
-	c := n.getController()
-
-	c.Lock()
-	serviceBindings := make([]*service, 0, len(c.serviceBindings))
-	for _, s := range c.serviceBindings {
-		serviceBindings = append(serviceBindings, s)
-	}
-	c.Unlock()
-
-	var lbs []*loadBalancer
-	for _, s := range serviceBindings {
-		s.Lock()
-		// Skip the serviceBindings that got deleted
-		if s.deleted {
-			s.Unlock()
-			continue
-		}
-		if lb, ok := s.loadBalancers[n.ID()]; ok {
-			lbs = append(lbs, lb)
-		}
-		s.Unlock()
-	}
-
-	return lbs
-}
-
 // Populate all loadbalancers on the network that the passed endpoint
 // belongs to, into this sandbox.
-func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
+func (sb *sandbox) populateLoadBalancers(ep *endpoint) {
-	var gwIP net.IP
-
 	// This is an interface less endpoint. Nothing to do.
 	if ep.Iface() == nil {
 		return
@@ -74,130 +43,113 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
 
 	if n.ingress {
 		if err := addRedirectRules(sb.Key(), eIP, ep.ingressPorts); err != nil {
-			logrus.Errorf("Failed to add redirect rules for ep %s (%s): %v", ep.Name(), ep.ID()[0:7], err)
+			logrus.Errorf("Failed to add redirect rules for ep %s (%.7s): %v", ep.Name(), ep.ID(), err)
 		}
 	}
-
-	if sb.ingress {
-		// For the ingress sandbox if this is not gateway
-		// endpoint do nothing.
-		if ep != sb.getGatewayEndpoint() {
-			return
-		}
-
-		// This is the gateway endpoint. Now get the ingress
-		// network and plumb the loadbalancers.
-		gwIP = ep.Iface().Address().IP
-		for _, ep := range sb.getConnectedEndpoints() {
-			if !ep.endpointInGWNetwork() {
-				n = ep.getNetwork()
-				eIP = ep.Iface().Address()
-			}
-		}
-	}
-
-	for _, lb := range n.connectedLoadbalancers() {
-		// Skip if vip is not valid.
-		if len(lb.vip) == 0 {
-			continue
-		}
-
-		lb.service.Lock()
-		for _, be := range lb.backEnds {
-			if !be.disabled {
-				sb.addLBBackend(be.ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress)
-			}
-		}
-		lb.service.Unlock()
-	}
 }
 
-// Add loadbalancer backend to all sandboxes which has a connection to
+func (n *network) findLBEndpointSandbox() (*endpoint, *sandbox, error) {
-// this network. If needed add the service as well.
+	// TODO: get endpoint from store?  See EndpointInfo()
-func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig) {
+	var ep *endpoint
-	n.WalkEndpoints(func(e Endpoint) bool {
+	// Find this node's LB sandbox endpoint:  there should be exactly one
-		ep := e.(*endpoint)
+	for _, e := range n.Endpoints() {
-		if sb, ok := ep.getSandbox(); ok {
+		epi := e.Info()
-			if !sb.isEndpointPopulated(ep) {
+		if epi != nil && epi.LoadBalancer() {
-				return false
+			ep = e.(*endpoint)
-			}
+			break
-
-			var gwIP net.IP
-			if ep := sb.getGatewayEndpoint(); ep != nil {
-				gwIP = ep.Iface().Address().IP
-			}
-
-			sb.addLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, n.ingress)
 		}
-
+	}
-		return false
+	if ep == nil {
-	})
+		return nil, nil, fmt.Errorf("Unable to find load balancing endpoint for network %s", n.ID())
+	}
+	// Get the load balancer sandbox itself as well
+	sb, ok := ep.getSandbox()
+	if !ok {
+		return nil, nil, fmt.Errorf("Unable to get sandbox for %s(%s) in for %s", ep.Name(), ep.ID(), n.ID())
+	}
+	ep = sb.getEndpoint(ep.ID())
+	if ep == nil {
+		return nil, nil, fmt.Errorf("Load balancing endpoint %s(%s) removed from %s", ep.Name(), ep.ID(), n.ID())
+	}
+	return ep, sb, nil
 }
 
-// Remove loadbalancer backend from all sandboxes which has a
+// Searches the OS sandbox for the name of the endpoint interface
-// connection to this network. If needed remove the service entry as
+// within the sandbox.   This is required for adding/removing IP
-// well, as specified by the rmService bool.
+// aliases to the interface.
-func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
+func findIfaceDstName(sb *sandbox, ep *endpoint) string {
-	n.WalkEndpoints(func(e Endpoint) bool {
+	srcName := ep.Iface().SrcName()
-		ep := e.(*endpoint)
+	for _, i := range sb.osSbox.Info().Interfaces() {
-		if sb, ok := ep.getSandbox(); ok {
+		if i.SrcName() == srcName {
-			if !sb.isEndpointPopulated(ep) {
+			return i.DstName()
-				return false
-			}
-
-			var gwIP net.IP
-			if ep := sb.getGatewayEndpoint(); ep != nil {
-				gwIP = ep.Iface().Address().IP
-			}
-
-			sb.rmLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, rmService, fullRemove, n.ingress)
 		}
-
+	}
-		return false
+	return ""
-	})
 }
 
-// Add loadbalancer backend into one connected sandbox.
+// Add loadbalancer backend to the loadbalncer sandbox for the network.
-func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, isIngressNetwork bool) {
+// If needed add the service as well.
+func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
+	if len(lb.vip) == 0 {
+		return
+	}
+	ep, sb, err := n.findLBEndpointSandbox()
+	if err != nil {
+		logrus.Errorf("addLBBackend %s/%s: %v", n.ID(), n.Name(), err)
+		return
+	}
 	if sb.osSbox == nil {
 		return
 	}
 
-	if isIngressNetwork && !sb.ingress {
+	eIP := ep.Iface().Address()
-		return
-	}
 
 	i, err := ipvs.New(sb.Key())
 	if err != nil {
-		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb addition: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
+		logrus.Errorf("Failed to create an ipvs handle for sbox %.7s (%.7s,%s) for lb addition: %v", sb.ID(), sb.ContainerID(), sb.Key(), err)
 		return
 	}
 	defer i.Close()
 
 	s := &ipvs.Service{
 		AddressFamily: nl.FAMILY_V4,
-		FWMark:        fwMark,
+		FWMark:        lb.fwMark,
 		SchedName:     ipvs.RoundRobin,
 	}
 
 	if !i.IsServicePresent(s) {
-		var filteredPorts []*PortConfig
+		// Add IP alias for the VIP to the endpoint
+		ifName := findIfaceDstName(sb, ep)
+		if ifName == "" {
+			logrus.Errorf("Failed find interface name for endpoint %s(%s) to create LB alias", ep.ID(), ep.Name())
+			return
+		}
+		err := sb.osSbox.AddAliasIP(ifName, &net.IPNet{IP: lb.vip, Mask: net.CIDRMask(32, 32)})
+		if err != nil {
+			logrus.Errorf("Failed add IP alias %s to network %s LB endpoint interface %s: %v", lb.vip, n.ID(), ifName, err)
+			return
+		}
+
 		if sb.ingress {
-			filteredPorts = filterPortConfigs(ingressPorts, false)
+			var gwIP net.IP
+			if ep := sb.getGatewayEndpoint(); ep != nil {
+				gwIP = ep.Iface().Address().IP
+			}
+			filteredPorts := filterPortConfigs(lb.service.ingressPorts, false)
 			if err := programIngress(gwIP, filteredPorts, false); err != nil {
 				logrus.Errorf("Failed to add ingress: %v", err)
 				return
 			}
 		}
 
-		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %s (%s)", vip, fwMark, ingressPorts, sb.ID()[0:7], sb.ContainerID()[0:7])
+		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID())
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, false); err != nil {
+		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false); err != nil {
-			logrus.Errorf("Failed to add firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
 			return
 		}
 
 		if err := i.NewService(s); err != nil && err != syscall.EEXIST {
-			logrus.Errorf("Failed to create a new service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to create a new service for vip %s fwmark %d in sbox %.7s (%.7s): %v", lb.vip, lb.fwMark, sb.ID(), sb.ContainerID(), err)
 			return
 		}
 	}
@@ -212,30 +164,39 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 	// destination.
 	s.SchedName = ""
 	if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST {
-		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sbox %.7s (%.7s): %v", ip, lb.vip, lb.fwMark, sb.ID(), sb.ContainerID(), err)
 	}
 }
 
-// Remove loadbalancer backend from one connected sandbox.
+// Remove loadbalancer backend the load balancing endpoint for this
-func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, rmService bool, fullRemove bool, isIngressNetwork bool) {
+// network. If 'rmService' is true, then remove the service entry as well.
+// If 'fullRemove' is true then completely remove the entry, otherwise
+// just deweight it for now.
+func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullRemove bool) {
+	if len(lb.vip) == 0 {
+		return
+	}
+	ep, sb, err := n.findLBEndpointSandbox()
+	if err != nil {
+		logrus.Debugf("rmLBBackend for %s/%s: %v -- probably transient state", n.ID(), n.Name(), err)
+		return
+	}
 	if sb.osSbox == nil {
 		return
 	}
 
-	if isIngressNetwork && !sb.ingress {
+	eIP := ep.Iface().Address()
-		return
-	}
 
 	i, err := ipvs.New(sb.Key())
 	if err != nil {
-		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb removal: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
+		logrus.Errorf("Failed to create an ipvs handle for sbox %.7s (%.7s,%s) for lb removal: %v", sb.ID(), sb.ContainerID(), sb.Key(), err)
 		return
 	}
 	defer i.Close()
 
 	s := &ipvs.Service{
 		AddressFamily: nl.FAMILY_V4,
-		FWMark:        fwMark,
+		FWMark:        lb.fwMark,
 	}
 
 	d := &ipvs.Destination{
@@ -246,31 +207,45 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 
 	if fullRemove {
 		if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {
-			logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %.7s (%.7s): %v", ip, lb.vip, lb.fwMark, sb.ID(), sb.ContainerID(), err)
 		}
 	} else {
 		d.Weight = 0
 		if err := i.UpdateDestination(s, d); err != nil && err != syscall.ENOENT {
-			logrus.Errorf("Failed to set LB weight of real server %s to 0 for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to set LB weight of real server %s to 0 for vip %s fwmark %d in sbox %.7s (%.7s): %v", ip, lb.vip, lb.fwMark, sb.ID(), sb.ContainerID(), err)
 		}
 	}
 
 	if rmService {
 		s.SchedName = ipvs.RoundRobin
 		if err := i.DelService(s); err != nil && err != syscall.ENOENT {
-			logrus.Errorf("Failed to delete service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to delete service for vip %s fwmark %d in sbox %.7s (%.7s): %v", lb.vip, lb.fwMark, sb.ID(), sb.ContainerID(), err)
 		}
 
-		var filteredPorts []*PortConfig
 		if sb.ingress {
-			filteredPorts = filterPortConfigs(ingressPorts, true)
+			var gwIP net.IP
+			if ep := sb.getGatewayEndpoint(); ep != nil {
+				gwIP = ep.Iface().Address().IP
+			}
+			filteredPorts := filterPortConfigs(lb.service.ingressPorts, true)
 			if err := programIngress(gwIP, filteredPorts, true); err != nil {
 				logrus.Errorf("Failed to delete ingress: %v", err)
 			}
 		}
 
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, true); err != nil {
+		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true); err != nil {
-			logrus.Errorf("Failed to delete firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
+			logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
+		}
+
+		// Remove IP alias from the VIP to the endpoint
+		ifName := findIfaceDstName(sb, ep)
+		if ifName == "" {
+			logrus.Errorf("Failed find interface name for endpoint %s(%s) to create LB alias", ep.ID(), ep.Name())
+			return
+		}
+		err := sb.osSbox.RemoveAliasIP(ifName, &net.IPNet{IP: lb.vip, Mask: net.CIDRMask(32, 32)})
+		if err != nil {
+			logrus.Errorf("Failed add IP alias %s to network %s LB endpoint interface %s: %v", lb.vip, n.ID(), ifName, err)
 		}
 	}
 }
@@ -617,7 +592,7 @@ func fwMarker() {
 		ingressPorts, err = readPortsFromFile(os.Args[5])
 		if err != nil {
 			logrus.Errorf("Failed reading ingress ports file: %v", err)
-			os.Exit(6)
+			os.Exit(2)
 		}
 	}
 
@@ -625,7 +600,7 @@ func fwMarker() {
 	fwMark, err := strconv.ParseUint(os.Args[3], 10, 32)
 	if err != nil {
 		logrus.Errorf("bad fwmark value(%s) passed: %v", os.Args[3], err)
-		os.Exit(2)
+		os.Exit(3)
 	}
 	addDelOpt := os.Args[4]
 
@@ -639,20 +614,20 @@ func fwMarker() {
 	ns, err := netns.GetFromPath(os.Args[1])
 	if err != nil {
 		logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
-		os.Exit(3)
+		os.Exit(4)
 	}
 	defer ns.Close()
 
 	if err := netns.Set(ns); err != nil {
 		logrus.Errorf("setting into container net ns %v failed, %v", os.Args[1], err)
-		os.Exit(4)
+		os.Exit(5)
 	}
 
 	if addDelOpt == "-A" {
 		eIP, subnet, err := net.ParseCIDR(os.Args[6])
 		if err != nil {
 			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
-			os.Exit(9)
+			os.Exit(6)
 		}
 
 		ruleParams := strings.Fields(fmt.Sprintf("-m ipvs --ipvs -d %s -j SNAT --to-source %s", subnet, eIP))
@@ -663,21 +638,18 @@ func fwMarker() {
 			err := ioutil.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
 			if err != nil {
 				logrus.Errorf("Failed to write to /proc/sys/net/ipv4/vs/conntrack: %v", err)
-				os.Exit(8)
+				os.Exit(7)
 			}
 		}
 	}
 
-	rule := strings.Fields(fmt.Sprintf("-t mangle %s OUTPUT -d %s/32 -j MARK --set-mark %d", addDelOpt, vip, fwMark))
+	rule := strings.Fields(fmt.Sprintf("-t mangle %s INPUT -d %s/32 -j MARK --set-mark %d", addDelOpt, vip, fwMark))
-	rules = append(rules, rule)
-
-	rule = strings.Fields(fmt.Sprintf("-t nat %s OUTPUT -p icmp --icmp echo-request -d %s -j DNAT --to 127.0.0.1", addDelOpt, vip))
 	rules = append(rules, rule)
 
 	for _, rule := range rules {
 		if err := iptables.RawCombinedOutputNative(rule...); err != nil {
 			logrus.Errorf("setting up rule failed, %v: %v", rule, err)
-			os.Exit(5)
+			os.Exit(8)
 		}
 	}
 }

vendor/github.com/docker/libnetwork/service_unsupported.go

@@ -18,7 +18,7 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
 	return fmt.Errorf("not supported")
 }
 
-func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
+func (sb *sandbox) populateLoadBalancers(ep *endpoint) {
 }
 
 func arrangeIngressFilterRule() {

vendor/github.com/docker/libnetwork/service_windows.go

@@ -19,7 +19,13 @@ func init() {
 	lbPolicylistMap = make(map[*loadBalancer]*policyLists)
 }
 
-func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig) {
+func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
+	if len(lb.vip) == 0 {
+		return
+	}
+
+	vip := lb.vip
+	ingressPorts := lb.service.ingressPorts
 
 	if system.GetOSVersion().Build > 16236 {
 		lb.Lock()
@@ -117,11 +123,15 @@ func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []
 	}
 }
 
-func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
+func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullRemove bool) {
+	if len(lb.vip) == 0 {
+		return
+	}
+
 	if system.GetOSVersion().Build > 16236 {
 		if numEnabledBackends(lb) > 0 {
 			//Reprogram HNS (actually VFP) with the existing backends.
-			n.addLBBackend(ip, vip, lb, ingressPorts)
+			n.addLBBackend(ip, lb)
 		} else {
 			lb.Lock()
 			defer lb.Unlock()
@@ -156,7 +166,7 @@ func numEnabledBackends(lb *loadBalancer) int {
 	return nEnabled
 }
 
-func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
+func (sb *sandbox) populateLoadBalancers(ep *endpoint) {
 }
 
 func arrangeIngressFilterRule() {

vendor/github.com/docker/libnetwork/store.go

@@ -479,7 +479,7 @@ func (c *controller) networkCleanup() {
 	for _, n := range networks {
 		if n.inDelete {
 			logrus.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
-			if err := n.delete(true); err != nil {
+			if err := n.delete(true, true); err != nil {
 				logrus.Debugf("Error while removing stale network: %v", err)
 			}
 		}

vendor/github.com/docker/libnetwork/types/types.go

@@ -332,6 +332,8 @@ func CompareIPNet(a, b *net.IPNet) bool {
 }
 
 // GetMinimalIP returns the address in its shortest form
+// If ip contains an IPv4-mapped IPv6 address, the 4-octet form of the IPv4 address will be returned.
+// Otherwise ip is returned unchanged.
 func GetMinimalIP(ip net.IP) net.IP {
 	if ip != nil && ip.To4() != nil {
 		return ip.To4()

vendor/github.com/docker/libnetwork/vendor.conf

@@ -43,7 +43,7 @@ github.com/opencontainers/selinux b29023b86e4a69d1b46b7e7b4e2b6fda03f0b9cd
 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
 github.com/sirupsen/logrus v1.0.3
-github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a
+github.com/stretchr/testify v1.2.2
 github.com/syndtr/gocapability 33e07d32887e1e06b7c025f27ce52f62c7990bc0
 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065
 github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
@@ -55,8 +55,8 @@ golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5
 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
 github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb
 
-github.com/davecgh/go-spew 8991bc29aa16c548c550c7ff78260e27b9ab7c73
+github.com/davecgh/go-spew v1.1.0
-github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2
+github.com/pmezard/go-difflib v1.0.0
 github.com/cyphar/filepath-securejoin v0.2.1
 github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55
 github.com/hashicorp/go-immutable-radix 7f3cd4390caab3250a57f30efdb2a65dd7649ecf