Merge pull request #535 from sanimej/ext_conn

Support for default gateway for containers

libnetwork/default_gateway.go

@@ -0,0 +1,159 @@
+package libnetwork
+
+import (
+	"fmt"
+
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/options"
+	"github.com/docker/libnetwork/types"
+)
+
+const (
+	libnGWNetwork = "docker_gwbridge"
+	gwEPlen       = 12
+)
+
+/*
+   libnetwork creates a bridge network "docker_gw_bridge" for provding
+   default gateway for the containers if none of the container's endpoints
+   have GW set by the driver. ICC is set to false for the GW_bridge network.
+
+   If a driver can't provide external connectivity it can choose to not set
+   the GW IP for the endpoint.
+
+   endpoint on the GW_bridge network is managed dynamically by libnetwork.
+   ie:
+   - its created when an endpoint without GW joins the container
+   - its deleted when an endpoint with GW joins the container
+*/
+
+func (sb *sandbox) setupDefaultGW(srcEp *endpoint) error {
+	var createOptions []EndpointOption
+	c := srcEp.getNetwork().getController()
+
+	// check if the conitainer already has a GW endpoint
+	if ep := sb.getEndpointInGWNetwork(); ep != nil {
+		return nil
+	}
+
+	n, err := c.NetworkByName(libnGWNetwork)
+	if err != nil {
+		if _, ok := err.(types.NotFoundError); !ok {
+			return err
+		}
+		n, err = c.createGWNetwork()
+		if err != nil {
+			return err
+		}
+	}
+
+	if opt, ok := srcEp.generic[netlabel.PortMap]; ok {
+		if pb, ok := opt.([]types.PortBinding); ok {
+			createOptions = append(createOptions, CreateOptionPortMapping(pb))
+		}
+	}
+
+	if opt, ok := srcEp.generic[netlabel.ExposedPorts]; ok {
+		if exp, ok := opt.([]types.TransportPort); ok {
+			createOptions = append(createOptions, CreateOptionExposedPorts(exp))
+		}
+	}
+
+	eplen := gwEPlen
+	if len(sb.containerID) < gwEPlen {
+		eplen = len(sb.containerID)
+	}
+
+	newEp, err := n.CreateEndpoint("gateway_"+sb.containerID[0:eplen], createOptions...)
+	if err != nil {
+		return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err)
+	}
+
+	if err := newEp.Join(sb); err != nil {
+		return fmt.Errorf("container %s: endpoint join on GW Network failed: %v", sb.containerID, err)
+	}
+	return nil
+}
+
+func (sb *sandbox) clearDefaultGW() error {
+	var ep *endpoint
+
+	if ep = sb.getEndpointInGWNetwork(); ep == nil {
+		return nil
+	}
+
+	if err := ep.Leave(sb); err != nil {
+		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
+	}
+	if err := ep.Delete(); err != nil {
+		return fmt.Errorf("container %s: deleting endpoint on GW Network failed: %v", sb.containerID, err)
+	}
+	return nil
+}
+
+func (c *controller) createGWNetwork() (Network, error) {
+	netOption := options.Generic{
+		"BridgeName":            libnGWNetwork,
+		"EnableICC":             false,
+		"AllowNonDefaultBridge": true,
+		"EnableIPMasquerade":    true,
+	}
+
+	n, err := c.NewNetwork("bridge", libnGWNetwork,
+		NetworkOptionGeneric(options.Generic{
+			netlabel.GenericData: netOption,
+			netlabel.EnableIPv6:  false,
+		}))
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating external connectivity network: %v", err)
+	}
+	return n, err
+}
+
+func (sb *sandbox) needDefaultGW() bool {
+	var needGW bool
+
+	for _, ep := range sb.getConnectedEndpoints() {
+		if ep.endpointInGWNetwork() {
+			continue
+		}
+		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
+			continue
+		}
+		// TODO v6 needs to be handled.
+		if len(ep.Gateway()) > 0 {
+			return false
+		}
+		needGW = true
+	}
+	return needGW
+}
+
+func (sb *sandbox) getEndpointInGWNetwork() *endpoint {
+	for _, ep := range sb.getConnectedEndpoints() {
+		if ep.getNetwork().name == libnGWNetwork {
+			return ep
+		}
+	}
+	return nil
+}
+
+func (ep *endpoint) endpointInGWNetwork() bool {
+	if ep.getNetwork().name == libnGWNetwork {
+		return true
+	}
+	return false
+}
+
+func (sb *sandbox) getEPwithoutGateway() *endpoint {
+	for _, ep := range sb.getConnectedEndpoints() {
+		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
+			continue
+		}
+		if len(ep.Gateway()) == 0 {
+			return ep
+		}
+	}
+	return nil
+}

libnetwork/drivers/overlay/joinleave.go

@@ -72,11 +72,6 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		}
 	}
 
-	err = jinfo.SetGateway(bridgeIP.IP)
-	if err != nil {
-		return err
-	}
-
 	d.peerDbAdd(nid, eid, ep.addr.IP, ep.mac,
 		d.serfInstance.LocalMember().Addr, true)
 	d.notifyCh <- ovNotify{

libnetwork/endpoint.go

@@ -304,7 +304,11 @@ func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
 	if err = sb.populateNetworkResources(ep); err != nil {
 		return err
 	}
-	return nil
+
+	if sb.needDefaultGW() {
+		return sb.setupDefaultGW(ep)
+	}
+	return sb.clearDefaultGW()
 }
 
 func (ep *endpoint) hasInterface(iName string) bool {
@@ -361,7 +365,18 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
 		return err
 	}
 
-	return sb.clearNetworkResources(ep)
+	if err := sb.clearNetworkResources(ep); err != nil {
+		return err
+	}
+
+	if sb.needDefaultGW() {
+		ep := sb.getEPwithoutGateway()
+		if ep == nil {
+			return fmt.Errorf("endpoint without GW expected, but not found")
+		}
+		return sb.setupDefaultGW(ep)
+	}
+	return sb.clearDefaultGW()
 }
 
 func (ep *endpoint) Delete() error {

libnetwork/sandbox.go

@@ -149,6 +149,11 @@ func (sb *sandbox) Delete() error {
 
 	// Detach from all endpoints
 	for _, ep := range sb.getConnectedEndpoints() {
+		// endpoint in the Gateway network will be cleaned up
+		// when when sandbox no longer needs external connectivity
+		if ep.endpointInGWNetwork() {
+			continue
+		}
 		if err := ep.Leave(sb); err != nil {
 			log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
 		}
@@ -748,6 +753,17 @@ func (eh epHeap) Less(i, j int) bool {
 	ci, _ := eh[i].getSandbox()
 	cj, _ := eh[j].getSandbox()
 
+	epi := eh[i]
+	epj := eh[j]
+
+	if epi.endpointInGWNetwork() {
+		return true
+	}
+
+	if epj.endpointInGWNetwork() {
+		return false
+	}
+
 	cip, ok := ci.epPriority[eh[i].ID()]
 	if !ok {
 		cip = 0