Optional Userland Proxy

- Port https://github.com/docker/docker/pull/12165 to libnetwork
- More tests will be added later

Signed-off-by: Alessandro Boch <aboch@docker.com>

libnetwork/drivers/bridge/bridge.go

@@ -51,6 +51,7 @@ type NetworkConfiguration struct {
 	DefaultGatewayIPv6    net.IP
 	DefaultBindingIP      net.IP
 	AllowNonDefaultBridge bool
+	EnableUserlandProxy   bool
 }
 
 // EndpointConfiguration represents the user specified configuration for the sandbox endpoint
@@ -309,6 +310,9 @@ func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) err
 		// specified subnet.
 		{config.FixedCIDRv6 != nil, setupFixedCIDRv6},
 
+		// Setup Loopback Adresses Routing
+		{!config.EnableUserlandProxy, setupLoopbackAdressesRouting},
+
 		// Setup IPTables.
 		{config.EnableIPTables, setupIPTables},
 
@@ -557,7 +561,7 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn
 	}
 
 	// Program any required port mapping and store them in the endpoint
-	endpoint.portMapping, err = allocatePorts(epConfig, intf, config.DefaultBindingIP)
+	endpoint.portMapping, err = allocatePorts(epConfig, intf, config.DefaultBindingIP, config.EnableUserlandProxy)
 	if err != nil {
 		return err
 	}

libnetwork/drivers/bridge/bridge_test.go

@@ -157,6 +157,14 @@ func (te *testEndpoint) SetResolvConfPath(path string) error {
 }
 
 func TestQueryEndpointInfo(t *testing.T) {
+	testQueryEndpointInfo(t, true)
+}
+
+func TestQueryEndpointInfoHairpin(t *testing.T) {
+	testQueryEndpointInfo(t, false)
+}
+
+func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) {
 	defer netutils.SetupTestNetNS(t)()
 	d := newDriver()
 	dd, _ := d.(*driver)
@@ -165,6 +173,7 @@ func TestQueryEndpointInfo(t *testing.T) {
 		BridgeName:          DefaultBridgeName,
 		EnableIPTables:      true,
 		EnableICC:           false,
+		EnableUserlandProxy: ulPxyEnabled,
 	}
 	genericOption := make(map[string]interface{})
 	genericOption[netlabel.GenericData] = config

libnetwork/drivers/bridge/port_mapping.go

@@ -15,7 +15,7 @@ var (
 	defaultBindingIP = net.IPv4(0, 0, 0, 0)
 )
 
-func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, reqDefBindIP net.IP) ([]netutils.PortBinding, error) {
+func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, reqDefBindIP net.IP, ulPxyEnabled bool) ([]netutils.PortBinding, error) {
 	if epConfig == nil || epConfig.PortBindings == nil {
 		return nil, nil
 	}
@@ -25,14 +25,14 @@ func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, req
 		defHostIP = reqDefBindIP
 	}
 
-	return allocatePortsInternal(epConfig.PortBindings, intf.Address.IP, defHostIP)
+	return allocatePortsInternal(epConfig.PortBindings, intf.Address.IP, defHostIP, ulPxyEnabled)
 }
 
-func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHostIP net.IP) ([]netutils.PortBinding, error) {
+func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]netutils.PortBinding, error) {
 	bs := make([]netutils.PortBinding, 0, len(bindings))
 	for _, c := range bindings {
 		b := c.GetCopy()
-		if err := allocatePort(&b, containerIP, defHostIP); err != nil {
+		if err := allocatePort(&b, containerIP, defHostIP, ulPxyEnabled); err != nil {
 			// On allocation failure, release previously allocated ports. On cleanup error, just log a warning message
 			if cuErr := releasePortsInternal(bs); cuErr != nil {
 				logrus.Warnf("Upon allocation failure for %v, failed to clear previously allocated port bindings: %v", b, cuErr)
@@ -44,7 +44,7 @@ func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHost
 	return bs, nil
 }
 
-func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP) error {
+func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) error {
 	var (
 		host net.Addr
 		err  error
@@ -66,7 +66,7 @@ func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP) erro
 
 	// Try up to maxAllocatePortAttempts times to get a port that's not already allocated.
 	for i := 0; i < maxAllocatePortAttempts; i++ {
-		if host, err = portMapper.Map(container, bnd.HostIP, int(bnd.HostPort)); err == nil {
+		if host, err = portMapper.Map(container, bnd.HostIP, int(bnd.HostPort), ulPxyEnabled); err == nil {
 			break
 		}
 		// There is no point in immediately retrying to map an explicitly chosen port.

libnetwork/drivers/bridge/setup_ip_tables.go

@@ -19,20 +19,22 @@ func setupIPTables(config *NetworkConfiguration, i *bridgeInterface) error {
 		return ipTableCfgError(config.BridgeName)
 	}
 
+	hairpinMode := !config.EnableUserlandProxy
+
 	addrv4, _, err := netutils.GetIfaceAddr(config.BridgeName)
 	if err != nil {
 		return fmt.Errorf("Failed to setup IP tables, cannot acquire Interface address: %s", err.Error())
 	}
-	if err = setupIPTablesInternal(config.BridgeName, addrv4, config.EnableICC, config.EnableIPMasquerade, true); err != nil {
+	if err = setupIPTablesInternal(config.BridgeName, addrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, true); err != nil {
 		return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
 	}
 
-	_, err = iptables.NewChain(DockerChain, config.BridgeName, iptables.Nat)
+	_, err = iptables.NewChain(DockerChain, config.BridgeName, iptables.Nat, hairpinMode)
 	if err != nil {
 		return fmt.Errorf("Failed to create NAT chain: %s", err.Error())
 	}
 
-	chain, err := iptables.NewChain(DockerChain, config.BridgeName, iptables.Filter)
+	chain, err := iptables.NewChain(DockerChain, config.BridgeName, iptables.Filter, hairpinMode)
 	if err != nil {
 		return fmt.Errorf("Failed to create FILTER chain: %s", err.Error())
 	}
@@ -49,11 +51,12 @@ type iptRule struct {
 	args    []string
 }
 
-func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, enable bool) error {
+func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairpin, enable bool) error {
 
 	var (
 		address   = addr.String()
 		natRule   = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
+		hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}}
 		outRule   = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
 		inRule    = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
 	)
@@ -65,6 +68,13 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, enabl
 		}
 	}
 
+	// In hairpin mode, masquerade traffic from localhost
+	if hairpin {
+		if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil {
+			return err
+		}
+	}
+
 	// Set Inter Container Communication.
 	if err := setIcc(bridgeIface, icc, enable); err != nil {
 		return err

libnetwork/drivers/bridge/setup_ipv4.go

@@ -1,8 +1,12 @@
 package bridge
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net"
 
+	"path/filepath"
+
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/netutils"
 	"github.com/vishvananda/netlink"
@@ -121,3 +125,12 @@ func setupGatewayIPv4(config *NetworkConfiguration, i *bridgeInterface) error {
 
 	return nil
 }
+
+func setupLoopbackAdressesRouting(config *NetworkConfiguration, i *bridgeInterface) error {
+	// Enable loopback adresses routing
+	sysPath := filepath.Join("/proc/sys/net/ipv4/conf", config.BridgeName, "route_localnet")
+	if err := ioutil.WriteFile(sysPath, []byte{'1', '\n'}, 0644); err != nil {
+		return fmt.Errorf("Unable to enable local routing for hairpin mode: %v", err)
+	}
+	return nil
+}

libnetwork/iptables/firewalld.go

@@ -14,8 +14,8 @@ type IPV string
 const (
 	// Iptables point ipv4 table
 	Iptables IPV = "ipv4"
-	// IP6tables point to ipv6 table
+	// IP6Tables point to ipv6 table
-	IP6tables IPV = "ipv6"
+	IP6Tables IPV = "ipv6"
 	// Ebtables point to bridge table
 	Ebtables IPV = "eb"
 )
@@ -156,7 +156,6 @@ func checkRunning() bool {
 // Passthrough method simply passes args through to iptables/ip6tables
 func Passthrough(ipv IPV, args ...string) ([]byte, error) {
 	var output string
-
 	logrus.Debugf("Firewalld passthrough: %s, %s", ipv, args)
 	if err := connection.sysobj.Call(dbusInterface+".direct.passthrough", 0, ipv, args).Store(&output); err != nil {
 		return nil, err

libnetwork/iptables/firewalld_test.go

@@ -7,14 +7,19 @@ import (
 )
 
 func TestFirewalldInit(t *testing.T) {
-	FirewalldInit()
+	if !checkRunning() {
+		t.Skip("firewalld is not running")
+	}
+	if err := FirewalldInit(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestReloaded(t *testing.T) {
 	var err error
 	var fwdChain *Chain
 
-	fwdChain, err = NewChain("FWD", "lo", Filter)
+	fwdChain, err = NewChain("FWD", "lo", Filter, false)
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/iptables/iptables.go

@@ -74,7 +74,7 @@ func initCheck() error {
 }
 
 // NewChain adds a new chain to ip table.
-func NewChain(name, bridge string, table Table) (*Chain, error) {
+func NewChain(name, bridge string, table Table, hairpinMode bool) (*Chain, error) {
 	c := &Chain{
 		Name:   name,
 		Bridge: bridge,
@@ -106,8 +106,10 @@ func NewChain(name, bridge string, table Table) (*Chain, error) {
 		}
 		output := []string{
 			"-m", "addrtype",
-			"--dst-type", "LOCAL",
+			"--dst-type", "LOCAL"}
-			"!", "--dst", "127.0.0.0/8"}
+		if !hairpinMode {
+			output = append(output, "!", "--dst", "127.0.0.0/8")
+		}
 		if !Exists(Nat, "OUTPUT", output...) {
 			if err := c.Output(Append, output...); err != nil {
 				return nil, fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
@@ -141,7 +143,7 @@ func RemoveExistingChain(name string, table Table) error {
 	return c.Remove()
 }
 
-//Forward adds forwarding rule to 'filter' table and corresponding nat rule to 'nat' table
+// Forward adds forwarding rule to 'filter' table and corresponding nat rule to 'nat' table.
 func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr string, destPort int) error {
 	daddr := ip.String()
 	if ip.IsUnspecified() {
@@ -154,7 +156,6 @@ func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr stri
 		"-p", proto,
 		"-d", daddr,
 		"--dport", strconv.Itoa(port),
-		"!", "-i", c.Bridge,
 		"-j", "DNAT",
 		"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))); err != nil {
 		return err
@@ -230,7 +231,7 @@ func (c *Chain) Prerouting(action Action, args ...string) error {
 	return nil
 }
 
-//Output adds linking rule to an OUTPUT chain
+// Output adds linking rule to an OUTPUT chain.
 func (c *Chain) Output(action Action, args ...string) error {
 	a := []string{"-t", string(c.Table), string(action), "OUTPUT"}
 	if len(args) > 0 {
@@ -244,7 +245,7 @@ func (c *Chain) Output(action Action, args ...string) error {
 	return nil
 }
 
-// Remove removes the chain
+// Remove removes the chain.
 func (c *Chain) Remove() error {
 	// Ignore errors - This could mean the chains were never set up
 	if c.Table == Nat {
@@ -291,7 +292,7 @@ func Exists(table Table, chain string, rule ...string) bool {
 	)
 }
 
-//Raw calls 'iptables' system command, passing supplied arguments
+// Raw calls 'iptables' system command, passing supplied arguments.
 func Raw(args ...string) ([]byte, error) {
 	if firewalldRunning {
 		output, err := Passthrough(Iptables, args...)

libnetwork/iptables/iptables_test.go

@@ -7,11 +7,9 @@ import (
 	"strings"
 	"sync"
 	"testing"
-
-	_ "github.com/docker/libnetwork/netutils"
 )
 
-const chainName = "DOCKERTEST"
+const chainName = "DOCKER-TEST"
 
 var natChain *Chain
 var filterChain *Chain
@@ -19,12 +17,12 @@ var filterChain *Chain
 func TestNewChain(t *testing.T) {
 	var err error
 
-	natChain, err = NewChain(chainName, "lo", Nat)
+	natChain, err = NewChain(chainName, "lo", Nat, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	filterChain, err = NewChain(chainName, "lo", Filter)
+	filterChain, err = NewChain(chainName, "lo", Filter, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -43,7 +41,6 @@ func TestForward(t *testing.T) {
 	}
 
 	dnatRule := []string{
-		"!", "-i", filterChain.Bridge,
 		"-d", ip.String(),
 		"-p", proto,
 		"--dport", strconv.Itoa(port),

libnetwork/libnetwork_test.go

@@ -579,7 +579,7 @@ func TestControllerQuery(t *testing.T) {
 
 	g, err := controller.NetworkByID("network1")
 	if err == nil {
-		t.Fatalf("Unexpected success for NetworkByID(): %g", g)
+		t.Fatalf("Unexpected success for NetworkByID(): %v", g)
 	}
 	if err != libnetwork.ErrNoSuchNetwork {
 		t.Fatalf("NetworkByID() failed with unexpected error: %v", err)

libnetwork/portmapper/mapper.go

@@ -59,7 +59,7 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.Chain) {
 }
 
 // Map maps the specified container transport address to the host's network address and transport port
-func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host net.Addr, err error) {
+func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
 	pm.lock.Lock()
 	defer pm.lock.Unlock()
 
@@ -67,7 +67,6 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 		m                 *mapping
 		proto             string
 		allocatedHostPort int
-		proxy             userlandProxy
 	)
 
 	switch container.(type) {
@@ -83,7 +82,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 			container: container,
 		}
 
-		proxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
+		if useProxy {
+			m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
+		}
 	case *net.UDPAddr:
 		proto = "udp"
 		if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil {
@@ -96,7 +97,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 			container: container,
 		}
 
-		proxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
+		if useProxy {
+			m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
+		}
 	default:
 		return nil, ErrUnknownBackendAddressType
 	}
@@ -120,7 +123,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 
 	cleanup := func() error {
 		// need to undo the iptables rules before we return
-		proxy.Stop()
+		if m.userlandProxy != nil {
+			m.userlandProxy.Stop()
+		}
 		pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
 		if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
 			return err
@@ -129,13 +134,15 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 		return nil
 	}
 
-	if err := proxy.Start(); err != nil {
+	if m.userlandProxy != nil {
+		if err := m.userlandProxy.Start(); err != nil {
 			if err := cleanup(); err != nil {
 				return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
 			}
 			return nil, err
 		}
-	m.userlandProxy = proxy
+	}
+
 	pm.currentMappings[key] = m
 	return m.host, nil
 }
@@ -151,7 +158,9 @@ func (pm *PortMapper) Unmap(host net.Addr) error {
 		return ErrPortNotMapped
 	}
 
+	if data.userlandProxy != nil {
 		data.userlandProxy.Stop()
+	}
 
 	delete(pm.currentMappings, key)
 

libnetwork/portmapper/mapper_test.go

@@ -51,22 +51,22 @@ func TestMapTCPPorts(t *testing.T) {
 		return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
 	}
 
-	if host, err := pm.Map(srcAddr1, dstIP1, 80); err != nil {
+	if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	} else if !addrEqual(dstAddr1, host) {
 		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
 			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
 	}
 
-	if _, err := pm.Map(srcAddr1, dstIP1, 80); err == nil {
+	if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIP1, 80); err == nil {
+	if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIP2, 80); err != nil {
+	if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	}
 
@@ -131,22 +131,22 @@ func TestMapUDPPorts(t *testing.T) {
 		return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
 	}
 
-	if host, err := pm.Map(srcAddr1, dstIP1, 80); err != nil {
+	if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	} else if !addrEqual(dstAddr1, host) {
 		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
 			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
 	}
 
-	if _, err := pm.Map(srcAddr1, dstIP1, 80); err == nil {
+	if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIP1, 80); err == nil {
+	if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIP2, 80); err != nil {
+	if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	}
 
@@ -180,14 +180,14 @@ func TestMapAllPortsSingleInterface(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		start, end := pm.Allocator.Begin, pm.Allocator.End
 		for i := start; i < end; i++ {
-			if host, err = pm.Map(srcAddr1, dstIP1, 0); err != nil {
+			if host, err = pm.Map(srcAddr1, dstIP1, 0, true); err != nil {
 				t.Fatal(err)
 			}
 
 			hosts = append(hosts, host)
 		}
 
-		if _, err := pm.Map(srcAddr1, dstIP1, start); err == nil {
+		if _, err := pm.Map(srcAddr1, dstIP1, start, true); err == nil {
 			t.Fatalf("Port %d should be bound but is not", start)
 		}