1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Optional Userland Proxy

- Port https://github.com/docker/docker/pull/12165 to libnetwork
- More tests will be added later

Signed-off-by: Alessandro Boch <aboch@docker.com>
This commit is contained in:
Alessandro Boch 2015-05-18 16:49:12 -07:00
parent 32b1657a28
commit 902e8746d3
12 changed files with 122 additions and 75 deletions

View file

@ -51,6 +51,7 @@ type NetworkConfiguration struct {
DefaultGatewayIPv6 net.IP
DefaultBindingIP net.IP
AllowNonDefaultBridge bool
EnableUserlandProxy bool
}
// EndpointConfiguration represents the user specified configuration for the sandbox endpoint
@ -309,6 +310,9 @@ func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) err
// specified subnet.
{config.FixedCIDRv6 != nil, setupFixedCIDRv6},
// Setup Loopback Adresses Routing
{!config.EnableUserlandProxy, setupLoopbackAdressesRouting},
// Setup IPTables.
{config.EnableIPTables, setupIPTables},
@ -557,7 +561,7 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn
}
// Program any required port mapping and store them in the endpoint
endpoint.portMapping, err = allocatePorts(epConfig, intf, config.DefaultBindingIP)
endpoint.portMapping, err = allocatePorts(epConfig, intf, config.DefaultBindingIP, config.EnableUserlandProxy)
if err != nil {
return err
}

View file

@ -157,14 +157,23 @@ func (te *testEndpoint) SetResolvConfPath(path string) error {
}
func TestQueryEndpointInfo(t *testing.T) {
testQueryEndpointInfo(t, true)
}
func TestQueryEndpointInfoHairpin(t *testing.T) {
testQueryEndpointInfo(t, false)
}
func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) {
defer netutils.SetupTestNetNS(t)()
d := newDriver()
dd, _ := d.(*driver)
config := &NetworkConfiguration{
BridgeName: DefaultBridgeName,
EnableIPTables: true,
EnableICC: false,
BridgeName: DefaultBridgeName,
EnableIPTables: true,
EnableICC: false,
EnableUserlandProxy: ulPxyEnabled,
}
genericOption := make(map[string]interface{})
genericOption[netlabel.GenericData] = config

View file

@ -15,7 +15,7 @@ var (
defaultBindingIP = net.IPv4(0, 0, 0, 0)
)
func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, reqDefBindIP net.IP) ([]netutils.PortBinding, error) {
func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, reqDefBindIP net.IP, ulPxyEnabled bool) ([]netutils.PortBinding, error) {
if epConfig == nil || epConfig.PortBindings == nil {
return nil, nil
}
@ -25,14 +25,14 @@ func allocatePorts(epConfig *EndpointConfiguration, intf *sandbox.Interface, req
defHostIP = reqDefBindIP
}
return allocatePortsInternal(epConfig.PortBindings, intf.Address.IP, defHostIP)
return allocatePortsInternal(epConfig.PortBindings, intf.Address.IP, defHostIP, ulPxyEnabled)
}
func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHostIP net.IP) ([]netutils.PortBinding, error) {
func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]netutils.PortBinding, error) {
bs := make([]netutils.PortBinding, 0, len(bindings))
for _, c := range bindings {
b := c.GetCopy()
if err := allocatePort(&b, containerIP, defHostIP); err != nil {
if err := allocatePort(&b, containerIP, defHostIP, ulPxyEnabled); err != nil {
// On allocation failure, release previously allocated ports. On cleanup error, just log a warning message
if cuErr := releasePortsInternal(bs); cuErr != nil {
logrus.Warnf("Upon allocation failure for %v, failed to clear previously allocated port bindings: %v", b, cuErr)
@ -44,7 +44,7 @@ func allocatePortsInternal(bindings []netutils.PortBinding, containerIP, defHost
return bs, nil
}
func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP) error {
func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) error {
var (
host net.Addr
err error
@ -66,7 +66,7 @@ func allocatePort(bnd *netutils.PortBinding, containerIP, defHostIP net.IP) erro
// Try up to maxAllocatePortAttempts times to get a port that's not already allocated.
for i := 0; i < maxAllocatePortAttempts; i++ {
if host, err = portMapper.Map(container, bnd.HostIP, int(bnd.HostPort)); err == nil {
if host, err = portMapper.Map(container, bnd.HostIP, int(bnd.HostPort), ulPxyEnabled); err == nil {
break
}
// There is no point in immediately retrying to map an explicitly chosen port.

View file

@ -19,20 +19,22 @@ func setupIPTables(config *NetworkConfiguration, i *bridgeInterface) error {
return ipTableCfgError(config.BridgeName)
}
hairpinMode := !config.EnableUserlandProxy
addrv4, _, err := netutils.GetIfaceAddr(config.BridgeName)
if err != nil {
return fmt.Errorf("Failed to setup IP tables, cannot acquire Interface address: %s", err.Error())
}
if err = setupIPTablesInternal(config.BridgeName, addrv4, config.EnableICC, config.EnableIPMasquerade, true); err != nil {
if err = setupIPTablesInternal(config.BridgeName, addrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, true); err != nil {
return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
}
_, err = iptables.NewChain(DockerChain, config.BridgeName, iptables.Nat)
_, err = iptables.NewChain(DockerChain, config.BridgeName, iptables.Nat, hairpinMode)
if err != nil {
return fmt.Errorf("Failed to create NAT chain: %s", err.Error())
}
chain, err := iptables.NewChain(DockerChain, config.BridgeName, iptables.Filter)
chain, err := iptables.NewChain(DockerChain, config.BridgeName, iptables.Filter, hairpinMode)
if err != nil {
return fmt.Errorf("Failed to create FILTER chain: %s", err.Error())
}
@ -49,13 +51,14 @@ type iptRule struct {
args []string
}
func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, enable bool) error {
func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairpin, enable bool) error {
var (
address = addr.String()
natRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
outRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
inRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
address = addr.String()
natRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}}
outRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
inRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
)
// Set NAT.
@ -65,6 +68,13 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, enabl
}
}
// In hairpin mode, masquerade traffic from localhost
if hairpin {
if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil {
return err
}
}
// Set Inter Container Communication.
if err := setIcc(bridgeIface, icc, enable); err != nil {
return err

View file

@ -1,8 +1,12 @@
package bridge
import (
"fmt"
"io/ioutil"
"net"
"path/filepath"
log "github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/netutils"
"github.com/vishvananda/netlink"
@ -121,3 +125,12 @@ func setupGatewayIPv4(config *NetworkConfiguration, i *bridgeInterface) error {
return nil
}
func setupLoopbackAdressesRouting(config *NetworkConfiguration, i *bridgeInterface) error {
// Enable loopback adresses routing
sysPath := filepath.Join("/proc/sys/net/ipv4/conf", config.BridgeName, "route_localnet")
if err := ioutil.WriteFile(sysPath, []byte{'1', '\n'}, 0644); err != nil {
return fmt.Errorf("Unable to enable local routing for hairpin mode: %v", err)
}
return nil
}

View file

@ -14,8 +14,8 @@ type IPV string
const (
// Iptables point ipv4 table
Iptables IPV = "ipv4"
// IP6tables point to ipv6 table
IP6tables IPV = "ipv6"
// IP6Tables point to ipv6 table
IP6Tables IPV = "ipv6"
// Ebtables point to bridge table
Ebtables IPV = "eb"
)
@ -156,7 +156,6 @@ func checkRunning() bool {
// Passthrough method simply passes args through to iptables/ip6tables
func Passthrough(ipv IPV, args ...string) ([]byte, error) {
var output string
logrus.Debugf("Firewalld passthrough: %s, %s", ipv, args)
if err := connection.sysobj.Call(dbusInterface+".direct.passthrough", 0, ipv, args).Store(&output); err != nil {
return nil, err

View file

@ -7,14 +7,19 @@ import (
)
func TestFirewalldInit(t *testing.T) {
FirewalldInit()
if !checkRunning() {
t.Skip("firewalld is not running")
}
if err := FirewalldInit(); err != nil {
t.Fatal(err)
}
}
func TestReloaded(t *testing.T) {
var err error
var fwdChain *Chain
fwdChain, err = NewChain("FWD", "lo", Filter)
fwdChain, err = NewChain("FWD", "lo", Filter, false)
if err != nil {
t.Fatal(err)
}

View file

@ -13,24 +13,24 @@ import (
"github.com/Sirupsen/logrus"
)
//Action signifies the iptable action.
// Action signifies the iptable action.
type Action string
//Table refers to Nat, Filter or Mangle.
// Table refers to Nat, Filter or Mangle.
type Table string
const (
//Append appends the rule at the end of the chain.
// Append appends the rule at the end of the chain.
Append Action = "-A"
//Delete deletes the rule from the chain.
// Delete deletes the rule from the chain.
Delete Action = "-D"
//Insert inserts the rule at the top of the chain.
// Insert inserts the rule at the top of the chain.
Insert Action = "-I"
//Nat table is used for nat translation rules.
// Nat table is used for nat translation rules.
Nat Table = "nat"
//Filter table is used for filter rules.
// Filter table is used for filter rules.
Filter Table = "filter"
//Mangle table is used for mangling the packet.
// Mangle table is used for mangling the packet.
Mangle Table = "mangle"
)
@ -39,18 +39,18 @@ var (
supportsXlock = false
// used to lock iptables commands if xtables lock is not supported
bestEffortLock sync.Mutex
//ErrIptablesNotFound is returned when the rule is not found.
// ErrIptablesNotFound is returned when the rule is not found.
ErrIptablesNotFound = errors.New("Iptables not found")
)
//Chain defines the iptables chain.
// Chain defines the iptables chain.
type Chain struct {
Name string
Bridge string
Table Table
}
//ChainError is returned to represent errors during ip table operation.
// ChainError is returned to represent errors during ip table operation.
type ChainError struct {
Chain string
Output []byte
@ -73,8 +73,8 @@ func initCheck() error {
return nil
}
//NewChain adds a new chain to ip table.
func NewChain(name, bridge string, table Table) (*Chain, error) {
// NewChain adds a new chain to ip table.
func NewChain(name, bridge string, table Table, hairpinMode bool) (*Chain, error) {
c := &Chain{
Name: name,
Bridge: bridge,
@ -106,8 +106,10 @@ func NewChain(name, bridge string, table Table) (*Chain, error) {
}
output := []string{
"-m", "addrtype",
"--dst-type", "LOCAL",
"!", "--dst", "127.0.0.0/8"}
"--dst-type", "LOCAL"}
if !hairpinMode {
output = append(output, "!", "--dst", "127.0.0.0/8")
}
if !Exists(Nat, "OUTPUT", output...) {
if err := c.Output(Append, output...); err != nil {
return nil, fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
@ -129,7 +131,7 @@ func NewChain(name, bridge string, table Table) (*Chain, error) {
return c, nil
}
//RemoveExistingChain removes existing chain from the table.
// RemoveExistingChain removes existing chain from the table.
func RemoveExistingChain(name string, table Table) error {
c := &Chain{
Name: name,
@ -141,7 +143,7 @@ func RemoveExistingChain(name string, table Table) error {
return c.Remove()
}
//Forward adds forwarding rule to 'filter' table and corresponding nat rule to 'nat' table
// Forward adds forwarding rule to 'filter' table and corresponding nat rule to 'nat' table.
func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr string, destPort int) error {
daddr := ip.String()
if ip.IsUnspecified() {
@ -154,7 +156,6 @@ func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr stri
"-p", proto,
"-d", daddr,
"--dport", strconv.Itoa(port),
"!", "-i", c.Bridge,
"-j", "DNAT",
"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))); err != nil {
return err
@ -188,7 +189,7 @@ func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr stri
return nil
}
//Link adds reciprocal ACCEPT rule for two supplied IP addresses.
// Link adds reciprocal ACCEPT rule for two supplied IP addresses.
// Traffic is allowed from ip1 to ip2 and vice-versa
func (c *Chain) Link(action Action, ip1, ip2 net.IP, port int, proto string) error {
if output, err := Raw("-t", string(Filter), string(action), c.Name,
@ -216,7 +217,7 @@ func (c *Chain) Link(action Action, ip1, ip2 net.IP, port int, proto string) err
return nil
}
//Prerouting adds linking rule to nat/PREROUTING chain.
// Prerouting adds linking rule to nat/PREROUTING chain.
func (c *Chain) Prerouting(action Action, args ...string) error {
a := []string{"-t", string(Nat), string(action), "PREROUTING"}
if len(args) > 0 {
@ -230,7 +231,7 @@ func (c *Chain) Prerouting(action Action, args ...string) error {
return nil
}
//Output adds linking rule to an OUTPUT chain
// Output adds linking rule to an OUTPUT chain.
func (c *Chain) Output(action Action, args ...string) error {
a := []string{"-t", string(c.Table), string(action), "OUTPUT"}
if len(args) > 0 {
@ -244,7 +245,7 @@ func (c *Chain) Output(action Action, args ...string) error {
return nil
}
// Remove removes the chain
// Remove removes the chain.
func (c *Chain) Remove() error {
// Ignore errors - This could mean the chains were never set up
if c.Table == Nat {
@ -260,7 +261,7 @@ func (c *Chain) Remove() error {
return nil
}
//Exists checks if a rule exists
// Exists checks if a rule exists
func Exists(table Table, chain string, rule ...string) bool {
if string(table) == "" {
table = Filter
@ -291,7 +292,7 @@ func Exists(table Table, chain string, rule ...string) bool {
)
}
//Raw calls 'iptables' system command, passing supplied arguments
// Raw calls 'iptables' system command, passing supplied arguments.
func Raw(args ...string) ([]byte, error) {
if firewalldRunning {
output, err := Passthrough(Iptables, args...)

View file

@ -7,11 +7,9 @@ import (
"strings"
"sync"
"testing"
_ "github.com/docker/libnetwork/netutils"
)
const chainName = "DOCKERTEST"
const chainName = "DOCKER-TEST"
var natChain *Chain
var filterChain *Chain
@ -19,12 +17,12 @@ var filterChain *Chain
func TestNewChain(t *testing.T) {
var err error
natChain, err = NewChain(chainName, "lo", Nat)
natChain, err = NewChain(chainName, "lo", Nat, false)
if err != nil {
t.Fatal(err)
}
filterChain, err = NewChain(chainName, "lo", Filter)
filterChain, err = NewChain(chainName, "lo", Filter, false)
if err != nil {
t.Fatal(err)
}
@ -43,7 +41,6 @@ func TestForward(t *testing.T) {
}
dnatRule := []string{
"!", "-i", filterChain.Bridge,
"-d", ip.String(),
"-p", proto,
"--dport", strconv.Itoa(port),

View file

@ -579,7 +579,7 @@ func TestControllerQuery(t *testing.T) {
g, err := controller.NetworkByID("network1")
if err == nil {
t.Fatalf("Unexpected success for NetworkByID(): %g", g)
t.Fatalf("Unexpected success for NetworkByID(): %v", g)
}
if err != libnetwork.ErrNoSuchNetwork {
t.Fatalf("NetworkByID() failed with unexpected error: %v", err)

View file

@ -59,7 +59,7 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.Chain) {
}
// Map maps the specified container transport address to the host's network address and transport port
func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host net.Addr, err error) {
func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
pm.lock.Lock()
defer pm.lock.Unlock()
@ -67,7 +67,6 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
m *mapping
proto string
allocatedHostPort int
proxy userlandProxy
)
switch container.(type) {
@ -83,7 +82,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
container: container,
}
proxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
if useProxy {
m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
}
case *net.UDPAddr:
proto = "udp"
if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil {
@ -96,7 +97,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
container: container,
}
proxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
if useProxy {
m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
}
default:
return nil, ErrUnknownBackendAddressType
}
@ -120,7 +123,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
cleanup := func() error {
// need to undo the iptables rules before we return
proxy.Stop()
if m.userlandProxy != nil {
m.userlandProxy.Stop()
}
pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
return err
@ -129,13 +134,15 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
return nil
}
if err := proxy.Start(); err != nil {
if err := cleanup(); err != nil {
return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
if m.userlandProxy != nil {
if err := m.userlandProxy.Start(); err != nil {
if err := cleanup(); err != nil {
return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
}
return nil, err
}
return nil, err
}
m.userlandProxy = proxy
pm.currentMappings[key] = m
return m.host, nil
}
@ -151,7 +158,9 @@ func (pm *PortMapper) Unmap(host net.Addr) error {
return ErrPortNotMapped
}
data.userlandProxy.Stop()
if data.userlandProxy != nil {
data.userlandProxy.Stop()
}
delete(pm.currentMappings, key)

View file

@ -51,22 +51,22 @@ func TestMapTCPPorts(t *testing.T) {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr1, dstIP1, 80); err != nil {
if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr1, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
}
if _, err := pm.Map(srcAddr1, dstIP1, 80); err == nil {
if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP1, 80); err == nil {
if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP2, 80); err != nil {
if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
}
@ -131,22 +131,22 @@ func TestMapUDPPorts(t *testing.T) {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr1, dstIP1, 80); err != nil {
if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr1, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
}
if _, err := pm.Map(srcAddr1, dstIP1, 80); err == nil {
if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP1, 80); err == nil {
if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP2, 80); err != nil {
if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
}
@ -180,14 +180,14 @@ func TestMapAllPortsSingleInterface(t *testing.T) {
for i := 0; i < 10; i++ {
start, end := pm.Allocator.Begin, pm.Allocator.End
for i := start; i < end; i++ {
if host, err = pm.Map(srcAddr1, dstIP1, 0); err != nil {
if host, err = pm.Map(srcAddr1, dstIP1, 0, true); err != nil {
t.Fatal(err)
}
hosts = append(hosts, host)
}
if _, err := pm.Map(srcAddr1, dstIP1, start); err == nil {
if _, err := pm.Map(srcAddr1, dstIP1, start, true); err == nil {
t.Fatalf("Port %d should be bound but is not", start)
}