Allow drivers to supply static routes for interfaces

Signed-off-by: Tom Denham <tom.denham@metaswitch.com>

libnetwork/driverapi/driverapi.go

@@ -104,6 +104,10 @@ type JoinInfo interface {
 	// SetGatewayIPv6 sets the default IPv6 gateway when a container joins the endpoint.
 	SetGatewayIPv6(net.IP) error
 
+	// AddStaticRoute adds a routes to the sandbox.
+	// It may be used in addtion to or instead of a default gateway (as above).
+	AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error
+
 	// SetHostsPath sets the overriding /etc/hosts path to use for the container.
 	SetHostsPath(string) error
 

libnetwork/drivers/bridge/bridge_test.go

@@ -87,6 +87,7 @@ type testEndpoint struct {
 	gw6            net.IP
 	hostsPath      string
 	resolvConfPath string
+	routes         []types.StaticRoute
 }
 
 func (te *testEndpoint) Interfaces() []driverapi.InterfaceInfo {
@@ -157,6 +158,11 @@ func (te *testEndpoint) SetResolvConfPath(path string) error {
 	return nil
 }
 
+func (te *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {
+	te.routes = append(te.routes, types.StaticRoute{destination, routeType, nextHop, interfaceID})
+	return nil
+}
+
 func TestQueryEndpointInfo(t *testing.T) {
 	testQueryEndpointInfo(t, true)
 }

libnetwork/drivers/remote/driver_test.go

@@ -149,6 +149,11 @@ func (test *testEndpoint) SetNames(src string, dst string) error {
 	return nil
 }
 
+func (test *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {
+	//TODO
+	return nil
+}
+
 func (test *testEndpoint) ID() int {
 	return test.id
 }

libnetwork/endpoint.go

@@ -293,6 +293,7 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai
 			SrcName: i.srcName,
 			DstName: i.dstPrefix,
 			Address: &i.addr,
+			Routes:  i.routes,
 		}
 		if i.addrv6.IP.To16() != nil {
 			iface.AddressIPv6 = &i.addrv6
@@ -302,6 +303,13 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai
 			return nil, err
 		}
 	}
+	// Set up non-interface routes.
+	for _, r := range ep.joinInfo.StaticRoutes {
+		err = sb.AddStaticRoute(r)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	err = sb.SetGateway(joinInfo.gw)
 	if err != nil {
@@ -360,6 +368,14 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error {
 		}
 	}
 
+	// Remove non-interface routes.
+	for _, r := range ep.joinInfo.StaticRoutes {
+		err = sb.RemoveStaticRoute(r)
+		if err != nil {
+			logrus.Debugf("Remove route failed: %v", err)
+		}
+	}
+
 	ctrlr.sandboxRm(container.data.SandboxKey)
 
 	return err

libnetwork/endpoint_info.go

@@ -46,6 +46,7 @@ type endpointInterface struct {
 	addrv6    net.IPNet
 	srcName   string
 	dstPrefix string
+	routes    []*net.IPNet
 }
 
 type endpointJoinInfo struct {
@@ -53,6 +54,7 @@ type endpointJoinInfo struct {
 	gw6            net.IP
 	hostsPath      string
 	resolvConfPath string
+	StaticRoutes   []*types.StaticRoute
 }
 
 func (ep *endpoint) Info() EndpointInfo {
@@ -149,6 +151,35 @@ func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo {
 	return iList
 }
 
+func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {
+	ep.Lock()
+	defer ep.Unlock()
+
+	r := types.StaticRoute{destination, routeType, nextHop, interfaceID}
+
+	if routeType == types.NEXTHOP {
+		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).
+		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)
+	} else {
+		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.
+		if err := ep.addInterfaceRoute(&r); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ep *endpoint) addInterfaceRoute(route *types.StaticRoute) error {
+	for _, iface := range ep.iFaces {
+		if iface.id == route.InterfaceID {
+			iface.routes = append(iface.routes, route.Destination)
+			return nil
+		}
+	}
+	return types.BadRequestErrorf("Interface with ID %d doesn't exist.",
+		route.InterfaceID)
+}
+
 func (ep *endpoint) SandboxKey() string {
 	ep.Lock()
 	defer ep.Unlock()

libnetwork/sandbox/configure_linux.go

@@ -19,6 +19,7 @@ func configureInterface(iface netlink.Link, settings *Interface) error {
 		{setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, settings.DstName)},
 		{setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, settings.Address)},
 		{setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, settings.AddressIPv6)},
+		{setInterfaceRoutes, fmt.Sprintf("error setting interface %q routes to %q", ifaceName, settings.Routes)},
 	}
 
 	for _, config := range ifaceConfigurators {
@@ -63,6 +64,78 @@ func programGateway(path string, gw net.IP) error {
 	})
 }
 
+// Program a route in to the namespace routing table.
+func programRoute(path string, dest *net.IPNet, nh net.IP) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	origns, err := netns.Get()
+	if err != nil {
+		return err
+	}
+	defer origns.Close()
+
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return fmt.Errorf("failed get network namespace %q: %v", path, err)
+	}
+	defer f.Close()
+
+	nsFD := f.Fd()
+	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
+		return err
+	}
+	defer netns.Set(origns)
+
+	gwRoutes, err := netlink.RouteGet(nh)
+	if err != nil {
+		return fmt.Errorf("route for the next hop could not be found: %v", err)
+	}
+
+	return netlink.RouteAdd(&netlink.Route{
+		Scope:     netlink.SCOPE_UNIVERSE,
+		LinkIndex: gwRoutes[0].LinkIndex,
+		Gw:        gwRoutes[0].Gw,
+		Dst:       dest,
+	})
+}
+
+// Delete a route from the namespace routing table.
+func removeRoute(path string, dest *net.IPNet, nh net.IP) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	origns, err := netns.Get()
+	if err != nil {
+		return err
+	}
+	defer origns.Close()
+
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return fmt.Errorf("failed get network namespace %q: %v", path, err)
+	}
+	defer f.Close()
+
+	nsFD := f.Fd()
+	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
+		return err
+	}
+	defer netns.Set(origns)
+
+	gwRoutes, err := netlink.RouteGet(nh)
+	if err != nil {
+		return fmt.Errorf("route for the next hop could not be found: %v", err)
+	}
+
+	return netlink.RouteDel(&netlink.Route{
+		Scope:     netlink.SCOPE_UNIVERSE,
+		LinkIndex: gwRoutes[0].LinkIndex,
+		Gw:        gwRoutes[0].Gw,
+		Dst:       dest,
+	})
+}
+
 func setInterfaceIP(iface netlink.Link, settings *Interface) error {
 	ipAddr := &netlink.Addr{IPNet: settings.Address, Label: ""}
 	return netlink.AddrAdd(iface, ipAddr)
@@ -79,3 +152,17 @@ func setInterfaceIPv6(iface netlink.Link, settings *Interface) error {
 func setInterfaceName(iface netlink.Link, settings *Interface) error {
 	return netlink.LinkSetName(iface, settings.DstName)
 }
+
+func setInterfaceRoutes(iface netlink.Link, settings *Interface) error {
+	for _, route := range settings.Routes {
+		err := netlink.RouteAdd(&netlink.Route{
+			Scope:     netlink.SCOPE_UNIVERSE,
+			LinkIndex: iface.Attrs().Index,
+			Dst:       route,
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

libnetwork/sandbox/namespace_linux.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 )
@@ -263,6 +264,35 @@ func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error {
 	return err
 }
 
+func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error {
+	err := programRoute(n.path, r.Destination, r.NextHop)
+	if err == nil {
+		n.Lock()
+		n.sinfo.StaticRoutes = append(n.sinfo.StaticRoutes, r)
+		n.Unlock()
+	}
+	return err
+}
+
+func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error {
+	err := removeRoute(n.path, r.Destination, r.NextHop)
+	if err == nil {
+		n.Lock()
+		lastIndex := len(n.sinfo.StaticRoutes) - 1
+		for i, v := range n.sinfo.StaticRoutes {
+			if v == r {
+				// Overwrite the route we're removing with the last element
+				n.sinfo.StaticRoutes[i] = n.sinfo.StaticRoutes[lastIndex]
+				// Shorten the slice to trim the extra element
+				n.sinfo.StaticRoutes = n.sinfo.StaticRoutes[:lastIndex]
+				break
+			}
+		}
+		n.Unlock()
+	}
+	return err
+}
+
 func (n *networkNamespace) Interfaces() []*Interface {
 	n.Lock()
 	defer n.Unlock()

libnetwork/sandbox/sandbox.go

@@ -35,6 +35,12 @@ type Sandbox interface {
 	// Set default IPv6 gateway for the sandbox
 	SetGatewayIPv6(gw net.IP) error
 
+	// Add a static route to the sandbox.
+	AddStaticRoute(*types.StaticRoute) error
+
+	// Remove a static route from the sandbox.
+	RemoveStaticRoute(*types.StaticRoute) error
+
 	// Destroy the sandbox
 	Destroy() error
 }
@@ -51,7 +57,11 @@ type Info struct {
 	// IPv6 gateway for the sandbox.
 	GatewayIPv6 net.IP
 
-	// TODO: Add routes and ip tables etc.
+	// Additional static routes for the sandbox.  (Note that directly
+	// connected routes are stored on the particular interface they refer to.)
+	StaticRoutes []*types.StaticRoute
+
+	// TODO: Add ip tables etc.
 }
 
 // Interface represents the settings and identity of a network device. It is
@@ -74,15 +84,25 @@ type Interface struct {
 
 	// IPv6 address for the interface.
 	AddressIPv6 *net.IPNet
+
+	// IP routes for the interface.
+	Routes []*net.IPNet
 }
 
 // GetCopy returns a copy of this Interface structure
 func (i *Interface) GetCopy() *Interface {
+	copiedRoutes := make([]*net.IPNet, len(i.Routes))
+
+	for index := range i.Routes {
+		copiedRoutes[index] = types.GetIPNetCopy(i.Routes[index])
+	}
+
 	return &Interface{
 		SrcName:     i.SrcName,
 		DstName:     i.DstName,
 		Address:     types.GetIPNetCopy(i.Address),
 		AddressIPv6: types.GetIPNetCopy(i.AddressIPv6),
+		Routes:      copiedRoutes,
 	}
 }
 
@@ -108,6 +128,16 @@ func (i *Interface) Equal(o *Interface) bool {
 		return false
 	}
 
+	if len(i.Routes) != len(o.Routes) {
+		return false
+	}
+
+	for index := range i.Routes {
+		if !types.CompareIPNet(i.Routes[index], o.Routes[index]) {
+			return false
+		}
+	}
+
 	return true
 }
 
@@ -120,7 +150,15 @@ func (s *Info) GetCopy() *Info {
 	gw := types.GetIPCopy(s.Gateway)
 	gw6 := types.GetIPCopy(s.GatewayIPv6)
 
-	return &Info{Interfaces: list, Gateway: gw, GatewayIPv6: gw6}
+	routes := make([]*types.StaticRoute, len(s.StaticRoutes))
+	for i, r := range s.StaticRoutes {
+		routes[i] = r.GetCopy()
+	}
+
+	return &Info{Interfaces: list,
+		Gateway:      gw,
+		GatewayIPv6:  gw6,
+		StaticRoutes: routes}
 }
 
 // Equal checks if this instance of SandboxInfo is equal to the passed one
@@ -154,6 +192,17 @@ func (s *Info) Equal(o *Info) bool {
 		}
 	}
 
+	for index := range s.StaticRoutes {
+		ss := s.StaticRoutes[index]
+		oo := o.StaticRoutes[index]
+		if !types.CompareIPNet(ss.Destination, oo.Destination) {
+			return false
+		}
+		if !ss.NextHop.Equal(oo.NextHop) {
+			return false
+		}
+	}
+
 	return true
 
 }

libnetwork/sandbox/sandbox_linux_test.go

@@ -63,6 +63,13 @@ func newInfo(t *testing.T) (*Info, error) {
 	intf1.AddressIPv6 = addrv6
 	intf1.AddressIPv6.IP = ip6
 
+	_, route, err := net.ParseCIDR("192.168.2.1/32")
+	if err != nil {
+		return nil, err
+	}
+
+	intf1.Routes = []*net.IPNet{route}
+
 	veth = &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0},
 		PeerName:  vethName4}
@@ -92,6 +99,7 @@ func newInfo(t *testing.T) (*Info, error) {
 	intf2.AddressIPv6.IP = ip6
 
 	sinfo := &Info{Interfaces: []*Interface{intf1, intf2}}
+
 	sinfo.Gateway = net.ParseIP("192.168.1.1")
 	// sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1")
 	sinfo.GatewayIPv6 = net.ParseIP("fe80::1")

libnetwork/sandbox/sandbox_test.go

@@ -190,12 +190,14 @@ func getInterfaceList() []*Interface {
 			DstName:     "eth0",
 			Address:     netv4a,
 			AddressIPv6: netv6a,
+			Routes:      []*net.IPNet{netv4a, netv6a},
 		},
 		&Interface{
 			SrcName:     "veth7654321",
 			DstName:     "eth1",
 			Address:     netv4b,
 			AddressIPv6: netv6b,
+			Routes:      []*net.IPNet{netv4b, netv6b},
 		},
 	}
 }

libnetwork/types/types.go

@@ -184,6 +184,40 @@ func CompareIPNet(a, b *net.IPNet) bool {
 	return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask)
 }
 
+const (
+	// NEXTHOP indicates a StaticRoute with an IP next hop.
+	NEXTHOP = iota
+
+	// CONNECTED indicates a StaticRoute with a interface for directly connected peers.
+	CONNECTED
+)
+
+// StaticRoute is a statically-provisioned IP route.
+type StaticRoute struct {
+	Destination *net.IPNet
+
+	RouteType int // NEXT_HOP or CONNECTED
+
+	// NextHop will be resolved by the kernel (i.e. as a loose hop).
+	NextHop net.IP
+
+	// InterfaceID must refer to a defined interface on the
+	// Endpoint to which the routes are specified.  Routes specified this way
+	// are interpreted as directly connected to the specified interface (no
+	// next hop will be used).
+	InterfaceID int
+}
+
+// GetCopy returns a copy of this StaticRoute structure
+func (r *StaticRoute) GetCopy() *StaticRoute {
+	d := GetIPNetCopy(r.Destination)
+	nh := GetIPCopy(r.NextHop)
+	return &StaticRoute{Destination: d,
+		RouteType:   r.RouteType,
+		NextHop:     nh,
+		InterfaceID: r.InterfaceID}
+}
+
 /******************************
  * Well-known Error Interfaces
  ******************************/