From 65acaaf0b5e6cbc81758fd97b38bfe2cc56202ed Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Tue, 19 May 2015 17:08:56 -0700 Subject: [PATCH] Allow drivers to supply static routes for interfaces Signed-off-by: Tom Denham --- libnetwork/driverapi/driverapi.go | 4 ++ libnetwork/drivers/bridge/bridge_test.go | 6 ++ libnetwork/drivers/remote/driver_test.go | 5 ++ libnetwork/endpoint.go | 16 +++++ libnetwork/endpoint_info.go | 31 +++++++++ libnetwork/sandbox/configure_linux.go | 87 ++++++++++++++++++++++++ libnetwork/sandbox/namespace_linux.go | 30 ++++++++ libnetwork/sandbox/sandbox.go | 53 ++++++++++++++- libnetwork/sandbox/sandbox_linux_test.go | 8 +++ libnetwork/sandbox/sandbox_test.go | 2 + libnetwork/types/types.go | 34 +++++++++ 11 files changed, 274 insertions(+), 2 deletions(-) diff --git a/libnetwork/driverapi/driverapi.go b/libnetwork/driverapi/driverapi.go index 9fb41ff7b6..45c136dc45 100644 --- a/libnetwork/driverapi/driverapi.go +++ b/libnetwork/driverapi/driverapi.go @@ -104,6 +104,10 @@ type JoinInfo interface { // SetGatewayIPv6 sets the default IPv6 gateway when a container joins the endpoint. SetGatewayIPv6(net.IP) error + // AddStaticRoute adds a routes to the sandbox. + // It may be used in addtion to or instead of a default gateway (as above). + AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error + // SetHostsPath sets the overriding /etc/hosts path to use for the container. SetHostsPath(string) error diff --git a/libnetwork/drivers/bridge/bridge_test.go b/libnetwork/drivers/bridge/bridge_test.go index 28aee2e5e8..aba46977e6 100644 --- a/libnetwork/drivers/bridge/bridge_test.go +++ b/libnetwork/drivers/bridge/bridge_test.go @@ -87,6 +87,7 @@ type testEndpoint struct { gw6 net.IP hostsPath string resolvConfPath string + routes []types.StaticRoute } func (te *testEndpoint) Interfaces() []driverapi.InterfaceInfo { @@ -157,6 +158,11 @@ func (te *testEndpoint) SetResolvConfPath(path string) error { return nil } +func (te *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error { + te.routes = append(te.routes, types.StaticRoute{destination, routeType, nextHop, interfaceID}) + return nil +} + func TestQueryEndpointInfo(t *testing.T) { testQueryEndpointInfo(t, true) } diff --git a/libnetwork/drivers/remote/driver_test.go b/libnetwork/drivers/remote/driver_test.go index a9fb8b4c16..27703ea890 100644 --- a/libnetwork/drivers/remote/driver_test.go +++ b/libnetwork/drivers/remote/driver_test.go @@ -149,6 +149,11 @@ func (test *testEndpoint) SetNames(src string, dst string) error { return nil } +func (test *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error { + //TODO + return nil +} + func (test *testEndpoint) ID() int { return test.id } diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index 2935bca31c..5abe9a7e61 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -293,6 +293,7 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai SrcName: i.srcName, DstName: i.dstPrefix, Address: &i.addr, + Routes: i.routes, } if i.addrv6.IP.To16() != nil { iface.AddressIPv6 = &i.addrv6 @@ -302,6 +303,13 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai return nil, err } } + // Set up non-interface routes. + for _, r := range ep.joinInfo.StaticRoutes { + err = sb.AddStaticRoute(r) + if err != nil { + return nil, err + } + } err = sb.SetGateway(joinInfo.gw) if err != nil { @@ -360,6 +368,14 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error { } } + // Remove non-interface routes. + for _, r := range ep.joinInfo.StaticRoutes { + err = sb.RemoveStaticRoute(r) + if err != nil { + logrus.Debugf("Remove route failed: %v", err) + } + } + ctrlr.sandboxRm(container.data.SandboxKey) return err diff --git a/libnetwork/endpoint_info.go b/libnetwork/endpoint_info.go index f04521595a..1c7a85cce6 100644 --- a/libnetwork/endpoint_info.go +++ b/libnetwork/endpoint_info.go @@ -46,6 +46,7 @@ type endpointInterface struct { addrv6 net.IPNet srcName string dstPrefix string + routes []*net.IPNet } type endpointJoinInfo struct { @@ -53,6 +54,7 @@ type endpointJoinInfo struct { gw6 net.IP hostsPath string resolvConfPath string + StaticRoutes []*types.StaticRoute } func (ep *endpoint) Info() EndpointInfo { @@ -149,6 +151,35 @@ func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo { return iList } +func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error { + ep.Lock() + defer ep.Unlock() + + r := types.StaticRoute{destination, routeType, nextHop, interfaceID} + + if routeType == types.NEXTHOP { + // If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface). + ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r) + } else { + // If the route doesn't specify a next-hop, it must be a connected route, bound to an interface. + if err := ep.addInterfaceRoute(&r); err != nil { + return err + } + } + return nil +} + +func (ep *endpoint) addInterfaceRoute(route *types.StaticRoute) error { + for _, iface := range ep.iFaces { + if iface.id == route.InterfaceID { + iface.routes = append(iface.routes, route.Destination) + return nil + } + } + return types.BadRequestErrorf("Interface with ID %d doesn't exist.", + route.InterfaceID) +} + func (ep *endpoint) SandboxKey() string { ep.Lock() defer ep.Unlock() diff --git a/libnetwork/sandbox/configure_linux.go b/libnetwork/sandbox/configure_linux.go index cae77890fd..2a6984ba47 100644 --- a/libnetwork/sandbox/configure_linux.go +++ b/libnetwork/sandbox/configure_linux.go @@ -19,6 +19,7 @@ func configureInterface(iface netlink.Link, settings *Interface) error { {setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, settings.DstName)}, {setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, settings.Address)}, {setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, settings.AddressIPv6)}, + {setInterfaceRoutes, fmt.Sprintf("error setting interface %q routes to %q", ifaceName, settings.Routes)}, } for _, config := range ifaceConfigurators { @@ -63,6 +64,78 @@ func programGateway(path string, gw net.IP) error { }) } +// Program a route in to the namespace routing table. +func programRoute(path string, dest *net.IPNet, nh net.IP) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + origns, err := netns.Get() + if err != nil { + return err + } + defer origns.Close() + + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed get network namespace %q: %v", path, err) + } + defer f.Close() + + nsFD := f.Fd() + if err = netns.Set(netns.NsHandle(nsFD)); err != nil { + return err + } + defer netns.Set(origns) + + gwRoutes, err := netlink.RouteGet(nh) + if err != nil { + return fmt.Errorf("route for the next hop could not be found: %v", err) + } + + return netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gwRoutes[0].Gw, + Dst: dest, + }) +} + +// Delete a route from the namespace routing table. +func removeRoute(path string, dest *net.IPNet, nh net.IP) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + origns, err := netns.Get() + if err != nil { + return err + } + defer origns.Close() + + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed get network namespace %q: %v", path, err) + } + defer f.Close() + + nsFD := f.Fd() + if err = netns.Set(netns.NsHandle(nsFD)); err != nil { + return err + } + defer netns.Set(origns) + + gwRoutes, err := netlink.RouteGet(nh) + if err != nil { + return fmt.Errorf("route for the next hop could not be found: %v", err) + } + + return netlink.RouteDel(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gwRoutes[0].Gw, + Dst: dest, + }) +} + func setInterfaceIP(iface netlink.Link, settings *Interface) error { ipAddr := &netlink.Addr{IPNet: settings.Address, Label: ""} return netlink.AddrAdd(iface, ipAddr) @@ -79,3 +152,17 @@ func setInterfaceIPv6(iface netlink.Link, settings *Interface) error { func setInterfaceName(iface netlink.Link, settings *Interface) error { return netlink.LinkSetName(iface, settings.DstName) } + +func setInterfaceRoutes(iface netlink.Link, settings *Interface) error { + for _, route := range settings.Routes { + err := netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: iface.Attrs().Index, + Dst: route, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/libnetwork/sandbox/namespace_linux.go b/libnetwork/sandbox/namespace_linux.go index 01bc7f3fef..dd93e8334b 100644 --- a/libnetwork/sandbox/namespace_linux.go +++ b/libnetwork/sandbox/namespace_linux.go @@ -8,6 +8,7 @@ import ( "sync" "syscall" + "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) @@ -263,6 +264,35 @@ func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error { return err } +func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error { + err := programRoute(n.path, r.Destination, r.NextHop) + if err == nil { + n.Lock() + n.sinfo.StaticRoutes = append(n.sinfo.StaticRoutes, r) + n.Unlock() + } + return err +} + +func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error { + err := removeRoute(n.path, r.Destination, r.NextHop) + if err == nil { + n.Lock() + lastIndex := len(n.sinfo.StaticRoutes) - 1 + for i, v := range n.sinfo.StaticRoutes { + if v == r { + // Overwrite the route we're removing with the last element + n.sinfo.StaticRoutes[i] = n.sinfo.StaticRoutes[lastIndex] + // Shorten the slice to trim the extra element + n.sinfo.StaticRoutes = n.sinfo.StaticRoutes[:lastIndex] + break + } + } + n.Unlock() + } + return err +} + func (n *networkNamespace) Interfaces() []*Interface { n.Lock() defer n.Unlock() diff --git a/libnetwork/sandbox/sandbox.go b/libnetwork/sandbox/sandbox.go index d43a94d3ce..0f036034ab 100644 --- a/libnetwork/sandbox/sandbox.go +++ b/libnetwork/sandbox/sandbox.go @@ -35,6 +35,12 @@ type Sandbox interface { // Set default IPv6 gateway for the sandbox SetGatewayIPv6(gw net.IP) error + // Add a static route to the sandbox. + AddStaticRoute(*types.StaticRoute) error + + // Remove a static route from the sandbox. + RemoveStaticRoute(*types.StaticRoute) error + // Destroy the sandbox Destroy() error } @@ -51,7 +57,11 @@ type Info struct { // IPv6 gateway for the sandbox. GatewayIPv6 net.IP - // TODO: Add routes and ip tables etc. + // Additional static routes for the sandbox. (Note that directly + // connected routes are stored on the particular interface they refer to.) + StaticRoutes []*types.StaticRoute + + // TODO: Add ip tables etc. } // Interface represents the settings and identity of a network device. It is @@ -74,15 +84,25 @@ type Interface struct { // IPv6 address for the interface. AddressIPv6 *net.IPNet + + // IP routes for the interface. + Routes []*net.IPNet } // GetCopy returns a copy of this Interface structure func (i *Interface) GetCopy() *Interface { + copiedRoutes := make([]*net.IPNet, len(i.Routes)) + + for index := range i.Routes { + copiedRoutes[index] = types.GetIPNetCopy(i.Routes[index]) + } + return &Interface{ SrcName: i.SrcName, DstName: i.DstName, Address: types.GetIPNetCopy(i.Address), AddressIPv6: types.GetIPNetCopy(i.AddressIPv6), + Routes: copiedRoutes, } } @@ -108,6 +128,16 @@ func (i *Interface) Equal(o *Interface) bool { return false } + if len(i.Routes) != len(o.Routes) { + return false + } + + for index := range i.Routes { + if !types.CompareIPNet(i.Routes[index], o.Routes[index]) { + return false + } + } + return true } @@ -120,7 +150,15 @@ func (s *Info) GetCopy() *Info { gw := types.GetIPCopy(s.Gateway) gw6 := types.GetIPCopy(s.GatewayIPv6) - return &Info{Interfaces: list, Gateway: gw, GatewayIPv6: gw6} + routes := make([]*types.StaticRoute, len(s.StaticRoutes)) + for i, r := range s.StaticRoutes { + routes[i] = r.GetCopy() + } + + return &Info{Interfaces: list, + Gateway: gw, + GatewayIPv6: gw6, + StaticRoutes: routes} } // Equal checks if this instance of SandboxInfo is equal to the passed one @@ -154,6 +192,17 @@ func (s *Info) Equal(o *Info) bool { } } + for index := range s.StaticRoutes { + ss := s.StaticRoutes[index] + oo := o.StaticRoutes[index] + if !types.CompareIPNet(ss.Destination, oo.Destination) { + return false + } + if !ss.NextHop.Equal(oo.NextHop) { + return false + } + } + return true } diff --git a/libnetwork/sandbox/sandbox_linux_test.go b/libnetwork/sandbox/sandbox_linux_test.go index 5c4448a3e9..678cd0d2a6 100644 --- a/libnetwork/sandbox/sandbox_linux_test.go +++ b/libnetwork/sandbox/sandbox_linux_test.go @@ -63,6 +63,13 @@ func newInfo(t *testing.T) (*Info, error) { intf1.AddressIPv6 = addrv6 intf1.AddressIPv6.IP = ip6 + _, route, err := net.ParseCIDR("192.168.2.1/32") + if err != nil { + return nil, err + } + + intf1.Routes = []*net.IPNet{route} + veth = &netlink.Veth{ LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0}, PeerName: vethName4} @@ -92,6 +99,7 @@ func newInfo(t *testing.T) (*Info, error) { intf2.AddressIPv6.IP = ip6 sinfo := &Info{Interfaces: []*Interface{intf1, intf2}} + sinfo.Gateway = net.ParseIP("192.168.1.1") // sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1") sinfo.GatewayIPv6 = net.ParseIP("fe80::1") diff --git a/libnetwork/sandbox/sandbox_test.go b/libnetwork/sandbox/sandbox_test.go index c0dd6c8e75..03b250bacd 100644 --- a/libnetwork/sandbox/sandbox_test.go +++ b/libnetwork/sandbox/sandbox_test.go @@ -190,12 +190,14 @@ func getInterfaceList() []*Interface { DstName: "eth0", Address: netv4a, AddressIPv6: netv6a, + Routes: []*net.IPNet{netv4a, netv6a}, }, &Interface{ SrcName: "veth7654321", DstName: "eth1", Address: netv4b, AddressIPv6: netv6b, + Routes: []*net.IPNet{netv4b, netv6b}, }, } } diff --git a/libnetwork/types/types.go b/libnetwork/types/types.go index 3b83485f75..6ada47fb4e 100644 --- a/libnetwork/types/types.go +++ b/libnetwork/types/types.go @@ -184,6 +184,40 @@ func CompareIPNet(a, b *net.IPNet) bool { return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask) } +const ( + // NEXTHOP indicates a StaticRoute with an IP next hop. + NEXTHOP = iota + + // CONNECTED indicates a StaticRoute with a interface for directly connected peers. + CONNECTED +) + +// StaticRoute is a statically-provisioned IP route. +type StaticRoute struct { + Destination *net.IPNet + + RouteType int // NEXT_HOP or CONNECTED + + // NextHop will be resolved by the kernel (i.e. as a loose hop). + NextHop net.IP + + // InterfaceID must refer to a defined interface on the + // Endpoint to which the routes are specified. Routes specified this way + // are interpreted as directly connected to the specified interface (no + // next hop will be used). + InterfaceID int +} + +// GetCopy returns a copy of this StaticRoute structure +func (r *StaticRoute) GetCopy() *StaticRoute { + d := GetIPNetCopy(r.Destination) + nh := GetIPCopy(r.NextHop) + return &StaticRoute{Destination: d, + RouteType: r.RouteType, + NextHop: nh, + InterfaceID: r.InterfaceID} +} + /****************************** * Well-known Error Interfaces ******************************/