From e2d8531a4087f2c13669c2d8095808cc9244211d Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 28 Apr 2015 05:57:36 +0000 Subject: [PATCH] - Added support for Join/Leave methods to Endpoint. - Removed sandbox key argument for CreateEndpoint. - Refactored bridge driver code to remove sandbox key. - Fixed bridge driver code for gaps in ipv6 behavior observed during docker integration. - Updated test code, readme code, README.md according api change. - Fixed some sandbox issues while testing docker ipv6 integration. Signed-off-by: Jana Radhakrishnan --- libnetwork/README.md | 67 +++----- libnetwork/ROADMAP.md | 2 +- libnetwork/cmd/readme_test/readme.go | 26 +-- libnetwork/driverapi/driverapi.go | 4 +- libnetwork/drivers/bridge/bridge.go | 79 ++++----- libnetwork/drivers/bridge/bridge_test.go | 4 +- libnetwork/drivers/bridge/network_test.go | 28 +--- libnetwork/drivers/bridge/setup_ipv6.go | 11 +- libnetwork/error.go | 11 ++ libnetwork/libnetwork_test.go | 147 ++++++++++++++++- libnetwork/network.go | 188 ++++++++++++++++++---- libnetwork/sandbox/configure_linux.go | 10 +- libnetwork/sandbox/namespace_linux.go | 35 +++- libnetwork/sandbox/sandbox_linux_test.go | 6 +- 14 files changed, 443 insertions(+), 175 deletions(-) diff --git a/libnetwork/README.md b/libnetwork/README.md index 744c388d5c..dec947dc0d 100644 --- a/libnetwork/README.md +++ b/libnetwork/README.md @@ -19,54 +19,33 @@ Please refer to the [roadmap](ROADMAP.md) for more information. There are many networking solutions available to suit a broad range of use-cases. libnetwork uses a driver / plugin model to support all of these solutions while abstracting the complexity of the driver implementations by exposing a simple and consistent Network Model to users. + ```go - // Create a new controller instance - controller := libnetwork.New() + // Create a new controller instance + controller := libnetwork.New() - // This option is only needed for in-tree drivers. Plugins(in future) will get - // their options through plugin infrastructure. - option := options.Generic{} - driver, err := controller.NewNetworkDriver("bridge", option) - if err != nil { - return - } + // This option is only needed for in-tree drivers. Plugins(in future) will get + // their options through plugin infrastructure. + option := options.Generic{} + err := controller.NewNetworkDriver("bridge", option) + if err != nil { + return + } - netOptions := options.Generic{} - // Create a network for containers to join. - network, err := controller.NewNetwork(driver, "network1", netOptions) - if err != nil { - return - } + netOptions := options.Generic{} + // Create a network for containers to join. + network, err := controller.NewNetwork("bridge", "network1", netOptions) + if err != nil { + return + } - // For a new container: create a sandbox instance (providing a unique key). - // For linux it is a filesystem path - networkPath := "/var/lib/docker/.../4d23e" - networkNamespace, err := sandbox.NewSandbox(networkPath) - if err != nil { - return - } - - // For each new container: allocate IP and interfaces. The returned network - // settings will be used for container infos (inspect and such), as well as - // iptables rules for port publishing. This info is contained or accessible - // from the returned endpoint. - ep, err := network.CreateEndpoint("Endpoint1", networkNamespace.Key(), "") - if err != nil { - return - } - - // Add interfaces to the namespace. - sinfo := ep.SandboxInfo() - for _, iface := range sinfo.Interfaces { - if err := networkNamespace.AddInterface(iface); err != nil { - return - } - } - - // Set the gateway IP - if err := networkNamespace.SetGateway(sinfo.Gateway); err != nil { - return - } + // For each new container: allocate IP and interfaces. The returned network + // settings will be used for container infos (inspect and such), as well as + // iptables rules for port publishing. + ep, err := network.CreateEndpoint("Endpoint1", nil) + if err != nil { + return + } ``` ## Future diff --git a/libnetwork/ROADMAP.md b/libnetwork/ROADMAP.md index af00c4c302..ec2221e095 100644 --- a/libnetwork/ROADMAP.md +++ b/libnetwork/ROADMAP.md @@ -12,7 +12,7 @@ To suggest changes to the roadmap, including additions, please write the change #### Concepts 1. Sandbox: An isolated environment. This is more or less a standard docker container. -2. Endpoint: An addressable endpoint used for communication over a specific network. Endpoints join exactly one network and are expected to create a method of network communication for a container. Endpoints are garbage collected when they no longer belong to any Sandboxes. Example : veth pair +2. Endpoint: An addressable endpoint used for communication over a specific network. Endpoints join exactly one network and are expected to create a method of network communication for a container. Example : veth pair 3. Network: A collection of endpoints that are able to communicate to each other. Networks are intended to be isolated from each other and to not cross communicate. #### axioms diff --git a/libnetwork/cmd/readme_test/readme.go b/libnetwork/cmd/readme_test/readme.go index 8bfcf852c0..30f9af557f 100644 --- a/libnetwork/cmd/readme_test/readme.go +++ b/libnetwork/cmd/readme_test/readme.go @@ -3,7 +3,6 @@ package main import ( "github.com/docker/libnetwork" "github.com/docker/libnetwork/pkg/options" - "github.com/docker/libnetwork/sandbox" ) func main() { @@ -25,33 +24,20 @@ func main() { return } - // For a new container: create a sandbox instance (providing a unique key). - // For linux it is a filesystem path - networkPath := "/var/lib/docker/.../4d23e" - networkNamespace, err := sandbox.NewSandbox(networkPath) - if err != nil { - return - } - // For each new container: allocate IP and interfaces. The returned network // settings will be used for container infos (inspect and such), as well as // iptables rules for port publishing. This info is contained or accessible // from the returned endpoint. - ep, err := network.CreateEndpoint("Endpoint1", networkNamespace.Key(), nil) + ep, err := network.CreateEndpoint("Endpoint1", nil) if err != nil { return } - // Add interfaces to the namespace. - sinfo := ep.SandboxInfo() - for _, iface := range sinfo.Interfaces { - if err := networkNamespace.AddInterface(iface); err != nil { - return - } - } - - // Set the gateway IP - if err := networkNamespace.SetGateway(sinfo.Gateway); err != nil { + // A container can join the endpoint by providing the container ID to the join + // api which returns the sandbox key which can be used to access the sandbox + // created for the container during join. + _, err = ep.Join("container1") + if err != nil { return } } diff --git a/libnetwork/driverapi/driverapi.go b/libnetwork/driverapi/driverapi.go index a8c7824849..184528137a 100644 --- a/libnetwork/driverapi/driverapi.go +++ b/libnetwork/driverapi/driverapi.go @@ -31,10 +31,10 @@ type Driver interface { DeleteNetwork(nid types.UUID) error // CreateEndpoint invokes the driver method to create an endpoint - // passing the network id, endpoint id, sandbox key and driver + // passing the network id, endpoint id and driver // specific config. The config mechanism will eventually be replaced // with labels which are yet to be introduced. - CreateEndpoint(nid, eid types.UUID, key string, config interface{}) (*sandbox.Info, error) + CreateEndpoint(nid, eid types.UUID, config interface{}) (*sandbox.Info, error) // DeleteEndpoint invokes the driver method to delete an endpoint // passing the network id and endpoint id. diff --git a/libnetwork/drivers/bridge/bridge.go b/libnetwork/drivers/bridge/bridge.go index f72933cc93..115197dddd 100644 --- a/libnetwork/drivers/bridge/bridge.go +++ b/libnetwork/drivers/bridge/bridge.go @@ -57,8 +57,8 @@ type bridgeEndpoint struct { type bridgeNetwork struct { id types.UUID - bridge *bridgeInterface // The bridge's L3 interface - endpoints map[string]*bridgeEndpoint // key: sandbox id + bridge *bridgeInterface // The bridge's L3 interface + endpoints map[types.UUID]*bridgeEndpoint // key: endpoint id sync.Mutex } @@ -118,21 +118,19 @@ func (c *Configuration) Validate() error { return nil } -func (n *bridgeNetwork) getEndpoint(eid types.UUID) (string, *bridgeEndpoint, error) { +func (n *bridgeNetwork) getEndpoint(eid types.UUID) (*bridgeEndpoint, error) { n.Lock() defer n.Unlock() if eid == "" { - return "", nil, InvalidEndpointIDError(eid) + return nil, InvalidEndpointIDError(eid) } - for sk, ep := range n.endpoints { - if ep.id == eid { - return sk, ep, nil - } + if ep, ok := n.endpoints[eid]; ok { + return ep, nil } - return "", nil, nil + return nil, nil } func (d *driver) Config(option interface{}) error { @@ -184,7 +182,7 @@ func (d *driver) CreateNetwork(id types.UUID, option interface{}) error { } // Create and set network handler in driver - d.network = &bridgeNetwork{id: id, endpoints: make(map[string]*bridgeEndpoint)} + d.network = &bridgeNetwork{id: id, endpoints: make(map[types.UUID]*bridgeEndpoint)} d.Unlock() // On failure make sure to reset driver network handler to nil @@ -299,7 +297,7 @@ func (d *driver) DeleteNetwork(nid types.UUID) error { return err } -func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions interface{}) (*sandbox.Info, error) { +func (d *driver) CreateEndpoint(nid, eid types.UUID, epOptions interface{}) (*sandbox.Info, error) { var ( ipv6Addr *net.IPNet err error @@ -323,7 +321,7 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions i n.Unlock() // Check if endpoint id is good and retrieve correspondent endpoint - _, ep, err := n.getEndpoint(eid) + ep, err := n.getEndpoint(eid) if err != nil { return nil, err } @@ -333,18 +331,6 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions i return nil, driverapi.ErrEndpointExists } - // Check if valid sandbox key - if sboxKey == "" { - return nil, InvalidSandboxIDError(sboxKey) - } - - // Check if endpoint already exists for this sandbox - n.Lock() - if _, ok := n.endpoints[sboxKey]; ok { - n.Unlock() - return nil, driverapi.ErrEndpointExists - } - // Try to convert the options to endpoint configuration epConfig, err := parseEndpointOptions(epOptions) if err != nil { @@ -353,15 +339,16 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions i } // Create and add the endpoint + n.Lock() endpoint := &bridgeEndpoint{id: eid, config: epConfig} - n.endpoints[sboxKey] = endpoint + n.endpoints[eid] = endpoint n.Unlock() // On failure make sure to remove the endpoint defer func() { if err != nil { n.Lock() - delete(n.endpoints, sboxKey) + delete(n.endpoints, eid) n.Unlock() } }() @@ -408,12 +395,15 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions i } }() + mac := netutils.GenerateRandomMAC() // Add user specified attributes if epConfig != nil && epConfig.MacAddress != nil { - err = netlink.LinkSetHardwareAddr(sbox, epConfig.MacAddress) - if err != nil { - return nil, err - } + mac = epConfig.MacAddress + } + + err = netlink.LinkSetHardwareAddr(sbox, mac) + if err != nil { + return nil, err } // Add bridge inherited attributes to pipe interfaces @@ -443,11 +433,28 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, sboxKey string, epOptions i // v6 address for the sandbox side pipe interface if config.EnableIPv6 { - ip6, err := ipAllocator.RequestIP(n.bridge.bridgeIPv6, nil) + var ip6 net.IP + + network := n.bridge.bridgeIPv6 + if config.FixedCIDRv6 != nil { + network = config.FixedCIDRv6 + } + + ones, _ := network.Mask.Size() + if ones <= 80 { + ip6 = make(net.IP, len(network.IP)) + copy(ip6, network.IP) + for i, h := range mac { + ip6[i+10] = h + } + } + + ip6, err := ipAllocator.RequestIP(network, ip6) if err != nil { return nil, err } - ipv6Addr = &net.IPNet{IP: ip6, Mask: n.bridge.bridgeIPv6.Mask} + + ipv6Addr = &net.IPNet{IP: ip6, Mask: network.Mask} } // Store the sandbox side pipe interface @@ -494,7 +501,7 @@ func (d *driver) DeleteEndpoint(nid, eid types.UUID) error { n.Unlock() // Check endpoint id and if an endpoint is actually there - sboxKey, ep, err := n.getEndpoint(eid) + ep, err := n.getEndpoint(eid) if err != nil { return err } @@ -504,7 +511,7 @@ func (d *driver) DeleteEndpoint(nid, eid types.UUID) error { // Remove it n.Lock() - delete(n.endpoints, sboxKey) + delete(n.endpoints, eid) n.Unlock() // On failure make sure to set back ep in n.endpoints, but only @@ -512,8 +519,8 @@ func (d *driver) DeleteEndpoint(nid, eid types.UUID) error { defer func() { if err != nil { n.Lock() - if _, ok := n.endpoints[sboxKey]; !ok { - n.endpoints[sboxKey] = ep + if _, ok := n.endpoints[eid]; !ok { + n.endpoints[eid] = ep } n.Unlock() } diff --git a/libnetwork/drivers/bridge/bridge_test.go b/libnetwork/drivers/bridge/bridge_test.go index 787ef09ba1..26ee5b482b 100644 --- a/libnetwork/drivers/bridge/bridge_test.go +++ b/libnetwork/drivers/bridge/bridge_test.go @@ -76,7 +76,7 @@ func TestCreateLinkWithOptions(t *testing.T) { mac := net.HardwareAddr([]byte{0x1e, 0x67, 0x66, 0x44, 0x55, 0x66}) epConf := &EndpointConfiguration{MacAddress: mac} - sinfo, err := d.CreateEndpoint("net1", "ep", "s1", epConf) + sinfo, err := d.CreateEndpoint("net1", "ep", epConf) if err != nil { t.Fatalf("Failed to create a link: %s", err.Error()) } @@ -207,7 +207,7 @@ func TestSetDefaultGw(t *testing.T) { t.Fatalf("Failed to create bridge: %v", err) } - sinfo, err := d.CreateEndpoint("dummy", "ep", "sb2", nil) + sinfo, err := d.CreateEndpoint("dummy", "ep", nil) if err != nil { t.Fatalf("Failed to create endpoint: %v", err) } diff --git a/libnetwork/drivers/bridge/network_test.go b/libnetwork/drivers/bridge/network_test.go index b7d1111698..a195013906 100644 --- a/libnetwork/drivers/bridge/network_test.go +++ b/libnetwork/drivers/bridge/network_test.go @@ -28,7 +28,7 @@ func TestLinkCreate(t *testing.T) { t.Fatalf("Failed to create bridge: %v", err) } - sinfo, err := d.CreateEndpoint("dummy", "", "sb1", nil) + sinfo, err := d.CreateEndpoint("dummy", "", nil) if err != nil { if _, ok := err.(InvalidEndpointIDError); !ok { t.Fatalf("Failed with a wrong error :%s", err.Error()) @@ -37,17 +37,8 @@ func TestLinkCreate(t *testing.T) { t.Fatalf("Failed to detect invalid config") } - sinfo, err = d.CreateEndpoint("dummy", "ep", "", nil) - if err != nil { - if _, ok := err.(InvalidSandboxIDError); !ok { - t.Fatalf("Failed with a wrong error :%s", err.Error()) - } - } else { - t.Fatalf("Failed to detect invalid config") - } - // Good endpoint creation - sinfo, err = d.CreateEndpoint("dummy", "ep", "cc", nil) + sinfo, err = d.CreateEndpoint("dummy", "ep", nil) if err != nil { t.Fatalf("Failed to create a link: %s", err.Error()) } @@ -63,16 +54,11 @@ func TestLinkCreate(t *testing.T) { // TODO: if we could get peer name from (sboxLnk.(*netlink.Veth)).PeerName // then we could check the MTU on hostLnk as well. - _, err = d.CreateEndpoint("dummy", "ep", "cc2", nil) + _, err = d.CreateEndpoint("dummy", "ep", nil) if err == nil { t.Fatalf("Failed to detect duplicate endpoint id on same network") } - _, err = d.CreateEndpoint("dummy", "ep2", "cc", nil) - if err == nil { - t.Fatalf("Failed to detect addition of more than one endpoint to same sandbox") - } - interfaces := sinfo.Interfaces if len(interfaces) != 1 { t.Fatalf("Expected exactly one interface. Instead got %d interface(s)", len(interfaces)) @@ -125,12 +111,12 @@ func TestLinkCreateTwo(t *testing.T) { t.Fatalf("Failed to create bridge: %v", err) } - _, err = d.CreateEndpoint("dummy", "ep", "s1", nil) + _, err = d.CreateEndpoint("dummy", "ep", nil) if err != nil { t.Fatalf("Failed to create a link: %s", err.Error()) } - _, err = d.CreateEndpoint("dummy", "ep", "s1", nil) + _, err = d.CreateEndpoint("dummy", "ep", nil) if err != nil { if err != driverapi.ErrEndpointExists { t.Fatalf("Failed with a wrong error :%s", err.Error()) @@ -155,7 +141,7 @@ func TestLinkCreateNoEnableIPv6(t *testing.T) { t.Fatalf("Failed to create bridge: %v", err) } - sinfo, err := d.CreateEndpoint("dummy", "ep", "sb2", nil) + sinfo, err := d.CreateEndpoint("dummy", "ep", nil) if err != nil { t.Fatalf("Failed to create a link: %s", err.Error()) } @@ -186,7 +172,7 @@ func TestLinkDelete(t *testing.T) { t.Fatalf("Failed to create bridge: %v", err) } - _, err = d.CreateEndpoint("dummy", "ep1", "s1", nil) + _, err = d.CreateEndpoint("dummy", "ep1", nil) if err != nil { t.Fatalf("Failed to create a link: %s", err.Error()) } diff --git a/libnetwork/drivers/bridge/setup_ipv6.go b/libnetwork/drivers/bridge/setup_ipv6.go index ca284a7081..70551e2e15 100644 --- a/libnetwork/drivers/bridge/setup_ipv6.go +++ b/libnetwork/drivers/bridge/setup_ipv6.go @@ -29,8 +29,15 @@ func setupBridgeIPv6(config *Configuration, i *bridgeInterface) error { return fmt.Errorf("Unable to enable IPv6 addresses on bridge: %v", err) } - if err := netlink.AddrAdd(i.Link, &netlink.Addr{IPNet: bridgeIPv6}); err != nil { - return &IPv6AddrAddError{ip: bridgeIPv6, err: err} + _, addrsv6, err := i.addresses() + if err != nil { + return err + } + + if len(addrsv6) == 0 { + if err := netlink.AddrAdd(i.Link, &netlink.Addr{IPNet: bridgeIPv6}); err != nil { + return &IPv6AddrAddError{ip: bridgeIPv6, err: err} + } } // Store bridge network and default gateway diff --git a/libnetwork/error.go b/libnetwork/error.go index da2d3f18f8..add594079d 100644 --- a/libnetwork/error.go +++ b/libnetwork/error.go @@ -12,6 +12,9 @@ var ( // ErrInvalidNetworkDriver is returned if an invalid driver // instance is passed. ErrInvalidNetworkDriver = errors.New("invalid driver bound to network") + // ErrInvalidJoin is returned if a join is attempted on an endpoint + // which already has a container joined. + ErrInvalidJoin = errors.New("A container has already joined the endpoint") ) // NetworkTypeError type is returned when the network type string is not @@ -61,3 +64,11 @@ type UnknownEndpointError struct { func (uee *UnknownEndpointError) Error() string { return fmt.Sprintf("unknown endpoint %s id %s", uee.name, uee.id) } + +// InvalidContainerIDError is returned when an invalid container id is passed +// in Join/Leave +type InvalidContainerIDError string + +func (id InvalidContainerIDError) Error() string { + return fmt.Sprintf("invalid container id %s", string(id)) +} diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index a2359cca31..e58ad9f4b8 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -69,7 +69,7 @@ func TestBridge(t *testing.T) { t.Fatal(err) } - ep, err := network.CreateEndpoint("testep", "sb1", nil) + ep, err := network.CreateEndpoint("testep", nil) if err != nil { t.Fatal(err) } @@ -199,7 +199,7 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) { t.Fatal(err) } - ep, err := network.CreateEndpoint("testep", "sb2", nil) + ep, err := network.CreateEndpoint("testep", nil) if err != nil { t.Fatal(err) } @@ -267,7 +267,7 @@ func TestUnknownEndpoint(t *testing.T) { t.Fatal(err) } - ep, err := network.CreateEndpoint("testep", "sb1", nil) + ep, err := network.CreateEndpoint("testep", nil) if err != nil { t.Fatal(err) } @@ -308,11 +308,11 @@ func TestNetworkEndpointsWalkers(t *testing.T) { if err != nil { t.Fatal(err) } - ep11, err := net1.CreateEndpoint("ep11", "sbox1", nil) + ep11, err := net1.CreateEndpoint("ep11", nil) if err != nil { t.Fatal(err) } - ep12, err := net1.CreateEndpoint("ep12", "sbox2", nil) + ep12, err := net1.CreateEndpoint("ep12", nil) if err != nil { t.Fatal(err) } @@ -434,11 +434,11 @@ func TestNetworkQuery(t *testing.T) { if err != nil { t.Fatal(err) } - ep11, err := net1.CreateEndpoint("ep11", "sbox1", nil) + ep11, err := net1.CreateEndpoint("ep11", nil) if err != nil { t.Fatal(err) } - ep12, err := net1.CreateEndpoint("ep12", "sbox2", nil) + ep12, err := net1.CreateEndpoint("ep12", nil) if err != nil { t.Fatal(err) } @@ -469,3 +469,136 @@ func TestNetworkQuery(t *testing.T) { } } + +const containerID = "valid_container" + +func TestEndpointJoin(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) + if err != nil { + t.Fatal(err) + } + + ep, err := n.CreateEndpoint("ep1", nil) + if err != nil { + t.Fatal(err) + } + + _, err = ep.Join(containerID) + if err != nil { + t.Fatal(err) + } + + err = ep.Leave(containerID) + if err != nil { + t.Fatal(err) + } +} + +func TestEndpointJoinInvalidContainerId(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) + if err != nil { + t.Fatal(err) + } + + ep, err := n.CreateEndpoint("ep1", nil) + if err != nil { + t.Fatal(err) + } + + _, err = ep.Join("") + if err == nil { + t.Fatal("Expected to fail join with empty container id string") + } + + if _, ok := err.(libnetwork.InvalidContainerIDError); !ok { + t.Fatalf("Failed for unexpected reason: %v", err) + } +} + +func TestEndpointMultipleJoins(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) + if err != nil { + t.Fatal(err) + } + + ep, err := n.CreateEndpoint("ep1", nil) + if err != nil { + t.Fatal(err) + } + + _, err = ep.Join(containerID) + if err != nil { + t.Fatal(err) + } + + _, err = ep.Join("container2") + if err == nil { + t.Fatal("Expected to fail multiple joins for the same endpoint") + } + + if err != libnetwork.ErrInvalidJoin { + t.Fatalf("Failed for unexpected reason: %v", err) + } + + err = ep.Leave(containerID) + if err != nil { + t.Fatal(err) + } +} + +func TestEndpointInvalidLeave(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) + if err != nil { + t.Fatal(err) + } + + ep, err := n.CreateEndpoint("ep1", nil) + if err != nil { + t.Fatal(err) + } + + err = ep.Leave(containerID) + if err == nil { + t.Fatal("Expected to fail leave from an endpoint which has no active join") + } + + if _, ok := err.(libnetwork.InvalidContainerIDError); !ok { + t.Fatalf("Failed for unexpected reason: %v", err) + } + + _, err = ep.Join(containerID) + if err != nil { + t.Fatal(err) + } + + err = ep.Leave("") + if err == nil { + t.Fatal("Expected to fail leave with empty container id") + } + + if _, ok := err.(libnetwork.InvalidContainerIDError); !ok { + t.Fatalf("Failed for unexpected reason: %v", err) + } + + err = ep.Leave("container2") + if err == nil { + t.Fatal("Expected to fail leave with wrong container id") + } + + if _, ok := err.(libnetwork.InvalidContainerIDError); !ok { + t.Fatalf("Failed for unexpected reason: %v", err) + } + + err = ep.Leave(containerID) + if err != nil { + t.Fatal(err) + } +} diff --git a/libnetwork/network.go b/libnetwork/network.go index 8b35ac3251..17708c13be 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -2,39 +2,40 @@ Package libnetwork provides the basic functionality and extension points to create network namespaces and allocate interfaces for containers to use. - // Create a new controller instance - controller := libnetwork.New() + // Create a new controller instance + controller := libnetwork.New() - // This option is only needed for in-tree drivers. Plugins(in future) will get - // their options through plugin infrastructure. - option := options.Generic{} - err := controller.NewNetworkDriver("bridge", option) - if err != nil { - return - } + // Select and configure the network driver + networkType := "bridge" + option := options.Generic{} + err := controller.ConfigureNetworkDriver(networkType, option) + if err != nil { + return + } - netOptions := options.Generic{} - // Create a network for containers to join. - network, err := controller.NewNetwork("bridge", "network1", netOptions) - if err != nil { - return - } + netOptions := options.Generic{} + // Create a network for containers to join. + network, err := controller.NewNetwork(networkType, "network1", netOptions) + if err != nil { + return + } - // For a new container: create a sandbox instance (providing a unique key). - // For linux it is a filesystem path - networkPath := "/var/lib/docker/.../4d23e" - networkNamespace, err := sandbox.NewSandbox(networkPath) - if err != nil { - return - } + // For each new container: allocate IP and interfaces. The returned network + // settings will be used for container infos (inspect and such), as well as + // iptables rules for port publishing. This info is contained or accessible + // from the returned endpoint. + ep, err := network.CreateEndpoint("Endpoint1", nil) + if err != nil { + return + } - // For each new container: allocate IP and interfaces. The returned network - // settings will be used for container infos (inspect and such), as well as - // iptables rules for port publishing. - ep, err := network.CreateEndpoint("Endpoint1", networkNamespace.Key(), nil) - if err != nil { - return - } + // A container can join the endpoint by providing the container ID to the join + // api which returns the sandbox key which can be used to access the sandbox + // created for the container during join. + _, err = ep.Join("container1") + if err != nil { + return + } */ package libnetwork @@ -85,7 +86,7 @@ type Network interface { // Create a new endpoint to this network symbolically identified by the // specified unique name. The options parameter carry driver specific options. // Labels support will be added in the near future. - CreateEndpoint(name string, sboxKey string, options interface{}) (Endpoint, error) + CreateEndpoint(name string, options interface{}) (Endpoint, error) // Delete the network. Delete() error @@ -118,6 +119,15 @@ type Endpoint interface { // Network returns the name of the network to which this endpoint is attached. Network() string + // Join creates a new sandbox for the given container ID and populates the + // network resources allocated for the endpoint and joins the sandbox to + // the endpoint. It returns the sandbox key to the caller + Join(containerID string) (string, error) + + // Leave removes the sandbox associated with container ID and detaches + // the network resources populated in the sandbox + Leave(containerID string) error + // SandboxInfo returns the sandbox information for this endpoint. SandboxInfo() *sandbox.Info @@ -144,20 +154,29 @@ type endpoint struct { id types.UUID network *network sandboxInfo *sandbox.Info + sandBox sandbox.Sandbox + containerID string +} + +type sandboxData struct { + sandbox sandbox.Sandbox + refCnt int } type networkTable map[types.UUID]*network type endpointTable map[types.UUID]*endpoint +type sandboxTable map[string]sandboxData type controller struct { - networks networkTable - drivers driverTable + networks networkTable + drivers driverTable + sandboxes sandboxTable sync.Mutex } // New creates a new instance of network controller. func New() NetworkController { - return &controller{networkTable{}, enumerateDrivers(), sync.Mutex{}} + return &controller{networkTable{}, enumerateDrivers(), sandboxTable{}, sync.Mutex{}} } func (c *controller) ConfigureNetworkDriver(networkType string, options interface{}) error { @@ -256,6 +275,51 @@ func (c *controller) NetworkByID(id string) Network { return nil } +func (c *controller) sandboxAdd(key string) (sandbox.Sandbox, error) { + c.Lock() + defer c.Unlock() + + sData, ok := c.sandboxes[key] + if !ok { + sb, err := sandbox.NewSandbox(key) + if err != nil { + return nil, err + } + + sData = sandboxData{sandbox: sb, refCnt: 1} + c.sandboxes[key] = sData + return sData.sandbox, nil + } + + sData.refCnt++ + return sData.sandbox, nil +} + +func (c *controller) sandboxRm(key string) { + c.Lock() + defer c.Unlock() + + sData := c.sandboxes[key] + sData.refCnt-- + + if sData.refCnt == 0 { + sData.sandbox.Destroy() + delete(c.sandboxes, key) + } +} + +func (c *controller) sandboxGet(key string) sandbox.Sandbox { + c.Lock() + defer c.Unlock() + + sData, ok := c.sandboxes[key] + if !ok { + return nil + } + + return sData.sandbox +} + func (n *network) Name() string { return n.name } @@ -304,13 +368,13 @@ func (n *network) Delete() error { return err } -func (n *network) CreateEndpoint(name string, sboxKey string, options interface{}) (Endpoint, error) { +func (n *network) CreateEndpoint(name string, options interface{}) (Endpoint, error) { ep := &endpoint{name: name} ep.id = types.UUID(stringid.GenerateRandomID()) ep.network = n d := n.driver - sinfo, err := d.CreateEndpoint(n.id, ep.id, sboxKey, options) + sinfo, err := d.CreateEndpoint(n.id, ep.id, options) if err != nil { return nil, err } @@ -387,6 +451,60 @@ func (ep *endpoint) SandboxInfo() *sandbox.Info { return ep.sandboxInfo.GetCopy() } +func (ep *endpoint) Join(containerID string) (string, error) { + if containerID == "" { + return "", InvalidContainerIDError(containerID) + } + + if ep.containerID != "" { + return "", ErrInvalidJoin + } + + sboxKey := sandbox.GenerateKey(containerID) + sb, err := ep.network.ctrlr.sandboxAdd(sboxKey) + if err != nil { + return "", err + } + defer func() { + if err != nil { + ep.network.ctrlr.sandboxRm(sboxKey) + } + }() + + sinfo := ep.SandboxInfo() + if sinfo != nil { + for _, i := range sinfo.Interfaces { + err = sb.AddInterface(i) + if err != nil { + return "", err + } + } + + err = sb.SetGateway(sinfo.Gateway) + if err != nil { + return "", err + } + + err = sb.SetGatewayIPv6(sinfo.GatewayIPv6) + if err != nil { + return "", err + } + } + + ep.containerID = containerID + return sb.Key(), nil +} + +func (ep *endpoint) Leave(containerID string) error { + if ep.containerID == "" || containerID == "" || ep.containerID != containerID { + return InvalidContainerIDError(containerID) + } + + ep.network.ctrlr.sandboxRm(sandbox.GenerateKey(containerID)) + ep.containerID = "" + return nil +} + func (ep *endpoint) Delete() error { var err error diff --git a/libnetwork/sandbox/configure_linux.go b/libnetwork/sandbox/configure_linux.go index 14af20320f..cae77890fd 100644 --- a/libnetwork/sandbox/configure_linux.go +++ b/libnetwork/sandbox/configure_linux.go @@ -51,9 +51,15 @@ func programGateway(path string, gw net.IP) error { } defer netns.Set(origns) + gwRoutes, err := netlink.RouteGet(gw) + if err != nil { + return fmt.Errorf("route for the gateway could not be found: %v", err) + } + return netlink.RouteAdd(&netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - Gw: gw, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gw, }) } diff --git a/libnetwork/sandbox/namespace_linux.go b/libnetwork/sandbox/namespace_linux.go index 22e318feed..85cef29b4e 100644 --- a/libnetwork/sandbox/namespace_linux.go +++ b/libnetwork/sandbox/namespace_linux.go @@ -5,12 +5,17 @@ import ( "net" "os" "runtime" + "sync" "syscall" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) +const prefix = "/var/lib/docker/network" + +var once sync.Once + // The networkNamespace type is the linux implementation of the Sandbox // interface. It represents a linux network namespace, and moves an interface // into it when called on method AddInterface or sets the gateway etc. @@ -19,6 +24,24 @@ type networkNamespace struct { sinfo *Info } +func creatBasePath() { + err := os.MkdirAll(prefix, 0644) + if err != nil && !os.IsExist(err) { + panic("Could not create net namespace path directory") + } +} + +// GenerateKey generates a sandbox key based on the passed +// container id. +func GenerateKey(containerID string) string { + maxLen := 12 + if len(containerID) < maxLen { + maxLen = len(containerID) + } + + return prefix + "/" + containerID[:maxLen] +} + // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox func NewSandbox(key string) (Sandbox, error) { @@ -63,6 +86,8 @@ func createNetworkNamespace(path string) (Sandbox, error) { func createNamespaceFile(path string) (err error) { var f *os.File + + once.Do(creatBasePath) if f, err = os.Create(path); err == nil { f.Close() } @@ -130,6 +155,10 @@ func (n *networkNamespace) AddInterface(i *Interface) error { } func (n *networkNamespace) SetGateway(gw net.IP) error { + if len(gw) == 0 { + return nil + } + err := programGateway(n.path, gw) if err == nil { n.sinfo.Gateway = gw @@ -162,5 +191,9 @@ func (n *networkNamespace) Key() string { func (n *networkNamespace) Destroy() error { // Assuming no running process is executing in this network namespace, // unmounting is sufficient to destroy it. - return syscall.Unmount(n.path, syscall.MNT_DETACH) + if err := syscall.Unmount(n.path, syscall.MNT_DETACH); err != nil { + return err + } + + return os.Remove(n.path) } diff --git a/libnetwork/sandbox/sandbox_linux_test.go b/libnetwork/sandbox/sandbox_linux_test.go index 0372399637..06e98c7846 100644 --- a/libnetwork/sandbox/sandbox_linux_test.go +++ b/libnetwork/sandbox/sandbox_linux_test.go @@ -54,7 +54,8 @@ func newInfo(t *testing.T) (*Info, error) { intf.Address = addr intf.Address.IP = ip4 - ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48") + // ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48") + ip6, addrv6, err := net.ParseCIDR("fe80::2/64") if err != nil { return nil, err } @@ -63,7 +64,8 @@ func newInfo(t *testing.T) (*Info, error) { sinfo := &Info{Interfaces: []*Interface{intf}} sinfo.Gateway = net.ParseIP("192.168.1.1") - sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1") + // sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1") + sinfo.GatewayIPv6 = net.ParseIP("fe80::1") return sinfo, nil }