From b9596c89d61d437a6f725f78f48d8e77730c3a1d Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Sat, 3 Oct 2015 21:25:57 -0700 Subject: [PATCH] libnetwork to handle allocation of ipv6 Signed-off-by: Alessandro Boch --- libnetwork/controller.go | 8 +- libnetwork/endpoint.go | 50 ++++++-- libnetwork/endpoint_info.go | 12 +- libnetwork/libnetwork_internal_test.go | 7 +- libnetwork/network.go | 163 +++++++++++++++++-------- 5 files changed, 169 insertions(+), 71 deletions(-) diff --git a/libnetwork/controller.go b/libnetwork/controller.go index dbe13a5490..4980849536 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -343,15 +343,13 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti return nil, err } - cnfs, err := network.ipamAllocate() + err := network.ipamAllocate() if err != nil { return nil, err } defer func() { if err != nil { - for _, cn := range cnfs { - cn() - } + network.ipamRelease() } }() @@ -386,7 +384,7 @@ func (c *controller) addNetwork(n *network) error { } // Create the network - if err := d.CreateNetwork(n.id, n.generic, n.getIPv4Data(), n.getIPv6Data()); err != nil { + if err := d.CreateNetwork(n.id, n.generic, n.getIPData(4), n.getIPData(6)); err != nil { return err } diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index a76cb235e7..3fcc1f26aa 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -630,13 +630,44 @@ func (ep *endpoint) assignAddress() error { if err != nil { return err } - for _, d := range n.getIPInfo() { - var addr *net.IPNet - addr, _, err = ipam.RequestAddress(d.PoolID, nil, nil) + err = ep.assignAddressVersion(4, ipam) + if err != nil { + return err + } + return ep.assignAddressVersion(6, ipam) +} + +func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error { + var ( + poolID *string + address **net.IPNet + ) + + n := ep.getNetwork() + switch ipVer { + case 4: + poolID = &ep.iface.v4PoolID + address = &ep.iface.addr + case 6: + poolID = &ep.iface.v6PoolID + address = &ep.iface.addrv6 + default: + return types.InternalErrorf("incorrect ip version number passed: %d", ipVer) + } + + ipInfo := n.getIPInfo(ipVer) + + // ipv6 address is not mandatory + if len(ipInfo) == 0 && ipVer == 6 { + return nil + } + + for _, d := range ipInfo { + addr, _, err := ipam.RequestAddress(d.PoolID, nil, nil) if err == nil { ep.Lock() - ep.iface.addr = addr - ep.iface.poolID = d.PoolID + *address = addr + *poolID = d.PoolID ep.Unlock() return nil } @@ -644,7 +675,7 @@ func (ep *endpoint) assignAddress() error { return err } } - return fmt.Errorf("no available ip addresses on this network address pools: %s (%s)", n.Name(), n.ID()) + return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID()) } func (ep *endpoint) releaseAddress() { @@ -657,7 +688,12 @@ func (ep *endpoint) releaseAddress() { log.Warnf("Failed to retrieve ipam driver to release interface address on delete of endpoint %s (%s): %v", ep.Name(), ep.ID(), err) return } - if err := ipam.ReleaseAddress(ep.iface.poolID, ep.iface.addr.IP); err != nil { + if err := ipam.ReleaseAddress(ep.iface.v4PoolID, ep.iface.addr.IP); err != nil { log.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addr.IP, ep.Name(), ep.ID(), err) } + if ep.iface.addrv6 != nil && ep.iface.addrv6.IP.IsGlobalUnicast() { + if err := ipam.ReleaseAddress(ep.iface.v6PoolID, ep.iface.addrv6.IP); err != nil { + log.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addrv6.IP, ep.Name(), ep.ID(), err) + } + } } diff --git a/libnetwork/endpoint_info.go b/libnetwork/endpoint_info.go index 7c765f4000..3ca6b2b834 100644 --- a/libnetwork/endpoint_info.go +++ b/libnetwork/endpoint_info.go @@ -48,7 +48,8 @@ type endpointInterface struct { srcName string dstPrefix string routes []*net.IPNet - poolID string + v4PoolID string + v6PoolID string } func (epi *endpointInterface) MarshalJSON() ([]byte, error) { @@ -69,7 +70,8 @@ func (epi *endpointInterface) MarshalJSON() ([]byte, error) { routes = append(routes, route.String()) } epMap["routes"] = routes - epMap["poolID"] = epi.poolID + epMap["v4PoolID"] = epi.v4PoolID + epMap["v6PoolID"] = epi.v6PoolID return json.Marshal(epMap) } @@ -111,7 +113,8 @@ func (epi *endpointInterface) UnmarshalJSON(b []byte) error { epi.routes = append(epi.routes, ipr) } } - epi.poolID = epMap["poolID"].(string) + epi.v4PoolID = epMap["v4PoolID"].(string) + epi.v6PoolID = epMap["v6PoolID"].(string) return nil } @@ -122,7 +125,8 @@ func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error { dstEpi.addrv6 = types.GetIPNetCopy(epi.addrv6) dstEpi.srcName = epi.srcName dstEpi.dstPrefix = epi.dstPrefix - dstEpi.poolID = epi.poolID + dstEpi.v4PoolID = epi.v4PoolID + dstEpi.v6PoolID = epi.v6PoolID for _, route := range epi.routes { dstEpi.routes = append(dstEpi.routes, types.GetIPNetCopy(route)) diff --git a/libnetwork/libnetwork_internal_test.go b/libnetwork/libnetwork_internal_test.go index 0e8241f6b5..7c7d26460f 100644 --- a/libnetwork/libnetwork_internal_test.go +++ b/libnetwork/libnetwork_internal_test.go @@ -197,7 +197,8 @@ func TestEndpointMarshalling(t *testing.T) { addrv6: nw6, srcName: "veth12ab1314", dstPrefix: "eth", - poolID: "poolpool", + v4PoolID: "poolpool", + v6PoolID: "poolv6", }, } @@ -224,7 +225,7 @@ func compareEndpointInterface(a, b *endpointInterface) bool { if a == nil || b == nil { return false } - return a.srcName == b.srcName && a.dstPrefix == b.dstPrefix && a.poolID == b.poolID && + return a.srcName == b.srcName && a.dstPrefix == b.dstPrefix && a.v4PoolID == b.v4PoolID && a.v6PoolID == b.v6PoolID && types.CompareIPNet(a.addr, b.addr) && types.CompareIPNet(a.addrv6, b.addrv6) } @@ -319,7 +320,7 @@ func TestAuxAddresses(t *testing.T) { n.ipamV4Config = []*IpamConf{&IpamConf{PreferredPool: i.masterPool, SubPool: i.subPool, AuxAddresses: i.auxAddresses}} - _, err := n.ipamAllocate() + err = n.ipamAllocate() if i.good != (err == nil) { t.Fatalf("Unexpected result for %v: %v", i, err) diff --git a/libnetwork/network.go b/libnetwork/network.go index be5577de9b..486f40380e 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -65,7 +65,7 @@ type IpamConf struct { // this becomes the container pool SubPool string // Input options for IPAM Driver (optional) - Options map[string]string // IPAM input options + Options map[string]string // IPv6 flag, Needed when no preferred pool is specified IsV6 bool // Preferred Network Gateway address (optional) @@ -281,6 +281,14 @@ func (n *network) CopyTo(o datastore.KVObject) error { dstN.ipamV4Info = append(dstN.ipamV4Info, dstV4Info) } + if n.ipamV6Info != nil { + for _, v6info := range n.ipamV6Info { + dstV6Info := &IpamInfo{} + v6info.CopyTo(dstV6Info) + dstN.ipamV6Info = append(dstN.ipamV6Info, dstV6Info) + } + } + dstN.generic = options.Generic{} for k, v := range n.generic { dstN.generic[k] = v @@ -786,44 +794,76 @@ func (n *network) getController() *controller { return n.ctrlr } -func (n *network) ipamAllocate() ([]func(), error) { - var ( - cnl []func() - err error - ) - +func (n *network) ipamAllocate() error { // For now also exclude bridge from using new ipam if n.Type() == "host" || n.Type() == "null" || n.Type() == "bridge" { - return cnl, nil + return nil } ipam, err := n.getController().getIpamDriver(n.ipamType) if err != nil { - return nil, err + return err } if n.addrSpace == "" { if n.addrSpace, err = n.deriveAddressSpace(); err != nil { - return nil, err + return err } } - if n.ipamV4Config == nil { - n.ipamV4Config = []*IpamConf{&IpamConf{}} + err = n.ipamAllocateVersion(4, ipam) + if err != nil { + return err } - n.ipamV4Info = make([]*IpamInfo, len(n.ipamV4Config)) + defer func() { + if err != nil { + n.ipamReleaseVersion(4, ipam) + } + }() - for i, cfg := range n.ipamV4Config { + return n.ipamAllocateVersion(6, ipam) +} + +func (n *network) ipamAllocateVersion(ipVer int, ipam ipamapi.Ipam) error { + var ( + cfgList *[]*IpamConf + infoList *[]*IpamInfo + err error + ) + + switch ipVer { + case 4: + cfgList = &n.ipamV4Config + infoList = &n.ipamV4Info + case 6: + cfgList = &n.ipamV6Config + infoList = &n.ipamV6Info + default: + return types.InternalErrorf("incorrect ip version passed to ipam allocate: %d", ipVer) + } + + if *cfgList == nil { + if ipVer == 6 { + return nil + } + *cfgList = []*IpamConf{&IpamConf{}} + } + + *infoList = make([]*IpamInfo, len(*cfgList)) + + log.Debugf("allocating IPv%d pools for network %s (%s)", ipVer, n.Name(), n.ID()) + + for i, cfg := range *cfgList { if err = cfg.Validate(); err != nil { - return nil, err + return err } d := &IpamInfo{} - n.ipamV4Info[i] = d + (*infoList)[i] = d d.PoolID, d.Pool, d.Meta, err = ipam.RequestPool(n.addrSpace, cfg.PreferredPool, cfg.SubPool, cfg.Options, cfg.IsV6) if err != nil { - return nil, err + return err } defer func() { @@ -836,7 +876,7 @@ func (n *network) ipamAllocate() ([]func(), error) { if gws, ok := d.Meta[netlabel.Gateway]; ok { if d.Gateway, err = types.ParseCIDR(gws); err != nil { - return nil, types.BadRequestErrorf("failed to parse gateway address (%v) returned by ipam driver: %v", gws, err) + return types.BadRequestErrorf("failed to parse gateway address (%v) returned by ipam driver: %v", gws, err) } } @@ -845,15 +885,10 @@ func (n *network) ipamAllocate() ([]func(), error) { // If none of the above is true, libnetwork will allocate one. if cfg.Gateway != "" || d.Gateway == nil { if d.Gateway, _, err = ipam.RequestAddress(d.PoolID, net.ParseIP(cfg.Gateway), nil); err != nil { - return nil, types.InternalErrorf("failed to allocate gateway (%v): %v", cfg.Gateway, err) + return types.InternalErrorf("failed to allocate gateway (%v): %v", cfg.Gateway, err) } } - cnl = append(cnl, func() { - if err := ipam.ReleaseAddress(d.PoolID, d.Gateway.IP); err != nil { - log.Warnf("Failed to release gw address %s after failure to create network %s (%s)", d.Gateway, n.Name(), n.ID()) - } - }) // Auxiliary addresses must be part of the master address pool // If they fall into the container addressable pool, libnetwork will reserve them if cfg.AuxAddresses != nil { @@ -861,27 +896,20 @@ func (n *network) ipamAllocate() ([]func(), error) { d.IPAMData.AuxAddresses = make(map[string]*net.IPNet, len(cfg.AuxAddresses)) for k, v := range cfg.AuxAddresses { if ip = net.ParseIP(v); ip == nil { - return nil, types.BadRequestErrorf("non parsable secondary ip address (%s:%s) passed for network %s", k, v, n.Name()) + return types.BadRequestErrorf("non parsable secondary ip address (%s:%s) passed for network %s", k, v, n.Name()) } if !d.Pool.Contains(ip) { - return cnl, types.ForbiddenErrorf("auxilairy address: (%s:%s) must belong to the master pool: %s", k, v, d.Pool) + return types.ForbiddenErrorf("auxilairy address: (%s:%s) must belong to the master pool: %s", k, v, d.Pool) } // Attempt reservation in the container addressable pool, silent the error if address does not belong to that pool if d.IPAMData.AuxAddresses[k], _, err = ipam.RequestAddress(d.PoolID, ip, nil); err != nil && err != ipamapi.ErrIPOutOfRange { - return nil, types.InternalErrorf("failed to allocate secondary ip address (%s:%s): %v", k, v, err) - } - if err == nil { - cnl = append(cnl, func() { - if err := ipam.ReleaseAddress(d.PoolID, ip); err != nil { - log.Warnf("Failed to release secondary ip address %s(%s) after failure to create network %s (%s)", k, v, ip, n.Name(), n.ID()) - } - }) + return types.InternalErrorf("failed to allocate secondary ip address (%s:%s): %v", k, v, err) } } } } - return cnl, nil + return nil } func (n *network) ipamRelease() { @@ -894,7 +922,30 @@ func (n *network) ipamRelease() { log.Warnf("Failed to retrieve ipam driver to release address pool(s) on delete of network %s (%s): %v", n.Name(), n.ID(), err) return } - for _, d := range n.ipamV4Info { + n.ipamReleaseVersion(4, ipam) + n.ipamReleaseVersion(6, ipam) +} + +func (n *network) ipamReleaseVersion(ipVer int, ipam ipamapi.Ipam) { + var infoList []*IpamInfo + + switch ipVer { + case 4: + infoList = n.ipamV4Info + case 6: + infoList = n.ipamV6Info + default: + log.Warnf("incorrect ip version passed to ipam release: %d", ipVer) + return + } + + if infoList == nil { + return + } + + log.Debugf("releasing IPv%d pools from network %s (%s)", ipVer, n.Name(), n.ID()) + + for _, d := range infoList { if d.Gateway != nil { if err := ipam.ReleaseAddress(d.PoolID, d.Gateway.IP); err != nil { log.Warnf("Failed to release gateway ip address %s on delete of network %s (%s): %v", d.Gateway.IP, n.Name(), n.ID(), err) @@ -915,30 +966,38 @@ func (n *network) ipamRelease() { } } -func (n *network) getIPInfo() []*IpamInfo { - n.Lock() - defer n.Unlock() - l := make([]*IpamInfo, 0, len(n.ipamV4Info)) - for _, d := range n.ipamV4Info { - l = append(l, d) +func (n *network) getIPInfo(ipVer int) []*IpamInfo { + var info []*IpamInfo + switch ipVer { + case 4: + info = n.ipamV4Info + case 6: + info = n.ipamV6Info + default: + return nil } - return l -} - -func (n *network) getIPv4Data() []driverapi.IPAMData { - l := make([]driverapi.IPAMData, 0, len(n.ipamV4Info)) + l := make([]*IpamInfo, 0, len(info)) n.Lock() - for _, d := range n.ipamV4Info { - l = append(l, d.IPAMData) + for _, d := range info { + l = append(l, d) } n.Unlock() return l } -func (n *network) getIPv6Data() []driverapi.IPAMData { - l := make([]driverapi.IPAMData, 0, len(n.ipamV6Info)) +func (n *network) getIPData(ipVer int) []driverapi.IPAMData { + var info []*IpamInfo + switch ipVer { + case 4: + info = n.ipamV4Info + case 6: + info = n.ipamV6Info + default: + return nil + } + l := make([]driverapi.IPAMData, 0, len(info)) n.Lock() - for _, d := range n.ipamV6Info { + for _, d := range info { l = append(l, d.IPAMData) } n.Unlock()