package ipam import ( "fmt" "net" "sort" "sync" "github.com/docker/docker/libnetwork/bitseq" "github.com/docker/docker/libnetwork/datastore" "github.com/docker/docker/libnetwork/discoverapi" "github.com/docker/docker/libnetwork/ipamapi" "github.com/docker/docker/libnetwork/ipamutils" "github.com/docker/docker/libnetwork/types" "github.com/sirupsen/logrus" ) const ( localAddressSpace = "LocalDefault" globalAddressSpace = "GlobalDefault" // datastore keyes for ipam objects dsConfigKey = "ipam/" + ipamapi.DefaultIPAM + "/config" dsDataKey = "ipam/" + ipamapi.DefaultIPAM + "/data" ) // Allocator provides per address space ipv4/ipv6 book keeping type Allocator struct { // Predefined pools for default address spaces // Separate from the addrSpace because they should not be serialized predefined map[string][]*net.IPNet predefinedStartIndices map[string]int // The (potentially serialized) address spaces addrSpaces map[string]*addrSpace // stores []datastore.Datastore // Allocated addresses in each address space's subnet addresses map[SubnetKey]*bitseq.Handle sync.Mutex } // NewAllocator returns an instance of libnetwork ipam func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) { a := &Allocator{} // Load predefined subnet pools a.predefined = map[string][]*net.IPNet{ localAddressSpace: ipamutils.GetLocalScopeDefaultNetworks(), globalAddressSpace: ipamutils.GetGlobalScopeDefaultNetworks(), } // Initialize asIndices map a.predefinedStartIndices = make(map[string]int) // Initialize bitseq map a.addresses = make(map[SubnetKey]*bitseq.Handle) // Initialize address spaces a.addrSpaces = make(map[string]*addrSpace) for _, aspc := range []struct { as string ds datastore.DataStore }{ {localAddressSpace, lcDs}, {globalAddressSpace, glDs}, } { a.initializeAddressSpace(aspc.as, aspc.ds) } return a, nil } func (a *Allocator) refresh(as string) error { aSpace, err := a.getAddressSpaceFromStore(as) if err != nil { return types.InternalErrorf("error getting pools config from store: %v", err) } if aSpace == nil { return nil } a.Lock() a.addrSpaces[as] = aSpace a.Unlock() return nil } func (a *Allocator) updateBitMasks(aSpace *addrSpace) error { var inserterList []func() error aSpace.Lock() for k, v := range aSpace.subnets { if v.Range == nil { kk := k vv := v inserterList = append(inserterList, func() error { return a.insertBitMask(kk, vv.Pool) }) } } aSpace.Unlock() // Add the bitmasks (data could come from datastore) for _, f := range inserterList { if err := f(); err != nil { return err } } return nil } // Checks for and fixes damaged bitmask. func (a *Allocator) checkConsistency(as string) { var sKeyList []SubnetKey // Retrieve this address space's configuration and bitmasks from the datastore a.refresh(as) a.Lock() aSpace, ok := a.addrSpaces[as] a.Unlock() if !ok { return } a.updateBitMasks(aSpace) aSpace.Lock() for sk, pd := range aSpace.subnets { if pd.Range != nil { continue } sKeyList = append(sKeyList, sk) } aSpace.Unlock() for _, sk := range sKeyList { a.Lock() bm := a.addresses[sk] a.Unlock() if err := bm.CheckConsistency(); err != nil { logrus.Warnf("Error while running consistency check for %s: %v", sk, err) } } } func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error { scope := "" if ds != nil { scope = ds.Scope() } a.Lock() if currAS, ok := a.addrSpaces[as]; ok { if currAS.ds != nil { a.Unlock() return types.ForbiddenErrorf("a datastore is already configured for the address space %s", as) } } a.addrSpaces[as] = &addrSpace{ subnets: map[SubnetKey]*PoolData{}, id: dsConfigKey + "/" + as, scope: scope, ds: ds, alloc: a, } a.Unlock() a.checkConsistency(as) return nil } // DiscoverNew informs the allocator about a new global scope datastore func (a *Allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { if dType != discoverapi.DatastoreConfig { return nil } dsc, ok := data.(discoverapi.DatastoreConfigData) if !ok { return types.InternalErrorf("incorrect data in datastore update notification: %v", data) } ds, err := datastore.NewDataStoreFromConfig(dsc) if err != nil { return err } return a.initializeAddressSpace(globalAddressSpace, ds) } // DiscoverDelete is a notification of no interest for the allocator func (a *Allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } // GetDefaultAddressSpaces returns the local and global default address spaces func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) { return localAddressSpace, globalAddressSpace, nil } // RequestPool returns an address pool along with its unique id. // addressSpace must be a valid address space name and must not be the empty string. // If pool is the empty string then the default predefined pool for addressSpace will be used, otherwise pool must be a valid IP address and length in CIDR notation. // If subPool is not empty, it must be a valid IP address and length in CIDR notation which is a sub-range of pool. // subPool must be empty if pool is empty. func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { logrus.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6) k, nw, ipr, err := a.parsePoolRequest(addressSpace, pool, subPool, v6) if err != nil { return "", nil, nil, types.InternalErrorf("failed to parse pool request for address space %q pool %q subpool %q: %v", addressSpace, pool, subPool, err) } pdf := k == nil retry: if pdf { if nw, err = a.getPredefinedPool(addressSpace, v6); err != nil { return "", nil, nil, err } k = &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String()} } if err := a.refresh(addressSpace); err != nil { return "", nil, nil, err } aSpace, err := a.getAddrSpace(addressSpace) if err != nil { return "", nil, nil, err } insert, err := aSpace.updatePoolDBOnAdd(*k, nw, ipr, pdf) if err != nil { if _, ok := err.(types.MaskableError); ok { logrus.Debugf("Retrying predefined pool search: %v", err) goto retry } return "", nil, nil, err } if err := a.writeToStore(aSpace); err != nil { if _, ok := err.(types.RetryError); !ok { return "", nil, nil, types.InternalErrorf("pool configuration failed because of %s", err.Error()) } goto retry } return k.String(), nw, nil, insert() } // ReleasePool releases the address pool identified by the passed id func (a *Allocator) ReleasePool(poolID string) error { logrus.Debugf("ReleasePool(%s)", poolID) k := SubnetKey{} if err := k.FromString(poolID); err != nil { return types.BadRequestErrorf("invalid pool id: %s", poolID) } retry: if err := a.refresh(k.AddressSpace); err != nil { return err } aSpace, err := a.getAddrSpace(k.AddressSpace) if err != nil { return err } remove, err := aSpace.updatePoolDBOnRemoval(k) if err != nil { return err } if err = a.writeToStore(aSpace); err != nil { if _, ok := err.(types.RetryError); !ok { return types.InternalErrorf("pool (%s) removal failed because of %v", poolID, err) } goto retry } return remove() } // Given the address space, returns the local or global PoolConfig based on whether the // address space is local or global. AddressSpace locality is registered with IPAM out of band. func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) { a.Lock() defer a.Unlock() aSpace, ok := a.addrSpaces[as] if !ok { return nil, types.BadRequestErrorf("cannot find address space %s (most likely the backing datastore is not configured)", as) } return aSpace, nil } // parsePoolRequest parses and validates a request to create a new pool under addressSpace and returns // a SubnetKey, network and range describing the request. func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) { var ( nw *net.IPNet ipr *AddressRange err error ) if addressSpace == "" { return nil, nil, nil, ipamapi.ErrInvalidAddressSpace } if pool == "" && subPool != "" { return nil, nil, nil, ipamapi.ErrInvalidSubPool } if pool == "" { return nil, nil, nil, nil } if _, nw, err = net.ParseCIDR(pool); err != nil { return nil, nil, nil, ipamapi.ErrInvalidPool } if subPool != "" { if ipr, err = getAddressRange(subPool, nw); err != nil { return nil, nil, nil, err } } return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, nil } func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error { //logrus.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String()) store := a.getStore(key.AddressSpace) ipVer := getAddressVersion(pool.IP) ones, bits := pool.Mask.Size() numAddresses := uint64(1 << uint(bits-ones)) // Allow /64 subnet if ipVer == v6 && numAddresses == 0 { numAddresses-- } // Generate the new address masks. AddressMask content may come from datastore h, err := bitseq.NewHandle(dsDataKey, store, key.String(), numAddresses) if err != nil { return err } // Do not let network identifier address be reserved // Do the same for IPv6 so that bridge ip starts with XXXX...::1 h.Set(0) // Do not let broadcast address be reserved if ipVer == v4 { h.Set(numAddresses - 1) } a.Lock() a.addresses[key] = h a.Unlock() return nil } func (a *Allocator) retrieveBitmask(k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) { a.Lock() bm, ok := a.addresses[k] a.Unlock() if !ok { logrus.Debugf("Retrieving bitmask (%s, %s)", k.String(), n.String()) if err := a.insertBitMask(k, n); err != nil { return nil, types.InternalErrorf("could not find bitmask in datastore for %s", k.String()) } a.Lock() bm = a.addresses[k] a.Unlock() } return bm, nil } func (a *Allocator) getPredefineds(as string) []*net.IPNet { a.Lock() defer a.Unlock() p := a.predefined[as] i := a.predefinedStartIndices[as] // defensive in case the list changed since last update if i >= len(p) { i = 0 } return append(p[i:], p[:i]...) } func (a *Allocator) updateStartIndex(as string, amt int) { a.Lock() i := a.predefinedStartIndices[as] + amt if i < 0 || i >= len(a.predefined[as]) { i = 0 } a.predefinedStartIndices[as] = i a.Unlock() } func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error) { var v ipVersion v = v4 if ipV6 { v = v6 } if as != localAddressSpace && as != globalAddressSpace { return nil, types.NotImplementedErrorf("no default pool available for non-default address spaces") } aSpace, err := a.getAddrSpace(as) if err != nil { return nil, err } predefined := a.getPredefineds(as) aSpace.Lock() for i, nw := range predefined { if v != getAddressVersion(nw.IP) { continue } // Checks whether pool has already been allocated if _, ok := aSpace.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]; ok { continue } // Shouldn't be necessary, but check prevents IP collisions should // predefined pools overlap for any reason. if !aSpace.contains(as, nw) { aSpace.Unlock() a.updateStartIndex(as, i+1) return nw, nil } } aSpace.Unlock() return nil, types.NotFoundErrorf("could not find an available, non-overlapping IPv%d address pool among the defaults to assign to the network", v) } // RequestAddress returns an address from the specified pool ID func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) { logrus.Debugf("RequestAddress(%s, %v, %v)", poolID, prefAddress, opts) k := SubnetKey{} if err := k.FromString(poolID); err != nil { return nil, nil, types.BadRequestErrorf("invalid pool id: %s", poolID) } if err := a.refresh(k.AddressSpace); err != nil { return nil, nil, err } aSpace, err := a.getAddrSpace(k.AddressSpace) if err != nil { return nil, nil, err } aSpace.Lock() p, ok := aSpace.subnets[k] if !ok { aSpace.Unlock() return nil, nil, types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID) } if prefAddress != nil && !p.Pool.Contains(prefAddress) { aSpace.Unlock() return nil, nil, ipamapi.ErrIPOutOfRange } c := p for c.Range != nil { k = c.ParentKey c = aSpace.subnets[k] } aSpace.Unlock() bm, err := a.retrieveBitmask(k, c.Pool) if err != nil { return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v", k.String(), prefAddress, poolID, err) } // In order to request for a serial ip address allocation, callers can pass in the option to request // IP allocation serially or first available IP in the subnet var serial bool if opts != nil { if val, ok := opts[ipamapi.AllocSerialPrefix]; ok { serial = (val == "true") } } ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial) if err != nil { return nil, nil, err } return &net.IPNet{IP: ip, Mask: p.Pool.Mask}, nil, nil } // ReleaseAddress releases the address from the specified pool ID func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error { logrus.Debugf("ReleaseAddress(%s, %v)", poolID, address) k := SubnetKey{} if err := k.FromString(poolID); err != nil { return types.BadRequestErrorf("invalid pool id: %s", poolID) } if err := a.refresh(k.AddressSpace); err != nil { return err } aSpace, err := a.getAddrSpace(k.AddressSpace) if err != nil { return err } aSpace.Lock() p, ok := aSpace.subnets[k] if !ok { aSpace.Unlock() return types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID) } if address == nil { aSpace.Unlock() return types.BadRequestErrorf("invalid address: nil") } if !p.Pool.Contains(address) { aSpace.Unlock() return ipamapi.ErrIPOutOfRange } c := p for c.Range != nil { k = c.ParentKey c = aSpace.subnets[k] } aSpace.Unlock() mask := p.Pool.Mask h, err := types.GetHostPartIP(address, mask) if err != nil { return types.InternalErrorf("failed to release address %s: %v", address.String(), err) } bm, err := a.retrieveBitmask(k, c.Pool) if err != nil { return types.InternalErrorf("could not find bitmask in datastore for %s on address %v release from pool %s: %v", k.String(), address, poolID, err) } defer logrus.Debugf("Released address PoolID:%s, Address:%v Sequence:%s", poolID, address, bm.String()) return bm.Unset(ipToUint64(h)) } func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) { var ( ordinal uint64 err error base *net.IPNet ) logrus.Debugf("Request address PoolID:%v %s Serial:%v PrefAddress:%v ", nw, bitmask.String(), serial, prefAddress) base = types.GetIPNetCopy(nw) if bitmask.Unselected() == 0 { return nil, ipamapi.ErrNoAvailableIPs } if ipr == nil && prefAddress == nil { ordinal, err = bitmask.SetAny(serial) } else if prefAddress != nil { hostPart, e := types.GetHostPartIP(prefAddress, base.Mask) if e != nil { return nil, types.InternalErrorf("failed to allocate requested address %s: %v", prefAddress.String(), e) } ordinal = ipToUint64(types.GetMinimalIP(hostPart)) err = bitmask.Set(ordinal) } else { ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial) } switch err { case nil: // Convert IP ordinal for this subnet into IP address return generateAddress(ordinal, base), nil case bitseq.ErrBitAllocated: return nil, ipamapi.ErrIPAlreadyAllocated case bitseq.ErrNoBitAvailable: return nil, ipamapi.ErrNoAvailableIPs default: return nil, err } } // DumpDatabase dumps the internal info func (a *Allocator) DumpDatabase() string { a.Lock() aspaces := make(map[string]*addrSpace, len(a.addrSpaces)) orderedAS := make([]string, 0, len(a.addrSpaces)) for as, aSpace := range a.addrSpaces { orderedAS = append(orderedAS, as) aspaces[as] = aSpace } a.Unlock() sort.Strings(orderedAS) var s string for _, as := range orderedAS { aSpace := aspaces[as] s = fmt.Sprintf("\n\n%s Config", as) aSpace.Lock() for k, config := range aSpace.subnets { s += fmt.Sprintf("\n%v: %v", k, config) if config.Range == nil { a.retrieveBitmask(k, config.Pool) } } aSpace.Unlock() } s = fmt.Sprintf("%s\n\nBitmasks", s) for k, bm := range a.addresses { s += fmt.Sprintf("\n%s: %s", k, bm) } return s } // IsBuiltIn returns true for builtin drivers func (a *Allocator) IsBuiltIn() bool { return true }