package ipam import ( "fmt" "net" "strings" "sync" log "github.com/Sirupsen/logrus" "github.com/docker/libkv/store" "github.com/docker/libnetwork/bitseq" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/types" ) const ( // The biggest configurable host subnets minNetSize = 8 minNetSizeV6 = 64 // The effective network size for v6 minNetSizeV6Eff = 96 // The size of the host subnet used internally, it's the most granular sequence addresses defaultInternalHostSize = 16 // datastore keyes for ipam obkects dsConfigKey = "ipam-config" // ipam-config// dsDataKey = "ipam-data" // ipam-data//// ) // Allocator provides per address space ipv4/ipv6 book keeping type Allocator struct { // The internal subnets host size internalHostSize int // Static subnet information subnets map[subnetKey]*SubnetInfo // Allocated addresses in each address space's internal subnet addresses map[subnetKey]*bitseq.Handle // Datastore store datastore.DataStore App string ID string dbIndex uint64 dbExists bool sync.Mutex } // NewAllocator returns an instance of libnetwork ipam func NewAllocator(ds datastore.DataStore) (*Allocator, error) { a := &Allocator{} a.subnets = make(map[subnetKey]*SubnetInfo) a.addresses = make(map[subnetKey]*bitseq.Handle) a.internalHostSize = defaultInternalHostSize a.store = ds a.App = "ipam" a.ID = dsConfigKey if a.store == nil { return a, nil } // Register for status changes a.watchForChanges() // Get the initial subnet configs status from the ds if present. kvPair, err := a.store.KVStore().Get(datastore.Key(a.Key()...)) if err != nil { if err != store.ErrKeyNotFound { return nil, fmt.Errorf("failed to retrieve the ipam subnet configs from datastore: %v", err) } return a, nil } a.subnetConfigFromStore(kvPair) // Now retrieve the list of small subnets var inserterList []func() error a.Lock() for k, v := range a.subnets { inserterList = append(inserterList, func() error { subnetList, err := getInternalSubnets(v.Subnet, a.internalHostSize) if err != nil { return fmt.Errorf("failed to load address bitmask for configured subnet %s because of %s", v.Subnet.String(), err.Error()) } a.insertAddressMasks(k, subnetList) return nil }) } a.Unlock() // Add the bitmasks, data could come from datastore for _, f := range inserterList { if err := f(); err != nil { return nil, err } } return a, nil } func (a *Allocator) subnetConfigFromStore(kvPair *store.KVPair) { a.Lock() if a.dbIndex < kvPair.LastIndex { a.subnets = byteArrayToSubnets(kvPair.Value) a.dbIndex = kvPair.LastIndex a.dbExists = true } a.Unlock() } // Pointer to the configured subnets in each address space type subnetKey struct { addressSpace AddressSpace subnet string childSubnet string } func (s *subnetKey) String() string { k := fmt.Sprintf("%s/%s", s.addressSpace, s.subnet) if s.childSubnet != "" { k = fmt.Sprintf("%s/%s", k, s.childSubnet) } return k } func (s *subnetKey) FromString(str string) error { if str == "" || !strings.Contains(str, "/") { return fmt.Errorf("invalid string form for subnetkey: %s", str) } p := strings.Split(str, "/") if len(p) != 3 && len(p) != 5 { return fmt.Errorf("invalid string form for subnetkey: %s", str) } s.addressSpace = AddressSpace(p[0]) s.subnet = fmt.Sprintf("%s/%s", p[1], p[2]) if len(p) == 5 { s.childSubnet = fmt.Sprintf("%s/%s", p[1], p[2]) } return nil } func (s *subnetKey) canonicalSubnet() *net.IPNet { if _, sub, err := net.ParseCIDR(s.subnet); err == nil { return sub } return nil } func (s *subnetKey) canonicalChildSubnet() *net.IPNet { if _, sub, err := net.ParseCIDR(s.childSubnet); err == nil { return sub } return nil } type ipVersion int const ( v4 = 4 v6 = 6 ) /******************* * IPAMConf Contract ********************/ // AddSubnet adds a subnet for the specified address space func (a *Allocator) AddSubnet(addrSpace AddressSpace, subnetInfo *SubnetInfo) error { // Sanity check if addrSpace == "" { return ErrInvalidAddressSpace } if subnetInfo == nil || subnetInfo.Subnet == nil { return ErrInvalidSubnet } // Convert to smaller internal subnets (if needed) subnetList, err := getInternalSubnets(subnetInfo.Subnet, a.internalHostSize) if err != nil { return err } retry: if a.contains(addrSpace, subnetInfo) { return ErrOverlapSubnet } // Store the configured subnet and sync to datatstore key := subnetKey{addrSpace, subnetInfo.Subnet.String(), ""} a.Lock() a.subnets[key] = subnetInfo a.Unlock() err = a.writeToStore() if err != nil { if _, ok := err.(types.RetryError); !ok { return types.InternalErrorf("subnet configuration failed because of %s", err.Error()) } // Update to latest if erru := a.readFromStore(); erru != nil { // Restore and bail out a.Lock() delete(a.addresses, key) a.Unlock() return fmt.Errorf("failed to get updated subnets config from datastore (%v) after (%v)", erru, err) } goto retry } // Insert respective bitmasks for this subnet a.insertAddressMasks(key, subnetList) return nil } // Create and insert the internal subnet(s) addresses masks into the address database. Mask data may come from the bitseq datastore. func (a *Allocator) insertAddressMasks(parentKey subnetKey, internalSubnetList []*net.IPNet) error { for _, intSub := range internalSubnetList { var err error ones, bits := intSub.Mask.Size() numAddresses := 1 << uint(bits-ones) smallKey := subnetKey{parentKey.addressSpace, parentKey.subnet, intSub.String()} // Insert the new address masks. AddressMask content may come from datastore a.Lock() a.addresses[smallKey], err = bitseq.NewHandle(dsDataKey, a.store, smallKey.String(), uint32(numAddresses)) a.Unlock() if err != nil { return err } } return nil } // Check subnets size. In case configured subnet is v6 and host size is // greater than 32 bits, adjust subnet to /96. func adjustAndCheckSubnetSize(subnet *net.IPNet) (*net.IPNet, error) { ones, bits := subnet.Mask.Size() if v6 == getAddressVersion(subnet.IP) { if ones < minNetSizeV6 { return nil, ErrInvalidSubnet } if ones < minNetSizeV6Eff { newMask := net.CIDRMask(minNetSizeV6Eff, bits) return &net.IPNet{IP: subnet.IP, Mask: newMask}, nil } } else { if ones < minNetSize { return nil, ErrInvalidSubnet } } return subnet, nil } // Checks whether the passed subnet is a superset or subset of any of the subset in the db func (a *Allocator) contains(space AddressSpace, subInfo *SubnetInfo) bool { a.Lock() defer a.Unlock() for k, v := range a.subnets { if space == k.addressSpace { if subInfo.Subnet.Contains(v.Subnet.IP) || v.Subnet.Contains(subInfo.Subnet.IP) { return true } } } return false } // Splits the passed subnet into N internal subnets with host size equal to internalHostSize. // If the subnet's host size is equal to or smaller than internalHostSize, there won't be any // split and the return list will contain only the passed subnet. func getInternalSubnets(inSubnet *net.IPNet, internalHostSize int) ([]*net.IPNet, error) { var subnetList []*net.IPNet // Sanity check and size adjustment for v6 subnet, err := adjustAndCheckSubnetSize(inSubnet) if err != nil { return subnetList, err } // Get network/host subnet information netBits, bits := subnet.Mask.Size() hostBits := bits - netBits extraBits := hostBits - internalHostSize if extraBits <= 0 { subnetList = make([]*net.IPNet, 1) subnetList[0] = subnet } else { // Split in smaller internal subnets numIntSubs := 1 << uint(extraBits) subnetList = make([]*net.IPNet, numIntSubs) // Construct one copy of the internal subnets's mask intNetBits := bits - internalHostSize intMask := net.CIDRMask(intNetBits, bits) // Construct the prefix portion for each internal subnet for i := 0; i < numIntSubs; i++ { intIP := make([]byte, len(subnet.IP)) copy(intIP, subnet.IP) // IPv6 is too big, just work on the extra portion addIntToIP(intIP, uint32(i< 192.168.53 func addIntToIP(array []byte, ordinal uint32) { for i := len(array) - 1; i >= 0; i-- { array[i] |= (byte)(ordinal & 0xff) ordinal >>= 8 } } // Convert an ordinal to the respective IP address func ipToUint32(ip []byte) uint32 { value := uint32(0) for i := 0; i < len(ip); i++ { j := len(ip) - 1 - i value += uint32(ip[i]) << uint(j*8) } return value } // Given an address and subnet, returns the host portion address func getHostPortionIP(address net.IP, subnet *net.IPNet) net.IP { hostPortion := make([]byte, len(address)) for i := 0; i < len(subnet.Mask); i++ { hostPortion[i] = address[i] &^ subnet.Mask[i] } return hostPortion }