1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
moby--moby/libnetwork/drivers/windows/windows.go
John Stephens c8a3950d8b Delete stale HNS endpoints when creating a network
Signed-off-by: John Stephens <johnstep@docker.com>
2018-03-05 00:12:00 -08:00

867 lines
22 KiB
Go

// +build windows
// Shim for the Host Network Service (HNS) to manage networking for
// Windows Server containers and Hyper-V containers. This module
// is a basic libnetwork driver that passes all the calls to HNS
// It implements the 4 networking modes supported by HNS L2Bridge,
// L2Tunnel, NAT and Transparent(DHCP)
//
// The network are stored in memory and docker daemon ensures discovering
// and loading these networks on startup
package windows
import (
"encoding/json"
"fmt"
"net"
"strconv"
"strings"
"sync"
"github.com/Microsoft/hcsshim"
"github.com/docker/docker/pkg/system"
"github.com/docker/libnetwork/datastore"
"github.com/docker/libnetwork/discoverapi"
"github.com/docker/libnetwork/driverapi"
"github.com/docker/libnetwork/netlabel"
"github.com/docker/libnetwork/types"
"github.com/sirupsen/logrus"
)
// networkConfiguration for network specific configuration
type networkConfiguration struct {
ID string
Type string
Name string
HnsID string
RDID string
VLAN uint
VSID uint
DNSServers string
MacPools []hcsshim.MacPool
DNSSuffix string
SourceMac string
NetworkAdapterName string
dbIndex uint64
dbExists bool
DisableGatewayDNS bool
EnableOutboundNat bool
OutboundNatExceptions []string
}
// endpointConfiguration represents the user specified configuration for the sandbox endpoint
type endpointOption struct {
MacAddress net.HardwareAddr
QosPolicies []types.QosPolicy
DNSServers []string
DisableDNS bool
DisableICC bool
}
// EndpointConnectivity stores the port bindings and exposed ports that the user has specified in epOptions.
type EndpointConnectivity struct {
PortBindings []types.PortBinding
ExposedPorts []types.TransportPort
}
type hnsEndpoint struct {
id string
nid string
profileID string
Type string
//Note: Currently, the sandboxID is the same as the containerID since windows does
//not expose the sandboxID.
//In the future, windows will support a proper sandboxID that is different
//than the containerID.
//Therefore, we are using sandboxID now, so that we won't have to change this code
//when windows properly supports a sandboxID.
sandboxID string
macAddress net.HardwareAddr
epOption *endpointOption // User specified parameters
epConnectivity *EndpointConnectivity // User specified parameters
portMapping []types.PortBinding // Operation port bindings
addr *net.IPNet
gateway net.IP
dbIndex uint64
dbExists bool
}
type hnsNetwork struct {
id string
created bool
config *networkConfiguration
endpoints map[string]*hnsEndpoint // key: endpoint id
driver *driver // The network's driver
sync.Mutex
}
type driver struct {
name string
networks map[string]*hnsNetwork
store datastore.DataStore
sync.Mutex
}
const (
errNotFound = "HNS failed with error : The object identifier does not represent a valid object. "
)
// IsBuiltinLocalDriver validates if network-type is a builtin local-scoped driver
func IsBuiltinLocalDriver(networkType string) bool {
if "l2bridge" == networkType || "l2tunnel" == networkType || "nat" == networkType || "ics" == networkType || "transparent" == networkType {
return true
}
return false
}
// New constructs a new bridge driver
func newDriver(networkType string) *driver {
return &driver{name: networkType, networks: map[string]*hnsNetwork{}}
}
// GetInit returns an initializer for the given network type
func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[string]interface{}) error {
return func(dc driverapi.DriverCallback, config map[string]interface{}) error {
if !IsBuiltinLocalDriver(networkType) {
return types.BadRequestErrorf("Network type not supported: %s", networkType)
}
d := newDriver(networkType)
err := d.initStore(config)
if err != nil {
return err
}
return dc.RegisterDriver(networkType, d, driverapi.Capability{
DataScope: datastore.LocalScope,
ConnectivityScope: datastore.LocalScope,
})
}
}
func (d *driver) getNetwork(id string) (*hnsNetwork, error) {
d.Lock()
defer d.Unlock()
if nw, ok := d.networks[id]; ok {
return nw, nil
}
return nil, types.NotFoundErrorf("network not found: %s", id)
}
func (n *hnsNetwork) getEndpoint(eid string) (*hnsEndpoint, error) {
n.Lock()
defer n.Unlock()
if ep, ok := n.endpoints[eid]; ok {
return ep, nil
}
return nil, types.NotFoundErrorf("Endpoint not found: %s", eid)
}
func (d *driver) parseNetworkOptions(id string, genericOptions map[string]string) (*networkConfiguration, error) {
config := &networkConfiguration{Type: d.name}
for label, value := range genericOptions {
switch label {
case NetworkName:
config.Name = value
case HNSID:
config.HnsID = value
case RoutingDomain:
config.RDID = value
case Interface:
config.NetworkAdapterName = value
case DNSSuffix:
config.DNSSuffix = value
case DNSServers:
config.DNSServers = value
case DisableGatewayDNS:
b, err := strconv.ParseBool(value)
if err != nil {
return nil, err
}
config.DisableGatewayDNS = b
case MacPool:
config.MacPools = make([]hcsshim.MacPool, 0)
s := strings.Split(value, ",")
if len(s)%2 != 0 {
return nil, types.BadRequestErrorf("Invalid mac pool. You must specify both a start range and an end range")
}
for i := 0; i < len(s)-1; i += 2 {
config.MacPools = append(config.MacPools, hcsshim.MacPool{
StartMacAddress: s[i],
EndMacAddress: s[i+1],
})
}
case VLAN:
vlan, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return nil, err
}
config.VLAN = uint(vlan)
case VSID:
vsid, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return nil, err
}
config.VSID = uint(vsid)
case EnableOutboundNat:
if system.GetOSVersion().Build <= 16236 {
return nil, fmt.Errorf("Invalid network option. OutboundNat is not supported on this OS version")
}
b, err := strconv.ParseBool(value)
if err != nil {
return nil, err
}
config.EnableOutboundNat = b
case OutboundNatExceptions:
s := strings.Split(value, ",")
config.OutboundNatExceptions = s
}
}
config.ID = id
config.Type = d.name
return config, nil
}
func (c *networkConfiguration) processIPAM(id string, ipamV4Data, ipamV6Data []driverapi.IPAMData) error {
if len(ipamV6Data) > 0 {
return types.ForbiddenErrorf("windowsshim driver doesn't support v6 subnets")
}
if len(ipamV4Data) == 0 {
return types.BadRequestErrorf("network %s requires ipv4 configuration", id)
}
return nil
}
func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) {
}
func (d *driver) DecodeTableEntry(tablename string, key string, value []byte) (string, map[string]string) {
return "", nil
}
func (d *driver) createNetwork(config *networkConfiguration) error {
network := &hnsNetwork{
id: config.ID,
endpoints: make(map[string]*hnsEndpoint),
config: config,
driver: d,
}
d.Lock()
d.networks[config.ID] = network
d.Unlock()
return nil
}
// Create a new network
func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
if _, err := d.getNetwork(id); err == nil {
return types.ForbiddenErrorf("network %s exists", id)
}
genData, ok := option[netlabel.GenericData].(map[string]string)
if !ok {
return fmt.Errorf("Unknown generic data option")
}
// Parse and validate the config. It should not conflict with existing networks' config
config, err := d.parseNetworkOptions(id, genData)
if err != nil {
return err
}
err = config.processIPAM(id, ipV4Data, ipV6Data)
if err != nil {
return err
}
err = d.createNetwork(config)
if err != nil {
return err
}
// A non blank hnsid indicates that the network was discovered
// from HNS. No need to call HNS if this network was discovered
// from HNS
if config.HnsID == "" {
subnets := []hcsshim.Subnet{}
for _, ipData := range ipV4Data {
subnet := hcsshim.Subnet{
AddressPrefix: ipData.Pool.String(),
}
if ipData.Gateway != nil {
subnet.GatewayAddress = ipData.Gateway.IP.String()
}
subnets = append(subnets, subnet)
}
network := &hcsshim.HNSNetwork{
Name: config.Name,
Type: d.name,
Subnets: subnets,
DNSServerList: config.DNSServers,
DNSSuffix: config.DNSSuffix,
MacPools: config.MacPools,
SourceMac: config.SourceMac,
NetworkAdapterName: config.NetworkAdapterName,
}
if config.VLAN != 0 {
vlanPolicy, err := json.Marshal(hcsshim.VlanPolicy{
Type: "VLAN",
VLAN: config.VLAN,
})
if err != nil {
return err
}
network.Policies = append(network.Policies, vlanPolicy)
}
if config.VSID != 0 {
vsidPolicy, err := json.Marshal(hcsshim.VsidPolicy{
Type: "VSID",
VSID: config.VSID,
})
if err != nil {
return err
}
network.Policies = append(network.Policies, vsidPolicy)
}
if network.Name == "" {
network.Name = id
}
configurationb, err := json.Marshal(network)
if err != nil {
return err
}
configuration := string(configurationb)
logrus.Debugf("HNSNetwork Request =%v Address Space=%v", configuration, subnets)
hnsresponse, err := hcsshim.HNSNetworkRequest("POST", "", configuration)
if err != nil {
return err
}
config.HnsID = hnsresponse.Id
genData[HNSID] = config.HnsID
} else {
// Delete any stale HNS endpoints for this network.
if endpoints, err := hcsshim.HNSListEndpointRequest(); err == nil {
for _, ep := range endpoints {
if ep.VirtualNetwork == config.HnsID {
logrus.Infof("Removing stale HNS endpoint %s", ep.Id)
_, err = hcsshim.HNSEndpointRequest("DELETE", ep.Id, "")
if err != nil {
logrus.Warnf("Error removing HNS endpoint %s", ep.Id)
}
}
}
} else {
logrus.Warnf("Error listing HNS endpoints for network %s", config.HnsID)
}
}
n, err := d.getNetwork(id)
if err != nil {
return err
}
n.created = true
return d.storeUpdate(config)
}
func (d *driver) DeleteNetwork(nid string) error {
n, err := d.getNetwork(nid)
if err != nil {
return types.InternalMaskableErrorf("%s", err)
}
n.Lock()
config := n.config
n.Unlock()
if n.created {
_, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "")
if err != nil && err.Error() != errNotFound {
return types.ForbiddenErrorf(err.Error())
}
}
d.Lock()
delete(d.networks, nid)
d.Unlock()
// delele endpoints belong to this network
for _, ep := range n.endpoints {
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
}
}
return d.storeDelete(config)
}
func convertQosPolicies(qosPolicies []types.QosPolicy) ([]json.RawMessage, error) {
var qps []json.RawMessage
// Enumerate through the qos policies specified by the user and convert
// them into the internal structure matching the JSON blob that can be
// understood by the HCS.
for _, elem := range qosPolicies {
encodedPolicy, err := json.Marshal(hcsshim.QosPolicy{
Type: "QOS",
MaximumOutgoingBandwidthInBytes: elem.MaxEgressBandwidth,
})
if err != nil {
return nil, err
}
qps = append(qps, encodedPolicy)
}
return qps, nil
}
// ConvertPortBindings converts PortBindings to JSON for HNS request
func ConvertPortBindings(portBindings []types.PortBinding) ([]json.RawMessage, error) {
var pbs []json.RawMessage
// Enumerate through the port bindings specified by the user and convert
// them into the internal structure matching the JSON blob that can be
// understood by the HCS.
for _, elem := range portBindings {
proto := strings.ToUpper(elem.Proto.String())
if proto != "TCP" && proto != "UDP" {
return nil, fmt.Errorf("invalid protocol %s", elem.Proto.String())
}
if elem.HostPort != elem.HostPortEnd {
return nil, fmt.Errorf("Windows does not support more than one host port in NAT settings")
}
if len(elem.HostIP) != 0 {
return nil, fmt.Errorf("Windows does not support host IP addresses in NAT settings")
}
encodedPolicy, err := json.Marshal(hcsshim.NatPolicy{
Type: "NAT",
ExternalPort: elem.HostPort,
InternalPort: elem.Port,
Protocol: elem.Proto.String(),
})
if err != nil {
return nil, err
}
pbs = append(pbs, encodedPolicy)
}
return pbs, nil
}
// ParsePortBindingPolicies parses HNS endpoint response message to PortBindings
func ParsePortBindingPolicies(policies []json.RawMessage) ([]types.PortBinding, error) {
var bindings []types.PortBinding
hcsPolicy := &hcsshim.NatPolicy{}
for _, elem := range policies {
if err := json.Unmarshal([]byte(elem), &hcsPolicy); err != nil || hcsPolicy.Type != "NAT" {
continue
}
binding := types.PortBinding{
HostPort: hcsPolicy.ExternalPort,
HostPortEnd: hcsPolicy.ExternalPort,
Port: hcsPolicy.InternalPort,
Proto: types.ParseProtocol(hcsPolicy.Protocol),
HostIP: net.IPv4(0, 0, 0, 0),
}
bindings = append(bindings, binding)
}
return bindings, nil
}
func parseEndpointOptions(epOptions map[string]interface{}) (*endpointOption, error) {
if epOptions == nil {
return nil, nil
}
ec := &endpointOption{}
if opt, ok := epOptions[netlabel.MacAddress]; ok {
if mac, ok := opt.(net.HardwareAddr); ok {
ec.MacAddress = mac
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
if opt, ok := epOptions[QosPolicies]; ok {
if policies, ok := opt.([]types.QosPolicy); ok {
ec.QosPolicies = policies
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
if opt, ok := epOptions[netlabel.DNSServers]; ok {
if dns, ok := opt.([]string); ok {
ec.DNSServers = dns
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
if opt, ok := epOptions[DisableICC]; ok {
if disableICC, ok := opt.(bool); ok {
ec.DisableICC = disableICC
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
if opt, ok := epOptions[DisableDNS]; ok {
if disableDNS, ok := opt.(bool); ok {
ec.DisableDNS = disableDNS
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
return ec, nil
}
// ParseEndpointConnectivity parses options passed to CreateEndpoint, specifically port bindings, and store in a endpointConnectivity object.
func ParseEndpointConnectivity(epOptions map[string]interface{}) (*EndpointConnectivity, error) {
if epOptions == nil {
return nil, nil
}
ec := &EndpointConnectivity{}
if opt, ok := epOptions[netlabel.PortMap]; ok {
if bs, ok := opt.([]types.PortBinding); ok {
ec.PortBindings = bs
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
if opt, ok := epOptions[netlabel.ExposedPorts]; ok {
if ports, ok := opt.([]types.TransportPort); ok {
ec.ExposedPorts = ports
} else {
return nil, fmt.Errorf("Invalid endpoint configuration")
}
}
return ec, nil
}
func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
n, err := d.getNetwork(nid)
if err != nil {
return err
}
// Check if endpoint id is good and retrieve corresponding endpoint
ep, err := n.getEndpoint(eid)
if err == nil && ep != nil {
return driverapi.ErrEndpointExists(eid)
}
endpointStruct := &hcsshim.HNSEndpoint{
VirtualNetwork: n.config.HnsID,
}
epOption, err := parseEndpointOptions(epOptions)
if err != nil {
return err
}
epConnectivity, err := ParseEndpointConnectivity(epOptions)
if err != nil {
return err
}
macAddress := ifInfo.MacAddress()
// Use the macaddress if it was provided
if macAddress != nil {
endpointStruct.MacAddress = strings.Replace(macAddress.String(), ":", "-", -1)
}
endpointStruct.Policies, err = ConvertPortBindings(epConnectivity.PortBindings)
if err != nil {
return err
}
qosPolicies, err := convertQosPolicies(epOption.QosPolicies)
if err != nil {
return err
}
endpointStruct.Policies = append(endpointStruct.Policies, qosPolicies...)
if ifInfo.Address() != nil {
endpointStruct.IPAddress = ifInfo.Address().IP
}
endpointStruct.DNSServerList = strings.Join(epOption.DNSServers, ",")
// overwrite the ep DisableDNS option if DisableGatewayDNS was set to true during the network creation option
if n.config.DisableGatewayDNS {
logrus.Debugf("n.config.DisableGatewayDNS[%v] overwrites epOption.DisableDNS[%v]", n.config.DisableGatewayDNS, epOption.DisableDNS)
epOption.DisableDNS = n.config.DisableGatewayDNS
}
if n.driver.name == "nat" && !epOption.DisableDNS {
logrus.Debugf("endpointStruct.EnableInternalDNS =[%v]", endpointStruct.EnableInternalDNS)
endpointStruct.EnableInternalDNS = true
}
endpointStruct.DisableICC = epOption.DisableICC
// Inherit OutboundNat policy from the network
if n.config.EnableOutboundNat {
outboundNatPolicy, err := json.Marshal(hcsshim.OutboundNatPolicy{
Policy: hcsshim.Policy{Type: hcsshim.OutboundNat},
Exceptions: n.config.OutboundNatExceptions,
})
if err != nil {
return err
}
endpointStruct.Policies = append(endpointStruct.Policies, outboundNatPolicy)
}
configurationb, err := json.Marshal(endpointStruct)
if err != nil {
return err
}
hnsresponse, err := hcsshim.HNSEndpointRequest("POST", "", string(configurationb))
if err != nil {
return err
}
mac, err := net.ParseMAC(hnsresponse.MacAddress)
if err != nil {
return err
}
// TODO For now the ip mask is not in the info generated by HNS
endpoint := &hnsEndpoint{
id: eid,
nid: n.id,
Type: d.name,
addr: &net.IPNet{IP: hnsresponse.IPAddress, Mask: hnsresponse.IPAddress.DefaultMask()},
macAddress: mac,
}
if hnsresponse.GatewayAddress != "" {
endpoint.gateway = net.ParseIP(hnsresponse.GatewayAddress)
}
endpoint.profileID = hnsresponse.Id
endpoint.epConnectivity = epConnectivity
endpoint.epOption = epOption
endpoint.portMapping, err = ParsePortBindingPolicies(hnsresponse.Policies)
if err != nil {
hcsshim.HNSEndpointRequest("DELETE", hnsresponse.Id, "")
return err
}
n.Lock()
n.endpoints[eid] = endpoint
n.Unlock()
if ifInfo.Address() == nil {
ifInfo.SetIPAddress(endpoint.addr)
}
if macAddress == nil {
ifInfo.SetMacAddress(endpoint.macAddress)
}
if err = d.storeUpdate(endpoint); err != nil {
logrus.Errorf("Failed to save endpoint %s to store: %v", endpoint.id[0:7], err)
}
return nil
}
func (d *driver) DeleteEndpoint(nid, eid string) error {
n, err := d.getNetwork(nid)
if err != nil {
return types.InternalMaskableErrorf("%s", err)
}
ep, err := n.getEndpoint(eid)
if err != nil {
return err
}
n.Lock()
delete(n.endpoints, eid)
n.Unlock()
_, err = hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "")
if err != nil && err.Error() != errNotFound {
return err
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
}
return nil
}
func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
network, err := d.getNetwork(nid)
if err != nil {
return nil, err
}
ep, err := network.getEndpoint(eid)
if err != nil {
return nil, err
}
data := make(map[string]interface{}, 1)
if network.driver.name == "nat" {
data["AllowUnqualifiedDNSQuery"] = true
}
data["hnsid"] = ep.profileID
if ep.epConnectivity.ExposedPorts != nil {
// Return a copy of the config data
epc := make([]types.TransportPort, 0, len(ep.epConnectivity.ExposedPorts))
for _, tp := range ep.epConnectivity.ExposedPorts {
epc = append(epc, tp.GetCopy())
}
data[netlabel.ExposedPorts] = epc
}
if ep.portMapping != nil {
// Return a copy of the operational data
pmc := make([]types.PortBinding, 0, len(ep.portMapping))
for _, pm := range ep.portMapping {
pmc = append(pmc, pm.GetCopy())
}
data[netlabel.PortMap] = pmc
}
if len(ep.macAddress) != 0 {
data[netlabel.MacAddress] = ep.macAddress
}
return data, nil
}
// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
network, err := d.getNetwork(nid)
if err != nil {
return err
}
// Ensure that the endpoint exists
endpoint, err := network.getEndpoint(eid)
if err != nil {
return err
}
err = jinfo.SetGateway(endpoint.gateway)
if err != nil {
return err
}
endpoint.sandboxID = sboxKey
err = hcsshim.HotAttachEndpoint(endpoint.sandboxID, endpoint.profileID)
if err != nil {
// If container doesn't exists in hcs, do not throw error for hot add/remove
if err != hcsshim.ErrComputeSystemDoesNotExist {
return err
}
}
jinfo.DisableGatewayService()
return nil
}
// Leave method is invoked when a Sandbox detaches from an endpoint.
func (d *driver) Leave(nid, eid string) error {
network, err := d.getNetwork(nid)
if err != nil {
return types.InternalMaskableErrorf("%s", err)
}
// Ensure that the endpoint exists
endpoint, err := network.getEndpoint(eid)
if err != nil {
return err
}
err = hcsshim.HotDetachEndpoint(endpoint.sandboxID, endpoint.profileID)
if err != nil {
// If container doesn't exists in hcs, do not throw error for hot add/remove
if err != hcsshim.ErrComputeSystemDoesNotExist {
return err
}
}
return nil
}
func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
return nil
}
func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
return nil
}
func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
return nil, types.NotImplementedErrorf("not implemented")
}
func (d *driver) NetworkFree(id string) error {
return types.NotImplementedErrorf("not implemented")
}
func (d *driver) Type() string {
return d.name
}
func (d *driver) IsBuiltIn() bool {
return true
}
// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
return nil
}
// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
return nil
}