diff --git a/libnetwork/config/config.go b/libnetwork/config/config.go index 2bae6f459f..b14691e2f7 100644 --- a/libnetwork/config/config.go +++ b/libnetwork/config/config.go @@ -15,9 +15,10 @@ import ( // Config encapsulates configurations of various Libnetwork components type Config struct { - Daemon DaemonCfg - Cluster ClusterCfg - Scopes map[string]*datastore.ScopeCfg + Daemon DaemonCfg + Cluster ClusterCfg + Scopes map[string]*datastore.ScopeCfg + ActiveSandboxes map[string]interface{} } // DaemonCfg represents libnetwork core configuration @@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option { c.Scopes[datastore.LocalScope].Client.Config = config } } + +// OptionActiveSandboxes function returns an option setter for passing the sandboxes +// which were active during previous daemon life +func OptionActiveSandboxes(sandboxes map[string]interface{}) Option { + return func(c *Config) { + c.ActiveSandboxes = sandboxes + } +} diff --git a/libnetwork/controller.go b/libnetwork/controller.go index 551e888b40..3655707bf6 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } } - // Reserve pools first before doing cleanup. This is because - // if the pools are not populated properly, the cleanups of - // endpoint/network and sandbox below will not be able to - // release ip subnets and addresses properly into the pool - // because the pools won't exist. + // Reserve pools first before doing cleanup. Otherwise the + // cleanups of endpoint/network and sandbox below will + // generate many unnecessary warnings c.reservePools() // Cleanup resources - c.sandboxCleanup() + c.sandboxCleanup(c.cfg.ActiveSandboxes) c.cleanupLocalEndpoints() c.networkCleanup() @@ -671,9 +669,27 @@ func (c *controller) reservePools() { c.Gateway = n.ipamV6Info[i].Gateway.IP.String() } } + // Reserve pools if err := n.ipamAllocate(); err != nil { log.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err) } + // Reserve existing endpoints' addresses + ipam, _, err := n.getController().getIPAMDriver(n.ipamType) + if err != nil { + log.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID()) + continue + } + epl, err := n.getEndpointsFromStore() + if err != nil { + log.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID()) + continue + } + for _, ep := range epl { + if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil { + log.Warnf("Failed to reserve current adress for endpoint %q (%s) on network %q (%s)", + ep.Name(), ep.ID(), n.Name(), n.ID()) + } + } } } @@ -832,7 +848,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s if sb.config.useDefaultSandBox { c.sboxOnce.Do(func() { - c.defOsSbox, err = osl.NewSandbox(sb.Key(), false) + c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false) }) if err != nil { @@ -844,7 +860,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s } if sb.osSbox == nil && !sb.config.useExternalKey { - if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil { + if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil { return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) } } diff --git a/libnetwork/drivers/bridge/bridge.go b/libnetwork/drivers/bridge/bridge.go index 4064e6272a..6d761a0e26 100644 --- a/libnetwork/drivers/bridge/bridge.go +++ b/libnetwork/drivers/bridge/bridge.go @@ -91,6 +91,7 @@ type connectivityConfiguration struct { type bridgeEndpoint struct { id string + nid string srcName string addr *net.IPNet addrv6 *net.IPNet @@ -99,6 +100,8 @@ type bridgeEndpoint struct { containerConfig *containerConfiguration extConnConfig *connectivityConfiguration portMapping []types.PortBinding // Operation port bindings + dbIndex uint64 + dbExists bool } type bridgeNetwork struct { @@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, // Create and add the endpoint n.Lock() - endpoint := &bridgeEndpoint{id: eid, config: epConfig} + endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig} n.endpoints[eid] = endpoint n.Unlock() @@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } } + if err = d.storeUpdate(endpoint); err != nil { + return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err) + } + return nil } @@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { d.nlh.LinkDel(link) } + if err := d.storeDelete(ep); err != nil { + logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err) + } + return nil } @@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string return err } + if err = d.storeUpdate(endpoint); err != nil { + endpoint.portMapping = nil + return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err) + } + if !network.config.EnableICC { return d.link(network, endpoint, true) } diff --git a/libnetwork/drivers/bridge/bridge_store.go b/libnetwork/drivers/bridge/bridge_store.go index e10a429ed2..0134c54072 100644 --- a/libnetwork/drivers/bridge/bridge_store.go +++ b/libnetwork/drivers/bridge/bridge_store.go @@ -12,7 +12,13 @@ import ( "github.com/docker/libnetwork/types" ) -const bridgePrefix = "bridge" +const ( + // network config prefix was not specific enough. + // To be backward compatible, need custom endpoint + // prefix with different root + bridgePrefix = "bridge" + bridgeEndpointPrefix = "bridge-endpoint" +) func (d *driver) initStore(option map[string]interface{}) error { if data, ok := option[netlabel.LocalKVClient]; ok { @@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error { return types.InternalErrorf("bridge driver failed to initialize data store: %v", err) } - return d.populateNetworks() + err = d.populateNetworks() + if err != nil { + return err + } + + err = d.populateEndpoints() + if err != nil { + return err + } } return nil @@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error { if err = d.createNetwork(ncfg); err != nil { logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err) } + logrus.Debugf("Network (%s) restored", ncfg.ID[0:7]) + } + + return nil +} + +func (d *driver) populateEndpoints() error { + kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to get bridge endpoints from store: %v", err) + } + + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ep := kvo.(*bridgeEndpoint) + n, ok := d.networks[ep.nid] + if !ok { + logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7]) + logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7]) + if err := d.storeDelete(ep); err != nil { + logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7]) + } + continue + } + n.endpoints[ep.id] = ep + n.restorePortAllocations(ep) + logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7]) } return nil @@ -184,7 +228,7 @@ func (ncfg *networkConfiguration) Exists() bool { } func (ncfg *networkConfiguration) Skip() bool { - return ncfg.DefaultBridge + return false } func (ncfg *networkConfiguration) New() datastore.KVObject { @@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error { func (ncfg *networkConfiguration) DataScope() string { return datastore.LocalScope } + +func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + epMap["id"] = ep.id + epMap["nid"] = ep.nid + epMap["SrcName"] = ep.srcName + epMap["MacAddress"] = ep.macAddress.String() + epMap["Addr"] = ep.addr.String() + if ep.addrv6 != nil { + epMap["Addrv6"] = ep.addrv6.String() + } + epMap["Config"] = ep.config + epMap["ContainerConfig"] = ep.containerConfig + epMap["ExternalConnConfig"] = ep.extConnConfig + epMap["PortMapping"] = ep.portMapping + + return json.Marshal(epMap) +} + +func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &epMap); err != nil { + return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err) + } + + if v, ok := epMap["MacAddress"]; ok { + if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addr"]; ok { + if ep.addr, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addrv6"]; ok { + if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err) + } + } + ep.id = epMap["id"].(string) + ep.nid = epMap["nid"].(string) + ep.srcName = epMap["SrcName"].(string) + d, _ := json.Marshal(epMap["Config"]) + if err := json.Unmarshal(d, &ep.config); err != nil { + logrus.Warnf("Failed to decode endpoint config %v", err) + } + d, _ = json.Marshal(epMap["ContainerConfig"]) + if err := json.Unmarshal(d, &ep.containerConfig); err != nil { + logrus.Warnf("Failed to decode endpoint container config %v", err) + } + d, _ = json.Marshal(epMap["ExternalConnConfig"]) + if err := json.Unmarshal(d, &ep.extConnConfig); err != nil { + logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err) + } + d, _ = json.Marshal(epMap["PortMapping"]) + if err := json.Unmarshal(d, &ep.portMapping); err != nil { + logrus.Warnf("Failed to decode endpoint port mapping %v", err) + } + + return nil +} + +func (ep *bridgeEndpoint) Key() []string { + return []string{bridgeEndpointPrefix, ep.id} +} + +func (ep *bridgeEndpoint) KeyPrefix() []string { + return []string{bridgeEndpointPrefix} +} + +func (ep *bridgeEndpoint) Value() []byte { + b, err := json.Marshal(ep) + if err != nil { + return nil + } + return b +} + +func (ep *bridgeEndpoint) SetValue(value []byte) error { + return json.Unmarshal(value, ep) +} + +func (ep *bridgeEndpoint) Index() uint64 { + return ep.dbIndex +} + +func (ep *bridgeEndpoint) SetIndex(index uint64) { + ep.dbIndex = index + ep.dbExists = true +} + +func (ep *bridgeEndpoint) Exists() bool { + return ep.dbExists +} + +func (ep *bridgeEndpoint) Skip() bool { + return false +} + +func (ep *bridgeEndpoint) New() datastore.KVObject { + return &bridgeEndpoint{} +} + +func (ep *bridgeEndpoint) CopyTo(o datastore.KVObject) error { + dstEp := o.(*bridgeEndpoint) + *dstEp = *ep + return nil +} + +func (ep *bridgeEndpoint) DataScope() string { + return datastore.LocalScope +} + +func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) { + if ep.extConnConfig == nil || + ep.extConnConfig.ExposedPorts == nil || + ep.extConnConfig.PortBindings == nil { + return + } + tmp := ep.extConnConfig.PortBindings + ep.extConnConfig.PortBindings = ep.portMapping + _, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy) + if err != nil { + logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err) + } + ep.extConnConfig.PortBindings = tmp +} diff --git a/libnetwork/drivers/bridge/bridge_test.go b/libnetwork/drivers/bridge/bridge_test.go index 53445d1412..6c53b162c9 100644 --- a/libnetwork/drivers/bridge/bridge_test.go +++ b/libnetwork/drivers/bridge/bridge_test.go @@ -1,6 +1,8 @@ package bridge import ( + "bytes" + "encoding/json" "fmt" "net" "regexp" @@ -20,6 +22,150 @@ func init() { ipamutils.InitNetworks() } +func TestEndpointMarshalling(t *testing.T) { + ip1, _ := types.ParseCIDR("172.22.0.9/16") + ip2, _ := types.ParseCIDR("2001:db8::9") + mac, _ := net.ParseMAC("ac:bd:24:57:66:77") + e := &bridgeEndpoint{ + id: "d2c015a1fe5930650cbcd50493efba0500bcebd8ee1f4401a16319f8a567de33", + nid: "ee33fbb43c323f1920b6b35a0101552ac22ede960d0e5245e9738bccc68b2415", + addr: ip1, + addrv6: ip2, + macAddress: mac, + srcName: "veth123456", + config: &endpointConfiguration{MacAddress: mac}, + containerConfig: &containerConfiguration{ + ParentEndpoints: []string{"one", "due", "three"}, + ChildEndpoints: []string{"four", "five", "six"}, + }, + extConnConfig: &connectivityConfiguration{ + ExposedPorts: []types.TransportPort{ + { + Proto: 6, + Port: uint16(18), + }, + }, + PortBindings: []types.PortBinding{ + { + Proto: 6, + IP: net.ParseIP("17210.33.9.56"), + Port: uint16(18), + HostPort: uint16(3000), + HostPortEnd: uint16(14000), + }, + }, + }, + portMapping: []types.PortBinding{ + { + Proto: 17, + IP: net.ParseIP("172.33.9.56"), + Port: uint16(99), + HostIP: net.ParseIP("10.10.100.2"), + HostPort: uint16(9900), + HostPortEnd: uint16(10000), + }, + { + Proto: 6, + IP: net.ParseIP("171.33.9.56"), + Port: uint16(55), + HostIP: net.ParseIP("10.11.100.2"), + HostPort: uint16(5500), + HostPortEnd: uint16(55000), + }, + }, + } + + b, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + + ee := &bridgeEndpoint{} + err = json.Unmarshal(b, ee) + if err != nil { + t.Fatal(err) + } + + if e.id != ee.id || e.nid != ee.nid || e.srcName != ee.srcName || !bytes.Equal(e.macAddress, ee.macAddress) || + !types.CompareIPNet(e.addr, ee.addr) || !types.CompareIPNet(e.addrv6, ee.addrv6) || + !compareEpConfig(e.config, ee.config) || + !compareContainerConfig(e.containerConfig, ee.containerConfig) || + !compareConnConfig(e.extConnConfig, ee.extConnConfig) || + !compareBindings(e.portMapping, ee.portMapping) { + t.Fatalf("JSON marsh/unmarsh failed.\nOriginal:\n%#v\nDecoded:\n%#v", e, ee) + } +} + +func compareEpConfig(a, b *endpointConfiguration) bool { + if a == b { + return true + } + if a == nil || b == nil { + return false + } + return bytes.Equal(a.MacAddress, b.MacAddress) +} + +func compareContainerConfig(a, b *containerConfiguration) bool { + if a == b { + return true + } + if a == nil || b == nil { + return false + } + if len(a.ParentEndpoints) != len(b.ParentEndpoints) || + len(a.ChildEndpoints) != len(b.ChildEndpoints) { + return false + } + for i := 0; i < len(a.ParentEndpoints); i++ { + if a.ParentEndpoints[i] != b.ParentEndpoints[i] { + return false + } + } + for i := 0; i < len(a.ChildEndpoints); i++ { + if a.ChildEndpoints[i] != b.ChildEndpoints[i] { + return false + } + } + return true +} + +func compareConnConfig(a, b *connectivityConfiguration) bool { + if a == b { + return true + } + if a == nil || b == nil { + return false + } + if len(a.ExposedPorts) != len(b.ExposedPorts) || + len(a.PortBindings) != len(b.PortBindings) { + return false + } + for i := 0; i < len(a.ExposedPorts); i++ { + if !a.ExposedPorts[i].Equal(&b.ExposedPorts[i]) { + return false + } + } + for i := 0; i < len(a.PortBindings); i++ { + if !a.PortBindings[i].Equal(&b.PortBindings[i]) { + return false + } + } + return true +} + +func compareBindings(a, b []types.PortBinding) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if !a[i].Equal(&b[i]) { + return false + } + } + return true +} + func getIPv4Data(t *testing.T) []driverapi.IPAMData { ipd := driverapi.IPAMData{AddressSpace: "full"} nw, _, err := netutils.ElectInterfaceAddresses("") diff --git a/libnetwork/drivers/ipvlan/ipvlan.go b/libnetwork/drivers/ipvlan/ipvlan.go index 8ea44fcbb4..aacea3df80 100644 --- a/libnetwork/drivers/ipvlan/ipvlan.go +++ b/libnetwork/drivers/ipvlan/ipvlan.go @@ -36,11 +36,14 @@ type driver struct { } type endpoint struct { - id string - mac net.HardwareAddr - addr *net.IPNet - addrv6 *net.IPNet - srcName string + id string + nid string + mac net.HardwareAddr + addr *net.IPNet + addrv6 *net.IPNet + srcName string + dbIndex uint64 + dbExists bool } type network struct { diff --git a/libnetwork/drivers/ipvlan/ipvlan_endpoint.go b/libnetwork/drivers/ipvlan/ipvlan_endpoint.go index 204c83f74b..76e6cdef09 100644 --- a/libnetwork/drivers/ipvlan/ipvlan_endpoint.go +++ b/libnetwork/drivers/ipvlan/ipvlan_endpoint.go @@ -28,9 +28,9 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } ep := &endpoint{ id: eid, + nid: nid, addr: ifInfo.Address(), addrv6: ifInfo.AddressIPv6(), - mac: ifInfo.MacAddress(), } if ep.addr == nil { return fmt.Errorf("create endpoint was not passed an IP address") @@ -51,6 +51,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } } } + + if err := d.storeUpdate(ep); err != nil { + return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err) + } + n.addEndpoint(ep) return nil @@ -74,5 +79,9 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { ns.NlHandle().LinkDel(link) } + if err := d.storeDelete(ep); err != nil { + logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err) + } + return nil } diff --git a/libnetwork/drivers/ipvlan/ipvlan_joinleave.go b/libnetwork/drivers/ipvlan/ipvlan_joinleave.go index b0be3d68d7..0c08dfce5d 100644 --- a/libnetwork/drivers/ipvlan/ipvlan_joinleave.go +++ b/libnetwork/drivers/ipvlan/ipvlan_joinleave.go @@ -116,6 +116,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, if err != nil { return err } + if err = d.storeUpdate(ep); err != nil { + return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err) + } return nil } diff --git a/libnetwork/drivers/ipvlan/ipvlan_store.go b/libnetwork/drivers/ipvlan/ipvlan_store.go index c6430835ae..5284e88e4d 100644 --- a/libnetwork/drivers/ipvlan/ipvlan_store.go +++ b/libnetwork/drivers/ipvlan/ipvlan_store.go @@ -3,6 +3,7 @@ package ipvlan import ( "encoding/json" "fmt" + "net" "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" @@ -11,7 +12,11 @@ import ( "github.com/docker/libnetwork/types" ) -const ipvlanPrefix = "ipvlan" // prefix used for persistent driver storage +const ( + ipvlanPrefix = "ipvlan" + ipvlanNetworkPrefix = ipvlanPrefix + "/network" + ipvlanEndpointPrefix = ipvlanPrefix + "/endpoint" +) // networkConfiguration for this driver's network specific configuration type configuration struct { @@ -58,7 +63,7 @@ func (d *driver) initStore(option map[string]interface{}) error { // populateNetworks is invoked at driver init to recreate persistently stored networks func (d *driver) populateNetworks() error { - kvol, err := d.store.List(datastore.Key(ipvlanPrefix), &configuration{}) + kvol, err := d.store.List(datastore.Key(ipvlanNetworkPrefix), &configuration{}) if err != nil && err != datastore.ErrKeyNotFound { return fmt.Errorf("failed to get ipvlan network configurations from store: %v", err) } @@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error { return nil } +func (d *driver) populateEndpoints() error { + kvol, err := d.store.List(datastore.Key(ipvlanEndpointPrefix), &endpoint{}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to get ipvlan endpoints from store: %v", err) + } + + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ep := kvo.(*endpoint) + n, ok := d.networks[ep.nid] + if !ok { + logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7]) + logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.nid[0:7]) + if err := d.storeDelete(ep); err != nil { + logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.nid[0:7]) + } + continue + } + n.endpoints[ep.id] = ep + logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7]) + } + + return nil +} + // storeUpdate used to update persistent ipvlan network records as they are created func (d *driver) storeUpdate(kvObject datastore.KVObject) error { if d.store == nil { @@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error { } func (config *configuration) Key() []string { - return []string{ipvlanPrefix, config.ID} + return []string{ipvlanNetworkPrefix, config.ID} } func (config *configuration) KeyPrefix() []string { - return []string{ipvlanPrefix} + return []string{ipvlanNetworkPrefix} } func (config *configuration) Value() []byte { @@ -214,3 +247,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error { func (config *configuration) DataScope() string { return datastore.LocalScope } + +func (ep *endpoint) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + epMap["id"] = ep.id + epMap["nid"] = ep.nid + epMap["SrcName"] = ep.srcName + if len(ep.mac) != 0 { + epMap["MacAddress"] = ep.mac.String() + } + if ep.addr != nil { + epMap["Addr"] = ep.addr.String() + } + if ep.addrv6 != nil { + epMap["Addrv6"] = ep.addrv6.String() + } + return json.Marshal(epMap) +} + +func (ep *endpoint) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &epMap); err != nil { + return fmt.Errorf("Failed to unmarshal to ipvlan endpoint: %v", err) + } + + if v, ok := epMap["MacAddress"]; ok { + if ep.mac, err = net.ParseMAC(v.(string)); err != nil { + return types.InternalErrorf("failed to decode ipvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addr"]; ok { + if ep.addr, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode ipvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addrv6"]; ok { + if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode ipvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err) + } + } + ep.id = epMap["id"].(string) + ep.nid = epMap["nid"].(string) + ep.srcName = epMap["SrcName"].(string) + + return nil +} + +func (ep *endpoint) Key() []string { + return []string{ipvlanEndpointPrefix, ep.id} +} + +func (ep *endpoint) KeyPrefix() []string { + return []string{ipvlanEndpointPrefix} +} + +func (ep *endpoint) Value() []byte { + b, err := json.Marshal(ep) + if err != nil { + return nil + } + return b +} + +func (ep *endpoint) SetValue(value []byte) error { + return json.Unmarshal(value, ep) +} + +func (ep *endpoint) Index() uint64 { + return ep.dbIndex +} + +func (ep *endpoint) SetIndex(index uint64) { + ep.dbIndex = index + ep.dbExists = true +} + +func (ep *endpoint) Exists() bool { + return ep.dbExists +} + +func (ep *endpoint) Skip() bool { + return false +} + +func (ep *endpoint) New() datastore.KVObject { + return &endpoint{} +} + +func (ep *endpoint) CopyTo(o datastore.KVObject) error { + dstEp := o.(*endpoint) + *dstEp = *ep + return nil +} + +func (ep *endpoint) DataScope() string { + return datastore.LocalScope +} diff --git a/libnetwork/drivers/macvlan/macvlan.go b/libnetwork/drivers/macvlan/macvlan.go index 5ace97f90c..b89b4b7845 100644 --- a/libnetwork/drivers/macvlan/macvlan.go +++ b/libnetwork/drivers/macvlan/macvlan.go @@ -38,11 +38,14 @@ type driver struct { } type endpoint struct { - id string - mac net.HardwareAddr - addr *net.IPNet - addrv6 *net.IPNet - srcName string + id string + nid string + mac net.HardwareAddr + addr *net.IPNet + addrv6 *net.IPNet + srcName string + dbIndex uint64 + dbExists bool } type network struct { diff --git a/libnetwork/drivers/macvlan/macvlan_endpoint.go b/libnetwork/drivers/macvlan/macvlan_endpoint.go index 3187a54562..54844c93da 100644 --- a/libnetwork/drivers/macvlan/macvlan_endpoint.go +++ b/libnetwork/drivers/macvlan/macvlan_endpoint.go @@ -26,6 +26,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } ep := &endpoint{ id: eid, + nid: nid, addr: ifInfo.Address(), addrv6: ifInfo.AddressIPv6(), mac: ifInfo.MacAddress(), @@ -55,6 +56,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } } } + + if err := d.storeUpdate(ep); err != nil { + return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err) + } + n.addEndpoint(ep) return nil @@ -77,6 +83,8 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil { ns.NlHandle().LinkDel(link) } - + if err := d.storeDelete(ep); err != nil { + logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err) + } return nil } diff --git a/libnetwork/drivers/macvlan/macvlan_joinleave.go b/libnetwork/drivers/macvlan/macvlan_joinleave.go index 3656fdfe3f..cf5c2a4bf9 100644 --- a/libnetwork/drivers/macvlan/macvlan_joinleave.go +++ b/libnetwork/drivers/macvlan/macvlan_joinleave.go @@ -77,7 +77,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, if err != nil { return err } - + if err := d.storeUpdate(ep); err != nil { + return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err) + } return nil } diff --git a/libnetwork/drivers/macvlan/macvlan_store.go b/libnetwork/drivers/macvlan/macvlan_store.go index 5f92feadd4..9b6f299cfd 100644 --- a/libnetwork/drivers/macvlan/macvlan_store.go +++ b/libnetwork/drivers/macvlan/macvlan_store.go @@ -3,6 +3,7 @@ package macvlan import ( "encoding/json" "fmt" + "net" "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" @@ -11,7 +12,11 @@ import ( "github.com/docker/libnetwork/types" ) -const macvlanPrefix = "macvlan" // prefix used for persistent driver storage +const ( + macvlanPrefix = "macvlan" + macvlanNetworkPrefix = macvlanPrefix + "/network" + macvlanEndpointPrefix = macvlanPrefix + "/endpoint" +) // networkConfiguration for this driver's network specific configuration type configuration struct { @@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error { return nil } +func (d *driver) populateEndpoints() error { + kvol, err := d.store.List(datastore.Key(macvlanEndpointPrefix), &endpoint{}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to get macvlan endpoints from store: %v", err) + } + + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ep := kvo.(*endpoint) + n, ok := d.networks[ep.nid] + if !ok { + logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7]) + logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.nid[0:7]) + if err := d.storeDelete(ep); err != nil { + logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.nid[0:7]) + } + continue + } + n.endpoints[ep.id] = ep + logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7]) + } + + return nil +} + // storeUpdate used to update persistent macvlan network records as they are created func (d *driver) storeUpdate(kvObject datastore.KVObject) error { if d.store == nil { @@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error { } func (config *configuration) Key() []string { - return []string{macvlanPrefix, config.ID} + return []string{macvlanNetworkPrefix, config.ID} } func (config *configuration) KeyPrefix() []string { - return []string{macvlanPrefix} + return []string{macvlanNetworkPrefix} } func (config *configuration) Value() []byte { @@ -216,3 +249,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error { func (config *configuration) DataScope() string { return datastore.LocalScope } + +func (ep *endpoint) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + epMap["id"] = ep.id + epMap["nid"] = ep.nid + epMap["SrcName"] = ep.srcName + if len(ep.mac) != 0 { + epMap["MacAddress"] = ep.mac.String() + } + if ep.addr != nil { + epMap["Addr"] = ep.addr.String() + } + if ep.addrv6 != nil { + epMap["Addrv6"] = ep.addrv6.String() + } + return json.Marshal(epMap) +} + +func (ep *endpoint) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &epMap); err != nil { + return fmt.Errorf("Failed to unmarshal to macvlan endpoint: %v", err) + } + + if v, ok := epMap["MacAddress"]; ok { + if ep.mac, err = net.ParseMAC(v.(string)); err != nil { + return types.InternalErrorf("failed to decode macvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addr"]; ok { + if ep.addr, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode macvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addrv6"]; ok { + if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode macvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err) + } + } + ep.id = epMap["id"].(string) + ep.nid = epMap["nid"].(string) + ep.srcName = epMap["SrcName"].(string) + + return nil +} + +func (ep *endpoint) Key() []string { + return []string{macvlanEndpointPrefix, ep.id} +} + +func (ep *endpoint) KeyPrefix() []string { + return []string{macvlanEndpointPrefix} +} + +func (ep *endpoint) Value() []byte { + b, err := json.Marshal(ep) + if err != nil { + return nil + } + return b +} + +func (ep *endpoint) SetValue(value []byte) error { + return json.Unmarshal(value, ep) +} + +func (ep *endpoint) Index() uint64 { + return ep.dbIndex +} + +func (ep *endpoint) SetIndex(index uint64) { + ep.dbIndex = index + ep.dbExists = true +} + +func (ep *endpoint) Exists() bool { + return ep.dbExists +} + +func (ep *endpoint) Skip() bool { + return false +} + +func (ep *endpoint) New() datastore.KVObject { + return &endpoint{} +} + +func (ep *endpoint) CopyTo(o datastore.KVObject) error { + dstEp := o.(*endpoint) + *dstEp = *ep + return nil +} + +func (ep *endpoint) DataScope() string { + return datastore.LocalScope +} diff --git a/libnetwork/drivers/overlay/encryption.go b/libnetwork/drivers/overlay/encryption.go index fc82ac3700..0f9a5e4767 100644 --- a/libnetwork/drivers/overlay/encryption.go +++ b/libnetwork/drivers/overlay/encryption.go @@ -10,6 +10,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" + "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" "strconv" @@ -214,12 +215,12 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f var ( crypt *netlink.XfrmStateAlgo action = "Removing" - xfrmProgram = netlink.XfrmStateDel + xfrmProgram = ns.NlHandle().XfrmStateDel ) if add { action = "Adding" - xfrmProgram = netlink.XfrmStateAdd + xfrmProgram = ns.NlHandle().XfrmStateAdd crypt = &netlink.XfrmStateAlgo{Name: "cbc(aes)", Key: k.value} } @@ -278,10 +279,10 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { action := "Removing" - xfrmProgram := netlink.XfrmPolicyDel + xfrmProgram := ns.NlHandle().XfrmPolicyDel if add { action = "Adding" - xfrmProgram = netlink.XfrmPolicyAdd + xfrmProgram = ns.NlHandle().XfrmPolicyAdd } fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src)) @@ -322,7 +323,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { } func saExists(sa *netlink.XfrmState) (bool, error) { - _, err := netlink.XfrmStateGet(sa) + _, err := ns.NlHandle().XfrmStateGet(sa) switch err { case nil: return true, nil @@ -336,7 +337,7 @@ func saExists(sa *netlink.XfrmState) (bool, error) { } func spExists(sp *netlink.XfrmPolicy) (bool, error) { - _, err := netlink.XfrmPolicyGet(sp) + _, err := ns.NlHandle().XfrmPolicyGet(sp) switch err { case nil: return true, nil @@ -482,7 +483,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, Limits: netlink.XfrmStateLimits{TimeSoft: timeout}, } log.Infof("Updating rSA0{%s}", rSA0) - if err := netlink.XfrmStateUpdate(rSA0); err != nil { + if err := ns.NlHandle().XfrmStateUpdate(rSA0); err != nil { log.Warnf("Failed to update rSA0{%s}: %v", rSA0, err) } } @@ -518,7 +519,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, }, } log.Infof("Updating fSP{%s}", fSP1) - if err := netlink.XfrmPolicyUpdate(fSP1); err != nil { + if err := ns.NlHandle().XfrmPolicyUpdate(fSP1); err != nil { log.Warnf("Failed to update fSP{%s}: %v", fSP1, err) } @@ -533,7 +534,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, Limits: netlink.XfrmStateLimits{TimeHard: timeout}, } log.Infof("Removing fSA0{%s}", fSA0) - if err := netlink.XfrmStateUpdate(fSA0); err != nil { + if err := ns.NlHandle().XfrmStateUpdate(fSA0); err != nil { log.Warnf("Failed to remove fSA0{%s}: %v", fSA0, err) } } diff --git a/libnetwork/drivers/overlay/joinleave.go b/libnetwork/drivers/overlay/joinleave.go index 48a9fcd25e..8cdf3194de 100644 --- a/libnetwork/drivers/overlay/joinleave.go +++ b/libnetwork/drivers/overlay/joinleave.go @@ -40,11 +40,11 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err) } - if err := n.joinSandbox(); err != nil { + if err := n.joinSandbox(false); err != nil { return fmt.Errorf("network sandbox join failed: %v", err) } - if err := n.joinSubnetSandbox(s); err != nil { + if err := n.joinSubnetSandbox(s, false); err != nil { return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err) } @@ -61,6 +61,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, ep.ifName = containerIfName + if err := d.writeEndpointToStore(ep); err != nil { + return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err) + } + nlh := ns.NlHandle() // Set the container interface and its peer MTU to 1450 to allow diff --git a/libnetwork/drivers/overlay/ov_endpoint.go b/libnetwork/drivers/overlay/ov_endpoint.go index 96757abc4e..7dcc530119 100644 --- a/libnetwork/drivers/overlay/ov_endpoint.go +++ b/libnetwork/drivers/overlay/ov_endpoint.go @@ -1,22 +1,30 @@ package overlay import ( + "encoding/json" "fmt" "net" log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" + "github.com/docker/libnetwork/types" ) type endpointTable map[string]*endpoint +const overlayEndpointPrefix = "overlay/endpoint" + type endpoint struct { - id string - ifName string - mac net.HardwareAddr - addr *net.IPNet + id string + nid string + ifName string + mac net.HardwareAddr + addr *net.IPNet + dbExists bool + dbIndex uint64 } func (n *network) endpoint(eid string) *endpoint { @@ -60,6 +68,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, ep := &endpoint{ id: eid, + nid: n.id, addr: ifInfo.Address(), mac: ifInfo.MacAddress(), } @@ -80,6 +89,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, n.addEndpoint(ep) + if err := d.writeEndpointToStore(ep); err != nil { + return fmt.Errorf("failed to update overlay endpoint %s to local store: %v", ep.id[0:7], err) + } + return nil } @@ -102,6 +115,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { n.deleteEndpoint(eid) + if err := d.deleteEndpointFromStore(ep); err != nil { + log.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err) + } + if ep.ifName == "" { return nil } @@ -121,3 +138,122 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) { return make(map[string]interface{}, 0), nil } + +func (d *driver) deleteEndpointFromStore(e *endpoint) error { + if d.localStore == nil { + return fmt.Errorf("overlay local store not initialized, ep not deleted") + } + + if err := d.localStore.DeleteObjectAtomic(e); err != nil { + return err + } + + return nil +} + +func (d *driver) writeEndpointToStore(e *endpoint) error { + if d.localStore == nil { + return fmt.Errorf("overlay local store not initialized, ep not added") + } + + if err := d.localStore.PutObjectAtomic(e); err != nil { + return err + } + return nil +} + +func (ep *endpoint) DataScope() string { + return datastore.LocalScope +} + +func (ep *endpoint) New() datastore.KVObject { + return &endpoint{} +} + +func (ep *endpoint) CopyTo(o datastore.KVObject) error { + dstep := o.(*endpoint) + *dstep = *ep + return nil +} + +func (ep *endpoint) Key() []string { + return []string{overlayEndpointPrefix, ep.id} +} + +func (ep *endpoint) KeyPrefix() []string { + return []string{overlayEndpointPrefix} +} + +func (ep *endpoint) Index() uint64 { + return ep.dbIndex +} + +func (ep *endpoint) SetIndex(index uint64) { + ep.dbIndex = index + ep.dbExists = true +} + +func (ep *endpoint) Exists() bool { + return ep.dbExists +} + +func (ep *endpoint) Skip() bool { + return false +} + +func (ep *endpoint) Value() []byte { + b, err := json.Marshal(ep) + if err != nil { + return nil + } + return b +} + +func (ep *endpoint) SetValue(value []byte) error { + return json.Unmarshal(value, ep) +} + +func (ep *endpoint) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + + epMap["id"] = ep.id + epMap["nid"] = ep.nid + if ep.ifName != "" { + epMap["ifName"] = ep.ifName + } + if ep.addr != nil { + epMap["addr"] = ep.addr.String() + } + if len(ep.mac) != 0 { + epMap["mac"] = ep.mac.String() + } + + return json.Marshal(epMap) +} + +func (ep *endpoint) UnmarshalJSON(value []byte) error { + var ( + err error + epMap map[string]interface{} + ) + + json.Unmarshal(value, &epMap) + + ep.id = epMap["id"].(string) + ep.nid = epMap["nid"].(string) + if v, ok := epMap["mac"]; ok { + if ep.mac, err = net.ParseMAC(v.(string)); err != nil { + return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string)) + } + } + if v, ok := epMap["addr"]; ok { + if ep.addr, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err) + } + } + if v, ok := epMap["ifName"]; ok { + ep.ifName = v.(string) + } + + return nil +} diff --git a/libnetwork/drivers/overlay/ov_network.go b/libnetwork/drivers/overlay/ov_network.go index 05ed34a8fa..7edb5077c5 100644 --- a/libnetwork/drivers/overlay/ov_network.go +++ b/libnetwork/drivers/overlay/ov_network.go @@ -195,21 +195,21 @@ func (n *network) incEndpointCount() { n.joinCnt++ } -func (n *network) joinSandbox() error { +func (n *network) joinSandbox(restore bool) error { // If there is a race between two go routines here only one will win // the other will wait. n.once.Do(func() { // save the error status of initSandbox in n.initErr so that // all the racing go routines are able to know the status. - n.initErr = n.initSandbox() + n.initErr = n.initSandbox(restore) }) return n.initErr } -func (n *network) joinSubnetSandbox(s *subnet) error { +func (n *network) joinSubnetSandbox(s *subnet, restore bool) error { s.once.Do(func() { - s.initErr = n.initSubnetSandbox(s) + s.initErr = n.initSubnetSandbox(s, restore) }) return s.initErr } @@ -386,9 +386,33 @@ func isOverlap(nw *net.IPNet) bool { return false } -func (n *network) initSubnetSandbox(s *subnet) error { - brName := n.generateBridgeName(s) - vxlanName := n.generateVxlanName(s) +func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error { + sbox := n.sandbox() + + // restore overlay osl sandbox + Ifaces := make(map[string][]osl.IfaceOption) + brIfaceOption := make([]osl.IfaceOption, 2) + brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP)) + brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true)) + Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption + + err := sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + return err + } + + Ifaces = make(map[string][]osl.IfaceOption) + vxlanIfaceOption := make([]osl.IfaceOption, 1) + vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName)) + Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption + err = sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + return err + } + return nil +} + +func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error { if hostMode { // Try to delete stale bridge interface if it exists @@ -451,6 +475,19 @@ func (n *network) initSubnetSandbox(s *subnet) error { } } + return nil +} + +func (n *network) initSubnetSandbox(s *subnet, restore bool) error { + brName := n.generateBridgeName(s) + vxlanName := n.generateVxlanName(s) + + if restore { + n.restoreSubnetSandbox(s, brName, vxlanName) + } else { + n.setupSubnetSandbox(s, brName, vxlanName) + } + n.Lock() s.vxlanName = vxlanName s.brName = brName @@ -494,32 +531,45 @@ func (n *network) cleanupStaleSandboxes() { }) } -func (n *network) initSandbox() error { +func (n *network) initSandbox(restore bool) error { n.Lock() n.initEpoch++ n.Unlock() networkOnce.Do(networkOnceInit) - if hostMode { - if err := addNetworkChain(n.id[:12]); err != nil { - return err + if !restore { + if hostMode { + if err := addNetworkChain(n.id[:12]); err != nil { + return err + } } + + // If there are any stale sandboxes related to this network + // from previous daemon life clean it up here + n.cleanupStaleSandboxes() } - // If there are any stale sandboxes related to this network - // from previous daemon life clean it up here - n.cleanupStaleSandboxes() + // In the restore case network sandbox already exist; but we don't know + // what epoch number it was created with. It has to be retrieved by + // searching the net namespaces. + key := "" + if restore { + key = osl.GenerateKey("-" + n.id) + } else { + key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id) + } - sbox, err := osl.NewSandbox( - osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode) + sbox, err := osl.NewSandbox(key, !hostMode, restore) if err != nil { - return fmt.Errorf("could not create network sandbox: %v", err) + return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err) } n.setSandbox(sbox) - n.driver.peerDbUpdateSandbox(n.id) + if !restore { + n.driver.peerDbUpdateSandbox(n.id) + } var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { diff --git a/libnetwork/drivers/overlay/overlay.go b/libnetwork/drivers/overlay/overlay.go index d9c5ab7961..7c5d6d548b 100644 --- a/libnetwork/drivers/overlay/overlay.go +++ b/libnetwork/drivers/overlay/overlay.go @@ -13,6 +13,7 @@ import ( "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/idm" "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" "github.com/hashicorp/serf/serf" ) @@ -41,6 +42,7 @@ type driver struct { serfInstance *serf.Serf networks networkTable store datastore.DataStore + localStore datastore.DataStore vxlanIdm *idm.Idm once sync.Once joinOnce sync.Once @@ -74,9 +76,75 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { } } + if data, ok := config[netlabel.LocalKVClient]; ok { + var err error + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore configuration: %v", data) + } + d.localStore, err = datastore.NewDataStoreFromConfig(dsc) + if err != nil { + return types.InternalErrorf("failed to initialize local data store: %v", err) + } + } + + d.restoreEndpoints() + return dc.RegisterDriver(networkType, d, c) } +// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox +func (d *driver) restoreEndpoints() error { + if d.localStore == nil { + logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing") + return nil + } + kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to read overlay endpoint from store: %v", err) + } + + if err == datastore.ErrKeyNotFound { + return nil + } + for _, kvo := range kvol { + ep := kvo.(*endpoint) + n := d.network(ep.nid) + if n == nil { + logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid, ep.id) + continue + } + n.addEndpoint(ep) + + s := n.getSubnetforIP(ep.addr) + if s == nil { + return fmt.Errorf("could not find subnet for endpoint %s", ep.id) + } + + if err := n.joinSandbox(true); err != nil { + return fmt.Errorf("restore network sandbox failed: %v", err) + } + + if err := n.joinSubnetSandbox(s, true); err != nil { + return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err) + } + + Ifaces := make(map[string][]osl.IfaceOption) + vethIfaceOption := make([]osl.IfaceOption, 1) + vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName)) + Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption + + err := n.sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + return fmt.Errorf("failed to restore overlay sandbox: %v", err) + } + + n.incEndpointCount() + d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true) + } + return nil +} + // Fini cleans up the driver resources func Fini(drv driverapi.Driver) { d := drv.(*driver) diff --git a/libnetwork/drivers/overlay/peerdb.go b/libnetwork/drivers/overlay/peerdb.go index 2c1112fc1d..5c1afbf782 100644 --- a/libnetwork/drivers/overlay/peerdb.go +++ b/libnetwork/drivers/overlay/peerdb.go @@ -271,7 +271,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err) } - if err := n.joinSubnetSandbox(s); err != nil { + if err := n.joinSubnetSandbox(s, false); err != nil { return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err) } diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index f84e8cb79c..043c3f1643 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) { epMap["name"] = ep.name epMap["id"] = ep.id epMap["ep_iface"] = ep.iface + epMap["joinInfo"] = ep.joinInfo epMap["exposed_ports"] = ep.exposedPorts if ep.generic != nil { epMap["generic"] = ep.generic @@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) { ib, _ := json.Marshal(epMap["ep_iface"]) json.Unmarshal(ib, &ep.iface) + jb, _ := json.Marshal(epMap["joinInfo"]) + json.Unmarshal(jb, &ep.joinInfo) + tb, _ := json.Marshal(epMap["exposed_ports"]) var tPorts []types.TransportPort json.Unmarshal(tb, &tPorts) @@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error { ep.iface.CopyTo(dstEp.iface) } + if ep.joinInfo != nil { + dstEp.joinInfo = &endpointJoinInfo{} + ep.joinInfo.CopyTo(dstEp.joinInfo) + } + dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts)) copy(dstEp.exposedPorts, ep.exposedPorts) @@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() { } func (c *controller) cleanupLocalEndpoints() { + // Get used endpoints + eps := make(map[string]interface{}) + for _, sb := range c.sandboxes { + for _, ep := range sb.endpoints { + eps[ep.id] = true + } + } nl, err := c.getNetworksForScope(datastore.LocalScope) if err != nil { log.Warnf("Could not get list of networks during endpoint cleanup: %v", err) @@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() { } for _, ep := range epl { + if _, ok := eps[ep.id]; ok { + continue + } log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id) if err := ep.Delete(true); err != nil { log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err) diff --git a/libnetwork/endpoint_info.go b/libnetwork/endpoint_info.go index cf295a4229..60f15518e7 100644 --- a/libnetwork/endpoint_info.go +++ b/libnetwork/endpoint_info.go @@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() { ep.joinInfo.disableGatewayService = true } + +func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + if epj.gw != nil { + epMap["gw"] = epj.gw.String() + } + if epj.gw6 != nil { + epMap["gw6"] = epj.gw6.String() + } + epMap["disableGatewayService"] = epj.disableGatewayService + epMap["StaticRoutes"] = epj.StaticRoutes + return json.Marshal(epMap) +} + +func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + if err = json.Unmarshal(b, &epMap); err != nil { + return err + } + if v, ok := epMap["gw"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + if v, ok := epMap["gw6"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + epj.disableGatewayService = epMap["disableGatewayService"].(bool) + + var tStaticRoute []types.StaticRoute + if v, ok := epMap["StaticRoutes"]; ok { + tb, _ := json.Marshal(v) + var tStaticRoute []types.StaticRoute + json.Unmarshal(tb, &tStaticRoute) + } + var StaticRoutes []*types.StaticRoute + for _, r := range tStaticRoute { + StaticRoutes = append(StaticRoutes, &r) + } + epj.StaticRoutes = StaticRoutes + + return nil +} + +func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error { + dstEpj.disableGatewayService = epj.disableGatewayService + dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes)) + copy(dstEpj.StaticRoutes, epj.StaticRoutes) + dstEpj.gw = types.GetIPCopy(epj.gw) + dstEpj.gw = types.GetIPCopy(epj.gw6) + return nil +} diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index b27bde9fe8..134391fbb6 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) { } // Create a new OS sandbox using the osl API before using it in SetKey - if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil { + if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil { t.Fatalf("Failed to create new osl sandbox") } else { defer func() { diff --git a/libnetwork/osl/namespace_linux.go b/libnetwork/osl/namespace_linux.go index c804caf783..b9a0201e16 100644 --- a/libnetwork/osl/namespace_linux.go +++ b/libnetwork/osl/namespace_linux.go @@ -2,10 +2,13 @@ package osl import ( "fmt" + "io/ioutil" "net" "os" "os/exec" "runtime" + "strconv" + "strings" "sync" "syscall" "time" @@ -133,6 +136,39 @@ func GC() { // container id. func GenerateKey(containerID string) string { maxLen := 12 + // Read sandbox key from host for overlay + if strings.HasPrefix(containerID, "-") { + var ( + index int + indexStr string + tmpkey string + ) + dir, err := ioutil.ReadDir(prefix) + if err != nil { + return "" + } + + for _, v := range dir { + id := v.Name() + if strings.HasSuffix(id, containerID[:maxLen-1]) { + indexStr = strings.TrimSuffix(id, containerID[:maxLen-1]) + tmpindex, err := strconv.Atoi(indexStr) + if err != nil { + return "" + } + if tmpindex > index { + index = tmpindex + tmpkey = id + } + + } + } + containerID = tmpkey + if containerID == "" { + return "" + } + } + if len(containerID) < maxLen { maxLen = len(containerID) } @@ -142,10 +178,14 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { - err := createNetworkNamespace(key, osCreate) - if err != nil { - return nil, err +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { + if !isRestore { + err := createNetworkNamespace(key, osCreate) + if err != nil { + return nil, err + } + } else { + once.Do(createBasePath) } n := &networkNamespace{path: key, isDefault: !osCreate} @@ -347,3 +387,105 @@ func (n *networkNamespace) Destroy() error { addToGarbagePaths(n.path) return nil } + +// Restore restore the network namespace +func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error { + // restore interfaces + for name, opts := range ifsopt { + if !strings.Contains(name, "+") { + return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name) + } + seps := strings.Split(name, "+") + srcName := seps[0] + dstPrefix := seps[1] + i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} + i.processInterfaceOptions(opts...) + if i.master != "" { + i.dstMaster = n.findDst(i.master, true) + if i.dstMaster == "" { + return fmt.Errorf("could not find an appropriate master %q for %q", + i.master, i.srcName) + } + } + if n.isDefault { + i.dstName = i.srcName + } else { + links, err := n.nlHandle.LinkList() + if err != nil { + return fmt.Errorf("failed to retrieve list of links in network namespace %q during restore", n.path) + } + // due to the docker network connect/disconnect, so the dstName should + // restore from the namespace + for _, link := range links { + addrs, err := n.nlHandle.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return err + } + ifaceName := link.Attrs().Name + if strings.HasPrefix(ifaceName, "vxlan") { + if i.dstName == "vxlan" { + i.dstName = ifaceName + break + } + } + // find the interface name by ip + if i.address != nil { + for _, addr := range addrs { + if addr.IPNet.String() == i.address.String() { + i.dstName = ifaceName + break + } + continue + } + if i.dstName == ifaceName { + break + } + } + // This is to find the interface name of the pair in overlay sandbox + if strings.HasPrefix(ifaceName, "veth") { + if i.master != "" && i.dstName == "veth" { + i.dstName = ifaceName + } + } + } + + var index int + indexStr := strings.TrimPrefix(i.dstName, dstPrefix) + if indexStr != "" { + index, err = strconv.Atoi(indexStr) + if err != nil { + return err + } + } + index++ + n.Lock() + if index > n.nextIfIndex { + n.nextIfIndex = index + } + n.iFaces = append(n.iFaces, i) + n.Unlock() + } + } + + // restore routes + for _, r := range routes { + n.Lock() + n.staticRoutes = append(n.staticRoutes, r) + n.Unlock() + } + + // restore gateway + if len(gw) > 0 { + n.Lock() + n.gw = gw + n.Unlock() + } + + if len(gw6) > 0 { + n.Lock() + n.gwv6 = gw6 + n.Unlock() + } + + return nil +} diff --git a/libnetwork/osl/namespace_windows.go b/libnetwork/osl/namespace_windows.go index 912d4a2e9f..a735623a44 100644 --- a/libnetwork/osl/namespace_windows.go +++ b/libnetwork/osl/namespace_windows.go @@ -15,7 +15,7 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, nil } diff --git a/libnetwork/osl/sandbox.go b/libnetwork/osl/sandbox.go index 5264b35073..18113e3b3c 100644 --- a/libnetwork/osl/sandbox.go +++ b/libnetwork/osl/sandbox.go @@ -58,6 +58,9 @@ type Sandbox interface { // Destroy the sandbox Destroy() error + + // restore sandbox + Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error } // NeighborOptionSetter interface defines the option setter methods for interface options diff --git a/libnetwork/osl/sandbox_freebsd.go b/libnetwork/osl/sandbox_freebsd.go index 7c6dcacead..0222afe3d8 100644 --- a/libnetwork/osl/sandbox_freebsd.go +++ b/libnetwork/osl/sandbox_freebsd.go @@ -15,7 +15,7 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, nil } diff --git a/libnetwork/osl/sandbox_test.go b/libnetwork/osl/sandbox_test.go index 019ec0e09e..d3bd1aceb8 100644 --- a/libnetwork/osl/sandbox_test.go +++ b/libnetwork/osl/sandbox_test.go @@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - _, err = NewSandbox(key, true) + _, err = NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) { // Create another sandbox with the same key to see if we handle it // gracefully. - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } diff --git a/libnetwork/osl/sandbox_unsupported.go b/libnetwork/osl/sandbox_unsupported.go index 3bc6c38500..51a656c806 100644 --- a/libnetwork/osl/sandbox_unsupported.go +++ b/libnetwork/osl/sandbox_unsupported.go @@ -11,7 +11,7 @@ var ( // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, ErrNotImplemented } diff --git a/libnetwork/resolver_unix.go b/libnetwork/resolver_unix.go index 2b3734fbac..cec2c7d493 100644 --- a/libnetwork/resolver_unix.go +++ b/libnetwork/resolver_unix.go @@ -19,6 +19,13 @@ func init() { reexec.Register("setup-resolver", reexecSetupResolver) } +const ( + // outputChain used for docker embed dns + outputChain = "DOCKER_OUTPUT" + //postroutingchain used for docker embed dns + postroutingchain = "DOCKER_POSTROUTING" +) + func reexecSetupResolver() { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -31,10 +38,10 @@ func reexecSetupResolver() { _, ipPort, _ := net.SplitHostPort(os.Args[2]) _, tcpPort, _ := net.SplitHostPort(os.Args[3]) rules := [][]string{ - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, } f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0) @@ -50,6 +57,23 @@ func reexecSetupResolver() { os.Exit(3) } + // insert outputChain and postroutingchain + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain) + } + + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + } + for _, rule := range rules { if iptables.RawCombinedOutputNative(rule...) != nil { log.Errorf("setting up rule failed, %v", rule) diff --git a/libnetwork/sandbox.go b/libnetwork/sandbox.go index 05f44809be..dce169bd04 100644 --- a/libnetwork/sandbox.go +++ b/libnetwork/sandbox.go @@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() { osSbox.Destroy() } +func (sb *sandbox) restoreOslSandbox() error { + var routes []*types.StaticRoute + + // restore osl sandbox + Ifaces := make(map[string][]osl.IfaceOption) + for _, ep := range sb.endpoints { + var ifaceOptions []osl.IfaceOption + ep.Lock() + joinInfo := ep.joinInfo + i := ep.iface + ep.Unlock() + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) + if i.addrv6 != nil && i.addrv6.IP.To16() != nil { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) + } + if i.mac != nil { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) + } + if len(i.llAddrs) != 0 { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) + } + Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions + if joinInfo != nil { + for _, r := range joinInfo.StaticRoutes { + routes = append(routes, r) + } + } + if ep.needResolver() { + sb.startResolver() + } + } + + gwep := sb.getGatewayEndpoint() + if gwep == nil { + return nil + } + + // restore osl sandbox + err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6) + if err != nil { + return err + } + + return nil +} + func (sb *sandbox) populateNetworkResources(ep *endpoint) error { sb.Lock() if sb.osSbox == nil { diff --git a/libnetwork/sandbox_dns_unix.go b/libnetwork/sandbox_dns_unix.go index 5a3edba498..3f531beb99 100644 --- a/libnetwork/sandbox_dns_unix.go +++ b/libnetwork/sandbox_dns_unix.go @@ -139,6 +139,16 @@ func (sb *sandbox) updateParentHosts() error { return nil } +func (sb *sandbox) restorePath() { + if sb.config.resolvConfPath == "" { + sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf" + } + sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash" + if sb.config.hostsPath == "" { + sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts" + } +} + func (sb *sandbox) setupDNS() error { var newRC *resolvconf.File diff --git a/libnetwork/sandbox_dns_windows.go b/libnetwork/sandbox_dns_windows.go index ef90ddaeef..f2f58d5b98 100644 --- a/libnetwork/sandbox_dns_windows.go +++ b/libnetwork/sandbox_dns_windows.go @@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error { return nil } +func (sb *sandbox) restorePath() { +} + func (sb *sandbox) updateHostsFile(ifaceIP string) error { return nil } diff --git a/libnetwork/sandbox_store.go b/libnetwork/sandbox_store.go index ae5ddc1566..5aa4839406 100644 --- a/libnetwork/sandbox_store.go +++ b/libnetwork/sandbox_store.go @@ -20,12 +20,13 @@ type epState struct { } type sbState struct { - ID string - Cid string - c *controller - dbIndex uint64 - dbExists bool - Eps []epState + ID string + Cid string + c *controller + dbIndex uint64 + dbExists bool + Eps []epState + EpPriority map[string]int } func (sbs *sbState) Key() []string { @@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error { dstSbs.Cid = sbs.Cid dstSbs.dbIndex = sbs.dbIndex dstSbs.dbExists = sbs.dbExists + dstSbs.EpPriority = sbs.EpPriority for _, eps := range sbs.Eps { dstSbs.Eps = append(dstSbs.Eps, eps) @@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string { func (sb *sandbox) storeUpdate() error { sbs := &sbState{ - c: sb.controller, - ID: sb.id, - Cid: sb.containerID, + c: sb.controller, + ID: sb.id, + Cid: sb.containerID, + EpPriority: sb.epPriority, } retry: @@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error { return sb.controller.deleteFromStore(sbs) } -func (c *controller) sandboxCleanup() { +func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") @@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() { controller: sbs.c, containerID: sbs.Cid, endpoints: epHeap{}, - epPriority: map[string]int{}, dbIndex: sbs.dbIndex, isStub: true, dbExists: true, } - sb.osSbox, err = osl.NewSandbox(sb.Key(), true) + msg := " for cleanup" + create := true + isRestore := false + if val, ok := activeSandboxes[sb.ID()]; ok { + msg = "" + sb.isStub = false + isRestore = true + opts := val.([]SandboxOption) + sb.processOptions(opts...) + sb.restorePath() + create = !sb.config.useDefaultSandBox + heap.Init(&sb.endpoints) + } + sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore) if err != nil { - logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err) + logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err) continue } @@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() { ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } } - heap.Push(&sb.endpoints, ep) } - logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) - if err := sb.delete(true); err != nil { - logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + if _, ok := activeSandboxes[sb.ID()]; !ok { + logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) + if err := sb.delete(true); err != nil { + logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + } + continue + } + + // reconstruct osl sandbox field + if !sb.config.useDefaultSandBox { + if err := sb.restoreOslSandbox(); err != nil { + logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID()) + continue + } + } else { + c.sboxOnce.Do(func() { + c.defOsSbox = sb.osSbox + }) + } + + for _, ep := range sb.endpoints { + // Watch for service records + if !c.isAgent() { + c.watchSvcRecord(ep) + } } } }