Merge pull request #880 from mavenugo/ddd

Force delete sandbox during sandboxCleanup
This commit is contained in:
aboch 2016-01-19 13:57:53 -05:00
commit 3e6a42bcdb
8 changed files with 71 additions and 37 deletions

View File

@ -387,7 +387,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
// Make sure we have a driver available for this network type // Make sure we have a driver available for this network type
// before we allocate anything. // before we allocate anything.
if _, err := network.driver(); err != nil { if _, err := network.driver(true); err != nil {
return nil, err return nil, err
} }
@ -432,7 +432,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
} }
func (c *controller) addNetwork(n *network) error { func (c *controller) addNetwork(n *network) error {
d, err := n.driver() d, err := n.driver(true)
if err != nil { if err != nil {
return err return err
} }

View File

@ -84,7 +84,7 @@ func (sb *sandbox) clearDefaultGW() error {
return nil return nil
} }
if err := ep.sbLeave(sb); err != nil { if err := ep.sbLeave(sb, false); err != nil {
return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err) return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
} }
if err := ep.Delete(false); err != nil { if err := ep.Delete(false); err != nil {

View File

@ -406,7 +406,7 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
ep.processOptions(options...) ep.processOptions(options...)
driver, err := network.driver() driver, err := network.driver(true)
if err != nil { if err != nil {
return fmt.Errorf("failed to join endpoint: %v", err) return fmt.Errorf("failed to join endpoint: %v", err)
} }
@ -533,10 +533,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
sb.joinLeaveStart() sb.joinLeaveStart()
defer sb.joinLeaveEnd() defer sb.joinLeaveEnd()
return ep.sbLeave(sbox, options...) return ep.sbLeave(sbox, false, options...)
} }
func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error { func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) error {
sb, ok := sbox.(*sandbox) sb, ok := sbox.(*sandbox)
if !ok { if !ok {
return types.BadRequestErrorf("not a valid Sandbox interface") return types.BadRequestErrorf("not a valid Sandbox interface")
@ -565,7 +565,7 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
ep.processOptions(options...) ep.processOptions(options...)
d, err := n.driver() d, err := n.driver(!force)
if err != nil { if err != nil {
return fmt.Errorf("failed to leave endpoint: %v", err) return fmt.Errorf("failed to leave endpoint: %v", err)
} }
@ -575,9 +575,11 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
ep.network = n ep.network = n
ep.Unlock() ep.Unlock()
if err := d.Leave(n.id, ep.id); err != nil { if d != nil {
if _, ok := err.(types.MaskableError); !ok { if err := d.Leave(n.id, ep.id); err != nil {
log.Warnf("driver error disconnecting container %s : %v", ep.name, err) if _, ok := err.(types.MaskableError); !ok {
log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
}
} }
} }
@ -649,7 +651,7 @@ func (ep *endpoint) Delete(force bool) error {
} }
if sb != nil { if sb != nil {
if e := ep.sbLeave(sb); e != nil { if e := ep.sbLeave(sb, force); e != nil {
log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e) log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
} }
} }
@ -681,7 +683,7 @@ func (ep *endpoint) Delete(force bool) error {
// unwatch for service records // unwatch for service records
n.getController().unWatchSvcRecord(ep) n.getController().unWatchSvcRecord(ep)
if err = ep.deleteEndpoint(); err != nil && !force { if err = ep.deleteEndpoint(force); err != nil && !force {
return err return err
} }
@ -690,18 +692,22 @@ func (ep *endpoint) Delete(force bool) error {
return nil return nil
} }
func (ep *endpoint) deleteEndpoint() error { func (ep *endpoint) deleteEndpoint(force bool) error {
ep.Lock() ep.Lock()
n := ep.network n := ep.network
name := ep.name name := ep.name
epid := ep.id epid := ep.id
ep.Unlock() ep.Unlock()
driver, err := n.driver() driver, err := n.driver(!force)
if err != nil { if err != nil {
return fmt.Errorf("failed to delete endpoint: %v", err) return fmt.Errorf("failed to delete endpoint: %v", err)
} }
if driver == nil {
return nil
}
if err := driver.DeleteEndpoint(n.id, epid); err != nil { if err := driver.DeleteEndpoint(n.id, epid); err != nil {
if _, ok := err.(types.ForbiddenError); ok { if _, ok := err.(types.ForbiddenError); ok {
return err return err
@ -956,7 +962,7 @@ func (c *controller) cleanupLocalEndpoints() {
} }
for _, ep := range epl { for _, ep := range epl {
if err := ep.Delete(false); err != nil { if err := ep.Delete(true); err != nil {
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err) log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
} }
} }

View File

@ -188,7 +188,7 @@ func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
return nil, fmt.Errorf("could not find network in store for driver info: %v", err) return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
} }
driver, err := n.driver() driver, err := n.driver(true)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get driver info: %v", err) return nil, fmt.Errorf("failed to get driver info: %v", err)
} }

View File

@ -149,6 +149,7 @@ type network struct {
name string name string
networkType string networkType string
id string id string
scope string
ipamType string ipamType string
ipamOptions map[string]string ipamOptions map[string]string
addrSpace string addrSpace string
@ -246,6 +247,7 @@ func (n *network) New() datastore.KVObject {
return &network{ return &network{
ctrlr: n.ctrlr, ctrlr: n.ctrlr,
drvOnce: &sync.Once{}, drvOnce: &sync.Once{},
scope: n.scope,
} }
} }
@ -295,6 +297,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
dstN.name = n.name dstN.name = n.name
dstN.id = n.id dstN.id = n.id
dstN.networkType = n.networkType dstN.networkType = n.networkType
dstN.scope = n.scope
dstN.ipamType = n.ipamType dstN.ipamType = n.ipamType
dstN.enableIPv6 = n.enableIPv6 dstN.enableIPv6 = n.enableIPv6
dstN.persist = n.persist dstN.persist = n.persist
@ -337,7 +340,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
} }
func (n *network) DataScope() string { func (n *network) DataScope() string {
return n.driverScope() return n.Scope()
} }
func (n *network) getEpCnt() *endpointCnt { func (n *network) getEpCnt() *endpointCnt {
@ -353,6 +356,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
netMap["name"] = n.name netMap["name"] = n.name
netMap["id"] = n.id netMap["id"] = n.id
netMap["networkType"] = n.networkType netMap["networkType"] = n.networkType
netMap["scope"] = n.scope
netMap["ipamType"] = n.ipamType netMap["ipamType"] = n.ipamType
netMap["addrSpace"] = n.addrSpace netMap["addrSpace"] = n.addrSpace
netMap["enableIPv6"] = n.enableIPv6 netMap["enableIPv6"] = n.enableIPv6
@ -456,6 +460,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
if v, ok := netMap["internal"]; ok { if v, ok := netMap["internal"]; ok {
n.internal = v.(bool) n.internal = v.(bool)
} }
if s, ok := netMap["scope"]; ok {
n.scope = s.(string)
}
return nil return nil
} }
@ -566,7 +573,7 @@ func (n *network) driverScope() string {
return dd.capability.DataScope return dd.capability.DataScope
} }
func (n *network) driver() (driverapi.Driver, error) { func (n *network) driver(load bool) (driverapi.Driver, error) {
c := n.getController() c := n.getController()
c.Lock() c.Lock()
@ -574,14 +581,20 @@ func (n *network) driver() (driverapi.Driver, error) {
dd, ok := c.drivers[n.networkType] dd, ok := c.drivers[n.networkType]
c.Unlock() c.Unlock()
if !ok { if !ok && load {
var err error var err error
dd, err = c.loadDriver(n.networkType) dd, err = c.loadDriver(n.networkType)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} else if !ok {
// dont fail if driver loading is not required
return nil, nil
} }
n.Lock()
n.scope = dd.capability.DataScope
n.Unlock()
return dd.driver, nil return dd.driver, nil
} }
@ -631,7 +644,7 @@ func (n *network) Delete() error {
} }
func (n *network) deleteNetwork() error { func (n *network) deleteNetwork() error {
d, err := n.driver() d, err := n.driver(true)
if err != nil { if err != nil {
return fmt.Errorf("failed deleting network: %v", err) return fmt.Errorf("failed deleting network: %v", err)
} }
@ -651,7 +664,7 @@ func (n *network) deleteNetwork() error {
} }
func (n *network) addEndpoint(ep *endpoint) error { func (n *network) addEndpoint(ep *endpoint) error {
d, err := n.driver() d, err := n.driver(true)
if err != nil { if err != nil {
return fmt.Errorf("failed to add endpoint: %v", err) return fmt.Errorf("failed to add endpoint: %v", err)
} }
@ -725,7 +738,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
} }
defer func() { defer func() {
if err != nil { if err != nil {
if e := ep.deleteEndpoint(); e != nil { if e := ep.deleteEndpoint(false); e != nil {
log.Warnf("cleaning up endpoint failed %s : %v", name, e) log.Warnf("cleaning up endpoint failed %s : %v", name, e)
} }
} }
@ -1169,7 +1182,9 @@ func (n *network) DriverOptions() map[string]string {
} }
func (n *network) Scope() string { func (n *network) Scope() string {
return n.driverScope() n.Lock()
defer n.Unlock()
return n.scope
} }
func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) { func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) {

View File

@ -160,6 +160,10 @@ func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
} }
func (sb *sandbox) Delete() error { func (sb *sandbox) Delete() error {
return sb.delete(false)
}
func (sb *sandbox) delete(force bool) error {
sb.Lock() sb.Lock()
if sb.inDelete { if sb.inDelete {
sb.Unlock() sb.Unlock()
@ -194,11 +198,13 @@ func (sb *sandbox) Delete() error {
continue continue
} }
if err := ep.Leave(sb); err != nil { if !force {
log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err) if err := ep.Leave(sb); err != nil {
log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
}
} }
if err := ep.Delete(false); err != nil { if err := ep.Delete(force); err != nil {
log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err) log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
} }
} }

View File

@ -226,7 +226,7 @@ func (c *controller) sandboxCleanup() {
heap.Push(&sb.endpoints, ep) heap.Push(&sb.endpoints, ep)
} }
if err := sb.Delete(); err != nil { if err := sb.delete(true); err != nil {
logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
} }
} }

View File

@ -75,6 +75,7 @@ func (c *controller) getNetworkFromStore(nid string) (*network, error) {
} }
n.epCnt = ec n.epCnt = ec
n.scope = store.Scope()
return n, nil return n, nil
} }
@ -107,6 +108,7 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
} }
n.epCnt = ec n.epCnt = ec
n.scope = scope
nl = append(nl, n) nl = append(nl, n)
} }
@ -140,6 +142,7 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
} }
n.epCnt = ec n.epCnt = ec
n.scope = store.Scope()
nl = append(nl, n) nl = append(nl, n)
} }
} }
@ -148,17 +151,21 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
} }
func (n *network) getEndpointFromStore(eid string) (*endpoint, error) { func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
store := n.ctrlr.getStore(n.Scope()) var errors []string
if store == nil { for _, store := range n.ctrlr.getStores() {
return nil, fmt.Errorf("could not find endpoint %s: datastore not found for scope %s", eid, n.Scope()) ep := &endpoint{id: eid, network: n}
err := store.GetObject(datastore.Key(ep.Key()...), ep)
// Continue searching in the next store if the key is not found in this store
if err != nil {
if err != datastore.ErrKeyNotFound {
errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
}
continue
}
return ep, nil
} }
return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
ep := &endpoint{id: eid, network: n}
err := store.GetObject(datastore.Key(ep.Key()...), ep)
if err != nil {
return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
}
return ep, nil
} }
func (n *network) getEndpointsFromStore() ([]*endpoint, error) { func (n *network) getEndpointsFromStore() ([]*endpoint, error) {