Merge pull request #880 from mavenugo/ddd

Force delete sandbox during sandboxCleanup
This commit is contained in:
aboch 2016-01-19 13:57:53 -05:00
commit 3e6a42bcdb
8 changed files with 71 additions and 37 deletions

View File

@ -387,7 +387,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
// Make sure we have a driver available for this network type
// before we allocate anything.
if _, err := network.driver(); err != nil {
if _, err := network.driver(true); err != nil {
return nil, err
}
@ -432,7 +432,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
}
func (c *controller) addNetwork(n *network) error {
d, err := n.driver()
d, err := n.driver(true)
if err != nil {
return err
}

View File

@ -84,7 +84,7 @@ func (sb *sandbox) clearDefaultGW() error {
return nil
}
if err := ep.sbLeave(sb); err != nil {
if err := ep.sbLeave(sb, false); err != nil {
return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
}
if err := ep.Delete(false); err != nil {

View File

@ -406,7 +406,7 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
ep.processOptions(options...)
driver, err := network.driver()
driver, err := network.driver(true)
if err != nil {
return fmt.Errorf("failed to join endpoint: %v", err)
}
@ -533,10 +533,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
sb.joinLeaveStart()
defer sb.joinLeaveEnd()
return ep.sbLeave(sbox, options...)
return ep.sbLeave(sbox, false, options...)
}
func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) error {
sb, ok := sbox.(*sandbox)
if !ok {
return types.BadRequestErrorf("not a valid Sandbox interface")
@ -565,7 +565,7 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
ep.processOptions(options...)
d, err := n.driver()
d, err := n.driver(!force)
if err != nil {
return fmt.Errorf("failed to leave endpoint: %v", err)
}
@ -575,9 +575,11 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
ep.network = n
ep.Unlock()
if err := d.Leave(n.id, ep.id); err != nil {
if _, ok := err.(types.MaskableError); !ok {
log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
if d != nil {
if err := d.Leave(n.id, ep.id); err != nil {
if _, ok := err.(types.MaskableError); !ok {
log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
}
}
}
@ -649,7 +651,7 @@ func (ep *endpoint) Delete(force bool) error {
}
if sb != nil {
if e := ep.sbLeave(sb); e != nil {
if e := ep.sbLeave(sb, force); e != nil {
log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
}
}
@ -681,7 +683,7 @@ func (ep *endpoint) Delete(force bool) error {
// unwatch for service records
n.getController().unWatchSvcRecord(ep)
if err = ep.deleteEndpoint(); err != nil && !force {
if err = ep.deleteEndpoint(force); err != nil && !force {
return err
}
@ -690,18 +692,22 @@ func (ep *endpoint) Delete(force bool) error {
return nil
}
func (ep *endpoint) deleteEndpoint() error {
func (ep *endpoint) deleteEndpoint(force bool) error {
ep.Lock()
n := ep.network
name := ep.name
epid := ep.id
ep.Unlock()
driver, err := n.driver()
driver, err := n.driver(!force)
if err != nil {
return fmt.Errorf("failed to delete endpoint: %v", err)
}
if driver == nil {
return nil
}
if err := driver.DeleteEndpoint(n.id, epid); err != nil {
if _, ok := err.(types.ForbiddenError); ok {
return err
@ -956,7 +962,7 @@ func (c *controller) cleanupLocalEndpoints() {
}
for _, ep := range epl {
if err := ep.Delete(false); err != nil {
if err := ep.Delete(true); err != nil {
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
}
}

View File

@ -188,7 +188,7 @@ func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
}
driver, err := n.driver()
driver, err := n.driver(true)
if err != nil {
return nil, fmt.Errorf("failed to get driver info: %v", err)
}

View File

@ -149,6 +149,7 @@ type network struct {
name string
networkType string
id string
scope string
ipamType string
ipamOptions map[string]string
addrSpace string
@ -246,6 +247,7 @@ func (n *network) New() datastore.KVObject {
return &network{
ctrlr: n.ctrlr,
drvOnce: &sync.Once{},
scope: n.scope,
}
}
@ -295,6 +297,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
dstN.name = n.name
dstN.id = n.id
dstN.networkType = n.networkType
dstN.scope = n.scope
dstN.ipamType = n.ipamType
dstN.enableIPv6 = n.enableIPv6
dstN.persist = n.persist
@ -337,7 +340,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
}
func (n *network) DataScope() string {
return n.driverScope()
return n.Scope()
}
func (n *network) getEpCnt() *endpointCnt {
@ -353,6 +356,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
netMap["name"] = n.name
netMap["id"] = n.id
netMap["networkType"] = n.networkType
netMap["scope"] = n.scope
netMap["ipamType"] = n.ipamType
netMap["addrSpace"] = n.addrSpace
netMap["enableIPv6"] = n.enableIPv6
@ -456,6 +460,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
if v, ok := netMap["internal"]; ok {
n.internal = v.(bool)
}
if s, ok := netMap["scope"]; ok {
n.scope = s.(string)
}
return nil
}
@ -566,7 +573,7 @@ func (n *network) driverScope() string {
return dd.capability.DataScope
}
func (n *network) driver() (driverapi.Driver, error) {
func (n *network) driver(load bool) (driverapi.Driver, error) {
c := n.getController()
c.Lock()
@ -574,14 +581,20 @@ func (n *network) driver() (driverapi.Driver, error) {
dd, ok := c.drivers[n.networkType]
c.Unlock()
if !ok {
if !ok && load {
var err error
dd, err = c.loadDriver(n.networkType)
if err != nil {
return nil, err
}
} else if !ok {
// dont fail if driver loading is not required
return nil, nil
}
n.Lock()
n.scope = dd.capability.DataScope
n.Unlock()
return dd.driver, nil
}
@ -631,7 +644,7 @@ func (n *network) Delete() error {
}
func (n *network) deleteNetwork() error {
d, err := n.driver()
d, err := n.driver(true)
if err != nil {
return fmt.Errorf("failed deleting network: %v", err)
}
@ -651,7 +664,7 @@ func (n *network) deleteNetwork() error {
}
func (n *network) addEndpoint(ep *endpoint) error {
d, err := n.driver()
d, err := n.driver(true)
if err != nil {
return fmt.Errorf("failed to add endpoint: %v", err)
}
@ -725,7 +738,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
}
defer func() {
if err != nil {
if e := ep.deleteEndpoint(); e != nil {
if e := ep.deleteEndpoint(false); e != nil {
log.Warnf("cleaning up endpoint failed %s : %v", name, e)
}
}
@ -1169,7 +1182,9 @@ func (n *network) DriverOptions() map[string]string {
}
func (n *network) Scope() string {
return n.driverScope()
n.Lock()
defer n.Unlock()
return n.scope
}
func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) {

View File

@ -160,6 +160,10 @@ func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
}
func (sb *sandbox) Delete() error {
return sb.delete(false)
}
func (sb *sandbox) delete(force bool) error {
sb.Lock()
if sb.inDelete {
sb.Unlock()
@ -194,11 +198,13 @@ func (sb *sandbox) Delete() error {
continue
}
if err := ep.Leave(sb); err != nil {
log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
if !force {
if err := ep.Leave(sb); err != nil {
log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
}
}
if err := ep.Delete(false); err != nil {
if err := ep.Delete(force); err != nil {
log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
}
}

View File

@ -226,7 +226,7 @@ func (c *controller) sandboxCleanup() {
heap.Push(&sb.endpoints, ep)
}
if err := sb.Delete(); err != nil {
if err := sb.delete(true); err != nil {
logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
}
}

View File

@ -75,6 +75,7 @@ func (c *controller) getNetworkFromStore(nid string) (*network, error) {
}
n.epCnt = ec
n.scope = store.Scope()
return n, nil
}
@ -107,6 +108,7 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
}
n.epCnt = ec
n.scope = scope
nl = append(nl, n)
}
@ -140,6 +142,7 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
}
n.epCnt = ec
n.scope = store.Scope()
nl = append(nl, n)
}
}
@ -148,17 +151,21 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
}
func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
store := n.ctrlr.getStore(n.Scope())
if store == nil {
return nil, fmt.Errorf("could not find endpoint %s: datastore not found for scope %s", eid, n.Scope())
var errors []string
for _, store := range n.ctrlr.getStores() {
ep := &endpoint{id: eid, network: n}
err := store.GetObject(datastore.Key(ep.Key()...), ep)
// Continue searching in the next store if the key is not found in this store
if err != nil {
if err != datastore.ErrKeyNotFound {
errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
}
continue
}
return ep, nil
}
ep := &endpoint{id: eid, network: n}
err := store.GetObject(datastore.Key(ep.Key()...), ep)
if err != nil {
return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
}
return ep, nil
return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
}
func (n *network) getEndpointsFromStore() ([]*endpoint, error) {