Fix panic in drivers/overlay/encryption.go

Issue - "index out of range" panic in drivers/overlay/encryption.go:539
due to a mismatch in indices between curKeys and spis due to
case where updateKeys might bail out due to an error and
not update the spis

Fix - Reconfigure keys when there is a key update failure

Signed-off-by: Arko Dasgupta <arko.dasgupta@docker.com>

libnetwork/agent.go

@@ -184,6 +184,16 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {
 		err := driver.DiscoverNew(discoverapi.EncryptionKeysUpdate, drvEnc)
 		if err != nil {
 			logrus.Warnf("Failed to update datapath keys in driver %s: %v", name, err)
+			// Attempt to reconfigure keys in case of a update failure
+			// which can arise due to a mismatch of keys
+			// if worker nodes get temporarily disconnected
+			logrus.Warnf("Reconfiguring datapath keys for  %s", name)
+			drvCfgEnc := discoverapi.DriverEncryptionConfig{}
+			drvCfgEnc.Keys, drvCfgEnc.Tags = c.getKeys(subsysIPSec)
+			err = driver.DiscoverNew(discoverapi.EncryptionKeysConfig, drvCfgEnc)
+			if err != nil {
+				logrus.Warnf("Failed to reset datapath keys in driver %s: %v", name, err)
+			}
 		}
 		return false
 	})

libnetwork/drivers/overlay/overlay.go

@@ -378,7 +378,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{})
 			}
 		}
 		if err := d.updateKeys(newKey, priKey, delKey); err != nil {
-			logrus.Warn(err)
+			return err
 		}
 	default:
 	}