Update key handling logic to process keyring with 3 keys

Signed-off-by: Santhosh Manohar <santhosh@docker.com>

libnetwork/agent.go

@@ -3,12 +3,10 @@ package libnetwork
 //go:generate protoc -I.:Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. agent.proto
 
 import (
-	"encoding/hex"
 	"fmt"
 	"net"
 	"os"
 	"sort"
-	"strconv"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/go-events"
@@ -20,6 +18,12 @@ import (
 	"github.com/gogo/protobuf/proto"
 )
 
+const (
+	subsysGossip = "networking:gossip"
+	subsysIPSec  = "networking:ipsec"
+	keyringSize  = 3
+)
+
 // ByTime implements sort.Interface for []*types.EncryptionKey based on
 // the LamportTime field.
 type ByTime []*types.EncryptionKey
@@ -80,6 +84,82 @@ func resolveAddr(addrOrInterface string) (string, error) {
 func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {
 	drvEnc := discoverapi.DriverEncryptionUpdate{}
 
+	a := c.agent
+	// Find the deleted key. If the deleted key was the primary key,
+	// a new primary key should be set before removing if from keyring.
+	deleted := []byte{}
+	j := len(c.keys)
+	for i := 0; i < j; {
+		same := false
+		for _, key := range keys {
+			if same = key.LamportTime == c.keys[i].LamportTime; same {
+				break
+			}
+		}
+		if !same {
+			cKey := c.keys[i]
+			if cKey.Subsystem == subsysGossip {
+				deleted = cKey.Key
+			}
+
+			if cKey.Subsystem == subsysGossip /* subsysIPSec */ {
+				drvEnc.Prune = cKey.Key
+				drvEnc.PruneTag = cKey.LamportTime
+			}
+			c.keys[i], c.keys[j-1] = c.keys[j-1], c.keys[i]
+			c.keys[j-1] = nil
+			j--
+		}
+		i++
+	}
+	c.keys = c.keys[:j]
+
+	// Find the new key and add it to the key ring
+	for _, key := range keys {
+		same := false
+		for _, cKey := range c.keys {
+			if same = cKey.LamportTime == key.LamportTime; same {
+				break
+			}
+		}
+		if !same {
+			c.keys = append(c.keys, key)
+			if key.Subsystem == subsysGossip {
+				a.networkDB.SetKey(key.Key)
+			}
+
+			if key.Subsystem == subsysGossip /*subsysIPSec*/ {
+				drvEnc.Key = key.Key
+				drvEnc.Tag = key.LamportTime
+			}
+		}
+	}
+
+	key, tag := c.getPrimaryKeyTag(subsysGossip)
+	a.networkDB.SetPrimaryKey(key)
+
+	//key, tag = c.getPrimaryKeyTag(subsysIPSec)
+	drvEnc.Primary = key
+	drvEnc.PrimaryTag = tag
+
+	if len(deleted) > 0 {
+		a.networkDB.RemoveKey(deleted)
+	}
+
+	c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
+		err := driver.DiscoverNew(discoverapi.EncryptionKeysUpdate, drvEnc)
+		if err != nil {
+			logrus.Warnf("Failed to update datapath keys in driver %s: %v", name, err)
+		}
+		return false
+	})
+
+	return nil
+}
+
+func (c *controller) handleKeyChangeV1(keys []*types.EncryptionKey) error {
+	drvEnc := discoverapi.DriverEncryptionUpdate{}
+
 	// Find the new key and add it to the key ring
 	a := c.agent
 	for _, key := range keys {
@@ -91,12 +171,12 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {
 		}
 		if !same {
 			c.keys = append(c.keys, key)
-			if key.Subsystem == "networking:gossip" {
+			if key.Subsystem == subsysGossip {
 				a.networkDB.SetKey(key.Key)
 			}
-			if key.Subsystem == "networking:gossip" /*"networking:ipsec"*/ {
+			if key.Subsystem == subsysGossip /*subsysIPSec*/ {
-				drvEnc.Key = hex.EncodeToString(key.Key)
+				drvEnc.Key = key.Key
-				drvEnc.Tag = strconv.FormatUint(key.LamportTime, 10)
+				drvEnc.Tag = key.LamportTime
 			}
 			break
 		}
@@ -112,12 +192,12 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {
 			}
 		}
 		if !same {
-			if cKey.Subsystem == "networking:gossip" {
+			if cKey.Subsystem == subsysGossip {
 				deleted = cKey.Key
 			}
-			if cKey.Subsystem == "networking:gossip" /*"networking:ipsec"*/ {
+			if cKey.Subsystem == subsysGossip /*subsysIPSec*/ {
-				drvEnc.Prune = hex.EncodeToString(cKey.Key)
+				drvEnc.Prune = cKey.Key
-				drvEnc.PruneTag = strconv.FormatUint(cKey.LamportTime, 10)
+				drvEnc.PruneTag = cKey.LamportTime
 			}
 			c.keys = append(c.keys[:i], c.keys[i+1:]...)
 			break
@@ -126,15 +206,15 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {
 
 	sort.Sort(ByTime(c.keys))
 	for _, key := range c.keys {
-		if key.Subsystem == "networking:gossip" {
+		if key.Subsystem == subsysGossip {
 			a.networkDB.SetPrimaryKey(key.Key)
 			break
 		}
 	}
 	for _, key := range c.keys {
-		if key.Subsystem == "networking:gossip" /*"networking:ipsec"*/ {
+		if key.Subsystem == subsysGossip /*subsysIPSec*/ {
-			drvEnc.Primary = hex.EncodeToString(key.Key)
+			drvEnc.Primary = key.Key
-			drvEnc.PrimaryTag = strconv.FormatUint(key.LamportTime, 10)
+			drvEnc.PrimaryTag = key.LamportTime
 			break
 		}
 	}
@@ -197,6 +277,41 @@ func (c *controller) agentSetup() error {
 	return nil
 }
 
+// For a given subsystem getKeys sorts the keys by lamport time and returns
+// slice of keys and lamport time which can used as a unique tag for the keys
+func (c *controller) getKeys(subsys string) ([][]byte, []uint64) {
+	sort.Sort(ByTime(c.keys))
+
+	keys := [][]byte{}
+	tags := []uint64{}
+	for _, key := range c.keys {
+		if key.Subsystem == subsys {
+			keys = append(keys, key.Key)
+			tags = append(tags, key.LamportTime)
+		}
+	}
+
+	if len(keys) < keyringSize {
+		return keys, tags
+	}
+	keys[0], keys[1] = keys[1], keys[0]
+	tags[0], tags[1] = tags[1], tags[0]
+	return keys, tags
+}
+
+// getPrimaryKeyTag returns the primary key for a given subsytem from the
+// list of sorted key and the associated tag
+func (c *controller) getPrimaryKeyTag(subsys string) ([]byte, uint64) {
+	sort.Sort(ByTime(c.keys))
+	keys := []*types.EncryptionKey{}
+	for _, key := range c.keys {
+		if key.Subsystem == subsys {
+			keys = append(keys, key)
+		}
+	}
+	return keys[1].Key, keys[1].LamportTime
+}
+
 func (c *controller) agentInit(bindAddrOrInterface string) error {
 	if !c.isAgent() {
 		return nil
@@ -204,19 +319,9 @@ func (c *controller) agentInit(bindAddrOrInterface string) error {
 
 	drvEnc := discoverapi.DriverEncryptionConfig{}
 
-	// sort the keys by lamport time
+	keys, tags := c.getKeys(subsysGossip) // getKeys(subsysIPSec)
-	sort.Sort(ByTime(c.keys))
+	drvEnc.Keys = keys
-
+	drvEnc.Tags = tags
-	gossipkey := [][]byte{}
-	for _, key := range c.keys {
-		if key.Subsystem == "networking:gossip" {
-			gossipkey = append(gossipkey, key.Key)
-		}
-		if key.Subsystem == "networking:gossip" /*"networking:ipsec"*/ {
-			drvEnc.Keys = append(drvEnc.Keys, hex.EncodeToString(key.Key))
-			drvEnc.Tags = append(drvEnc.Tags, strconv.FormatUint(key.LamportTime, 10))
-		}
-	}
 
 	bindAddr, err := resolveAddr(bindAddrOrInterface)
 	if err != nil {
@@ -227,7 +332,7 @@ func (c *controller) agentInit(bindAddrOrInterface string) error {
 	nDB, err := networkdb.New(&networkdb.Config{
 		BindAddr: bindAddr,
 		NodeName: hostname,
-		Keys:     gossipkey,
+		Keys:     keys,
 	})
 
 	if err != nil {
@@ -275,12 +380,10 @@ func (c *controller) agentDriverNotify(d driverapi.Driver) {
 	})
 
 	drvEnc := discoverapi.DriverEncryptionConfig{}
-	for _, key := range c.keys {
+	keys, tags := c.getKeys(subsysGossip) // getKeys(subsysIPSec)
-		if key.Subsystem == "networking:gossip" /*"networking:ipsec"*/ {
+	drvEnc.Keys = keys
-			drvEnc.Keys = append(drvEnc.Keys, hex.EncodeToString(key.Key))
+	drvEnc.Tags = tags
-			drvEnc.Tags = append(drvEnc.Tags, strconv.FormatUint(key.LamportTime, 10))
+
-		}
-	}
 	c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
 		err := driver.DiscoverNew(discoverapi.EncryptionKeysConfig, drvEnc)
 		if err != nil {

libnetwork/controller.go

@@ -264,6 +264,9 @@ func (c *controller) SetKeys(keys []*types.EncryptionKey) error {
 		c.Unlock()
 		return nil
 	}
+	if len(keys) < keyringSize {
+		return c.handleKeyChangeV1(keys)
+	}
 	return c.handleKeyChange(keys)
 }
 

libnetwork/discoverapi/discoverapi.go

@@ -42,18 +42,18 @@ type DatastoreConfigData struct {
 // Key in first position is the primary key, the one to be used in tx.
 // Original key and tag types are []byte and uint64
 type DriverEncryptionConfig struct {
-	Keys []string
+	Keys [][]byte
-	Tags []string
+	Tags []uint64
 }
 
 // DriverEncryptionUpdate carries an update to the encryption key(s) as:
 // a new key and/or set a primary key and/or a removal of an existing key.
 // Original key and tag types are []byte and uint64
 type DriverEncryptionUpdate struct {
-	Key        string
+	Key        []byte
-	Tag        string
+	Tag        uint64
-	Primary    string
+	Primary    []byte
-	PrimaryTag string
+	PrimaryTag uint64
-	Prune      string
+	Prune      []byte
-	PruneTag   string
+	PruneTag   uint64
 }

libnetwork/drivers/overlay/encryption.go

@@ -33,8 +33,11 @@ type key struct {
 }
 
 func (k *key) String() string {
+	if k != nil {
 		return fmt.Sprintf("(key: %s, tag: 0x%x)", hex.EncodeToString(k.value)[0:5], k.tag)
 	}
+	return ""
+}
 
 type spi struct {
 	forward int
@@ -557,23 +560,3 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 
 	return spis
 }
-
-func parseEncryptionKey(value, tag string) (*key, error) {
-	var (
-		k   *key
-		err error
-	)
-	if value == "" {
-		return nil, nil
-	}
-	k = &key{}
-	if k.value, err = hex.DecodeString(value); err != nil {
-		return nil, types.BadRequestErrorf("failed to decode key (%s): %v", value, err)
-	}
-	t, err := strconv.ParseUint(tag, 10, 64)
-	if err != nil {
-		return nil, types.BadRequestErrorf("failed to decode tag (%s): %v", tag, err)
-	}
-	k.tag = uint32(t)
-	return k, nil
-}

libnetwork/drivers/overlay/overlay.go

@@ -306,9 +306,9 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{})
 		}
 		keys := make([]*key, 0, len(encrData.Keys))
 		for i := 0; i < len(encrData.Keys); i++ {
-			k, err := parseEncryptionKey(encrData.Keys[i], encrData.Tags[i])
+			k := &key{
-			if err != nil {
+				value: encrData.Keys[i],
-				return err
+				tag:   uint32(encrData.Tags[i]),
 			}
 			keys = append(keys, k)
 		}
@@ -319,17 +319,23 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{})
 		if !ok {
 			return fmt.Errorf("invalid encryption key notification data")
 		}
-		newKey, err = parseEncryptionKey(encrData.Key, encrData.Tag)
+		if encrData.Key != nil {
-		if err != nil {
+			newKey = &key{
-			return err
+				value: encrData.Key,
+				tag:   uint32(encrData.Tag),
 			}
-		priKey, err = parseEncryptionKey(encrData.Primary, encrData.PrimaryTag)
-		if err != nil {
-			return err
 		}
-		delKey, err = parseEncryptionKey(encrData.Prune, encrData.PruneTag)
+		if encrData.Primary != nil {
-		if err != nil {
+			priKey = &key{
-			return err
+				value: encrData.Primary,
+				tag:   uint32(encrData.PrimaryTag),
+			}
+		}
+		if encrData.Prune != nil {
+			delKey = &key{
+				value: encrData.Prune,
+				tag:   uint32(encrData.PruneTag),
+			}
 		}
 		d.updateKeys(newKey, priKey, delKey)
 	default: