Merge pull request #1354 from aboch/pol

Few changes in encryption overlay

libnetwork/drivers/overlay/encryption.go

@@ -20,7 +20,7 @@ import (
 )
 
 const (
-	mark         = uint32(0xD0C4E3)
+	r            = 0xD0C4E3
 	timeout      = 30
 	pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8)
 )
@@ -31,6 +31,8 @@ const (
 	bidir
 )
 
+var spMark = netlink.XfrmMark{Value: uint32(r), Mask: 0xffffffff}
+
 type key struct {
 	value []byte
 	tag   uint32
@@ -201,7 +203,7 @@ func programMangle(vni uint32, add bool) (err error) {
 	var (
 		p      = strconv.FormatUint(uint64(vxlanPort), 10)
 		c      = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)
-		m      = strconv.FormatUint(uint64(mark), 10)
+		m      = strconv.FormatUint(uint64(r), 10)
 		chain  = "OUTPUT"
 		rule   = []string{"-p", "udp", "--dport", p, "-m", "u32", "--u32", c, "-j", "MARK", "--set-mark", m}
 		a      = "-A"
@@ -271,6 +273,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 			Proto: netlink.XFRM_PROTO_ESP,
 			Spi:   spi.reverse,
 			Mode:  netlink.XFRM_MODE_TRANSPORT,
+			Reqid: r,
 		}
 		if add {
 			rSA.Aead = buildAeadAlgo(k, spi.reverse)
@@ -296,6 +299,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 			Proto: netlink.XFRM_PROTO_ESP,
 			Spi:   spi.forward,
 			Mode:  netlink.XFRM_MODE_TRANSPORT,
+			Reqid: r,
 		}
 		if add {
 			fSA.Aead = buildAeadAlgo(k, spi.forward)
@@ -325,17 +329,18 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 		xfrmProgram = ns.NlHandle().XfrmPolicyAdd
 	}
 
-	fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src))
+	// Create a congruent cidr
+	s := types.GetMinimalIP(fSA.Src)
+	d := types.GetMinimalIP(fSA.Dst)
+	fullMask := net.CIDRMask(8*len(s), 8*len(s))
 
 	fPol := &netlink.XfrmPolicy{
-		Src:     &net.IPNet{IP: fSA.Src, Mask: fullMask},
+		Src:     &net.IPNet{IP: s, Mask: fullMask},
-		Dst:     &net.IPNet{IP: fSA.Dst, Mask: fullMask},
+		Dst:     &net.IPNet{IP: d, Mask: fullMask},
 		Dir:     netlink.XFRM_DIR_OUT,
 		Proto:   17,
 		DstPort: 4789,
-		Mark: &netlink.XfrmMark{
+		Mark:    &spMark,
-			Value: mark,
-		},
 		Tmpls: []netlink.XfrmPolicyTmpl{
 			{
 				Src:   fSA.Src,
@@ -343,6 +348,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 				Proto: netlink.XFRM_PROTO_ESP,
 				Mode:  netlink.XFRM_MODE_TRANSPORT,
 				Spi:   fSA.Spi,
+				Reqid: r,
 			},
 		},
 	}
@@ -426,6 +432,8 @@ func (d *driver) secMapWalk(f func(string, []*spi) ([]*spi, bool)) error {
 }
 
 func (d *driver) setKeys(keys []*key) error {
+	// Remove any stale policy, state
+	clearEncryptionStates()
 	// Accept the encryption keys and clear any stale encryption map
 	d.Lock()
 	d.keys = keys
@@ -526,7 +534,7 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr
 	}
 
 	if newIdx > -1 {
-		// +RSA2
+		// +rSA2
 		programSA(lIP, rIP, spis[newIdx], curKeys[newIdx], reverse, true)
 	}
 
@@ -535,16 +543,17 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr
 		fSA2, _, _ := programSA(lIP, rIP, spis[priIdx], curKeys[priIdx], forward, true)
 
 		// +fSP2, -fSP1
-		fullMask := net.CIDRMask(8*len(fSA2.Src), 8*len(fSA2.Src))
+		s := types.GetMinimalIP(fSA2.Src)
+		d := types.GetMinimalIP(fSA2.Dst)
+		fullMask := net.CIDRMask(8*len(s), 8*len(s))
+
 		fSP1 := &netlink.XfrmPolicy{
-			Src:     &net.IPNet{IP: fSA2.Src, Mask: fullMask},
+			Src:     &net.IPNet{IP: s, Mask: fullMask},
-			Dst:     &net.IPNet{IP: fSA2.Dst, Mask: fullMask},
+			Dst:     &net.IPNet{IP: d, Mask: fullMask},
 			Dir:     netlink.XFRM_DIR_OUT,
 			Proto:   17,
 			DstPort: 4789,
-			Mark: &netlink.XfrmMark{
+			Mark:    &spMark,
-				Value: mark,
-			},
 			Tmpls: []netlink.XfrmPolicyTmpl{
 				{
 					Src:   fSA2.Src,
@@ -552,6 +561,7 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr
 					Proto: netlink.XFRM_PROTO_ESP,
 					Mode:  netlink.XFRM_MODE_TRANSPORT,
 					Spi:   fSA2.Spi,
+					Reqid: r,
 				},
 			},
 		}
@@ -597,3 +607,33 @@ func (n *network) maxMTU() int {
 	}
 	return mtu
 }
+
+func clearEncryptionStates() {
+	nlh := ns.NlHandle()
+	spList, err := nlh.XfrmPolicyList(netlink.FAMILY_ALL)
+	if err != nil {
+		logrus.Warnf("Failed to retrieve SP list for cleanup: %v", err)
+	}
+	saList, err := nlh.XfrmStateList(netlink.FAMILY_ALL)
+	if err != nil {
+		logrus.Warnf("Failed to retrieve SA list for cleanup: %v", err)
+	}
+	for _, sp := range spList {
+		if sp.Mark != nil && sp.Mark.Value == spMark.Value {
+			if err := nlh.XfrmPolicyDel(&sp); err != nil {
+				logrus.Warnf("Failed to delete stale SP %s: %v", sp, err)
+				continue
+			}
+			logrus.Debugf("Removed stale SP: %s", sp)
+		}
+	}
+	for _, sa := range saList {
+		if sa.Reqid == r {
+			if err := nlh.XfrmStateDel(&sa); err != nil {
+				logrus.Warnf("Failed to delete stale SA %s: %v", sa, err)
+				continue
+			}
+			logrus.Debugf("Removed stale SA: %s", sa)
+		}
+	}
+}