From bee7d9219f5a6c65daad70cefdde725e90fc8049 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Mon, 25 Jul 2016 01:10:53 -0700 Subject: [PATCH 1/2] Properly construct CIDR in policy selector - Current code programs src/dst cidr like 192.168.100.126/128 Signed-off-by: Alessandro Boch --- libnetwork/drivers/overlay/encryption.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/libnetwork/drivers/overlay/encryption.go b/libnetwork/drivers/overlay/encryption.go index 14844f4cef..830ca9a3b0 100644 --- a/libnetwork/drivers/overlay/encryption.go +++ b/libnetwork/drivers/overlay/encryption.go @@ -291,11 +291,14 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { xfrmProgram = ns.NlHandle().XfrmPolicyAdd } - fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src)) + // Create a congruent cidr + s := types.GetMinimalIP(fSA.Src) + d := types.GetMinimalIP(fSA.Dst) + fullMask := net.CIDRMask(8*len(s), 8*len(s)) fPol := &netlink.XfrmPolicy{ - Src: &net.IPNet{IP: fSA.Src, Mask: fullMask}, - Dst: &net.IPNet{IP: fSA.Dst, Mask: fullMask}, + Src: &net.IPNet{IP: s, Mask: fullMask}, + Dst: &net.IPNet{IP: d, Mask: fullMask}, Dir: netlink.XFRM_DIR_OUT, Proto: 17, DstPort: 4789, @@ -491,7 +494,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, } if newIdx > -1 { - // +RSA2 + // +rSA2 programSA(lIP, rIP, spis[newIdx], curKeys[newIdx], reverse, true) } @@ -500,10 +503,13 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, fSA2, _, _ := programSA(lIP, rIP, spis[priIdx], curKeys[priIdx], forward, true) // +fSP2, -fSP1 - fullMask := net.CIDRMask(8*len(fSA2.Src), 8*len(fSA2.Src)) + s := types.GetMinimalIP(fSA2.Src) + d := types.GetMinimalIP(fSA2.Dst) + fullMask := net.CIDRMask(8*len(s), 8*len(s)) + fSP1 := &netlink.XfrmPolicy{ - Src: &net.IPNet{IP: fSA2.Src, Mask: fullMask}, - Dst: &net.IPNet{IP: fSA2.Dst, Mask: fullMask}, + Src: &net.IPNet{IP: s, Mask: fullMask}, + Dst: &net.IPNet{IP: d, Mask: fullMask}, Dir: netlink.XFRM_DIR_OUT, Proto: 17, DstPort: 4789, From 9272c89bb79f7acc81ff3b2f18df4e06227763e7 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Fri, 3 Feb 2017 23:34:19 -0800 Subject: [PATCH 2/2] Clear encryption states when joining cluster - Use the request id for labelling our SAs Signed-off-by: Alessandro Boch --- libnetwork/drivers/overlay/encryption.go | 50 ++++++++++++++++++++---- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/libnetwork/drivers/overlay/encryption.go b/libnetwork/drivers/overlay/encryption.go index 830ca9a3b0..bacd78f024 100644 --- a/libnetwork/drivers/overlay/encryption.go +++ b/libnetwork/drivers/overlay/encryption.go @@ -20,7 +20,7 @@ import ( ) const ( - mark = uint32(0xD0C4E3) + r = 0xD0C4E3 timeout = 30 pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8) ) @@ -31,6 +31,8 @@ const ( bidir ) +var spMark = netlink.XfrmMark{Value: uint32(r), Mask: 0xffffffff} + type key struct { value []byte tag uint32 @@ -196,7 +198,7 @@ func programMangle(vni uint32, add bool) (err error) { var ( p = strconv.FormatUint(uint64(vxlanPort), 10) c = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8) - m = strconv.FormatUint(uint64(mark), 10) + m = strconv.FormatUint(uint64(r), 10) chain = "OUTPUT" rule = []string{"-p", "udp", "--dport", p, "-m", "u32", "--u32", c, "-j", "MARK", "--set-mark", m} a = "-A" @@ -237,6 +239,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f Proto: netlink.XFRM_PROTO_ESP, Spi: spi.reverse, Mode: netlink.XFRM_MODE_TRANSPORT, + Reqid: r, } if add { rSA.Aead = buildAeadAlgo(k, spi.reverse) @@ -262,6 +265,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f Proto: netlink.XFRM_PROTO_ESP, Spi: spi.forward, Mode: netlink.XFRM_MODE_TRANSPORT, + Reqid: r, } if add { fSA.Aead = buildAeadAlgo(k, spi.forward) @@ -302,9 +306,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { Dir: netlink.XFRM_DIR_OUT, Proto: 17, DstPort: 4789, - Mark: &netlink.XfrmMark{ - Value: mark, - }, + Mark: &spMark, Tmpls: []netlink.XfrmPolicyTmpl{ { Src: fSA.Src, @@ -312,6 +314,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { Proto: netlink.XFRM_PROTO_ESP, Mode: netlink.XFRM_MODE_TRANSPORT, Spi: fSA.Spi, + Reqid: r, }, }, } @@ -395,6 +398,8 @@ func (d *driver) secMapWalk(f func(string, []*spi) ([]*spi, bool)) error { } func (d *driver) setKeys(keys []*key) error { + // Remove any stale policy, state + clearEncryptionStates() // Accept the encryption keys and clear any stale encryption map d.Lock() d.keys = keys @@ -513,9 +518,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, Dir: netlink.XFRM_DIR_OUT, Proto: 17, DstPort: 4789, - Mark: &netlink.XfrmMark{ - Value: mark, - }, + Mark: &spMark, Tmpls: []netlink.XfrmPolicyTmpl{ { Src: fSA2.Src, @@ -523,6 +526,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, Proto: netlink.XFRM_PROTO_ESP, Mode: netlink.XFRM_MODE_TRANSPORT, Spi: fSA2.Spi, + Reqid: r, }, }, } @@ -568,3 +572,33 @@ func (n *network) maxMTU() int { } return mtu } + +func clearEncryptionStates() { + nlh := ns.NlHandle() + spList, err := nlh.XfrmPolicyList(netlink.FAMILY_ALL) + if err != nil { + logrus.Warnf("Failed to retrieve SP list for cleanup: %v", err) + } + saList, err := nlh.XfrmStateList(netlink.FAMILY_ALL) + if err != nil { + logrus.Warnf("Failed to retrieve SA list for cleanup: %v", err) + } + for _, sp := range spList { + if sp.Mark != nil && sp.Mark.Value == spMark.Value { + if err := nlh.XfrmPolicyDel(&sp); err != nil { + logrus.Warnf("Failed to delete stale SP %s: %v", sp, err) + continue + } + logrus.Debugf("Removed stale SP: %s", sp) + } + } + for _, sa := range saList { + if sa.Reqid == r { + if err := nlh.XfrmStateDel(&sa); err != nil { + logrus.Warnf("Failed to delete stale SA %s: %v", sa, err) + continue + } + logrus.Debugf("Removed stale SA: %s", sa) + } + } +}