From 6368406c26b4159a124b330250939ca77d368df1 Mon Sep 17 00:00:00 2001 From: Madhu Venugopal Date: Tue, 19 Jul 2016 18:17:30 -0700 Subject: [PATCH] Adding Advertise-addr support With this change, all the auto-detection of the addresses are removed from libnetwork and the caller takes the responsibilty to have a proper advertise-addr in various scenarios (including externally facing public advertise-addr with an internal facing private listen-addr) Signed-off-by: Madhu Venugopal --- libnetwork/agent.go | 36 ++++++--------- libnetwork/cluster/provider.go | 3 +- libnetwork/cmd/dnet/dnet.go | 6 ++- libnetwork/discoverapi/discoverapi.go | 5 ++- libnetwork/drivers/overlay/encryption.go | 10 +++-- libnetwork/drivers/overlay/joinleave.go | 6 +-- libnetwork/drivers/overlay/ov_serf.go | 2 +- libnetwork/drivers/overlay/overlay.go | 56 +++++++++++++----------- libnetwork/networkdb/cluster.go | 2 +- libnetwork/networkdb/networkdb.go | 4 +- 10 files changed, 68 insertions(+), 62 deletions(-) diff --git a/libnetwork/agent.go b/libnetwork/agent.go index a5b05258ed..0e498cf84f 100644 --- a/libnetwork/agent.go +++ b/libnetwork/agent.go @@ -35,6 +35,7 @@ func (b ByTime) Less(i, j int) bool { return b[i].LamportTime < b[j].LamportTime type agent struct { networkDB *networkdb.NetworkDB bindAddr string + advertiseAddr string epTblCancel func() driverCancelFuncs map[string][]func() } @@ -236,25 +237,14 @@ func (c *controller) handleKeyChangeV1(keys []*types.EncryptionKey) error { func (c *controller) agentSetup() error { clusterProvider := c.cfg.Daemon.ClusterProvider - bindAddr, _, _ := net.SplitHostPort(clusterProvider.GetListenAddress()) + bindAddr := clusterProvider.GetLocalAddress() + advAddr := clusterProvider.GetAdvertiseAddress() remote := clusterProvider.GetRemoteAddress() remoteAddr, _, _ := net.SplitHostPort(remote) - // Determine the BindAddress from RemoteAddress or through best-effort routing - if !isValidClusteringIP(bindAddr) { - if !isValidClusteringIP(remoteAddr) { - remote = "8.8.8.8:53" - } - conn, err := net.Dial("udp", remote) - if err == nil { - bindHostPort := conn.LocalAddr().String() - bindAddr, _, _ = net.SplitHostPort(bindHostPort) - conn.Close() - } - } - - if bindAddr != "" && c.agent == nil { - if err := c.agentInit(bindAddr); err != nil { + logrus.Infof("Initializing Libnetwork Agent Local-addr=%s Adv-addr=%s Remote-addr =%s", bindAddr, advAddr, remoteAddr) + if advAddr != "" && c.agent == nil { + if err := c.agentInit(bindAddr, advAddr); err != nil { logrus.Errorf("Error in agentInit : %v", err) } else { c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { @@ -312,7 +302,7 @@ func (c *controller) getPrimaryKeyTag(subsys string) ([]byte, uint64) { return keys[1].Key, keys[1].LamportTime } -func (c *controller) agentInit(bindAddrOrInterface string) error { +func (c *controller) agentInit(bindAddrOrInterface, advertiseAddr string) error { if !c.isAgent() { return nil } @@ -325,9 +315,9 @@ func (c *controller) agentInit(bindAddrOrInterface string) error { keys, tags := c.getKeys(subsysGossip) hostname, _ := os.Hostname() nDB, err := networkdb.New(&networkdb.Config{ - BindAddr: bindAddr, - NodeName: hostname, - Keys: keys, + AdvertiseAddr: advertiseAddr, + NodeName: hostname, + Keys: keys, }) if err != nil { @@ -339,6 +329,7 @@ func (c *controller) agentInit(bindAddrOrInterface string) error { c.agent = &agent{ networkDB: nDB, bindAddr: bindAddr, + advertiseAddr: advertiseAddr, epTblCancel: cancel, driverCancelFuncs: make(map[string][]func()), } @@ -377,8 +368,9 @@ func (c *controller) agentDriverNotify(d driverapi.Driver) { } d.DiscoverNew(discoverapi.NodeDiscovery, discoverapi.NodeDiscoveryData{ - Address: c.agent.bindAddr, - Self: true, + Address: c.agent.advertiseAddr, + BindAddress: c.agent.bindAddr, + Self: true, }) drvEnc := discoverapi.DriverEncryptionConfig{} diff --git a/libnetwork/cluster/provider.go b/libnetwork/cluster/provider.go index 3b91a41ff8..f8dc6f09b8 100644 --- a/libnetwork/cluster/provider.go +++ b/libnetwork/cluster/provider.go @@ -4,7 +4,8 @@ package cluster type Provider interface { IsManager() bool IsAgent() bool - GetListenAddress() string + GetLocalAddress() string + GetAdvertiseAddress() string GetRemoteAddress() string ListenClusterEvents() <-chan struct{} } diff --git a/libnetwork/cmd/dnet/dnet.go b/libnetwork/cmd/dnet/dnet.go index 4d15c92c44..a2f9df523b 100644 --- a/libnetwork/cmd/dnet/dnet.go +++ b/libnetwork/cmd/dnet/dnet.go @@ -306,7 +306,11 @@ func (d *dnetConnection) IsAgent() bool { return d.Orchestration.Agent } -func (d *dnetConnection) GetListenAddress() string { +func (d *dnetConnection) GetAdvertiseAddress() string { + return d.Orchestration.Bind +} + +func (d *dnetConnection) GetLocalAddress() string { return d.Orchestration.Bind } diff --git a/libnetwork/discoverapi/discoverapi.go b/libnetwork/discoverapi/discoverapi.go index bee7ad1837..3710ea447e 100644 --- a/libnetwork/discoverapi/discoverapi.go +++ b/libnetwork/discoverapi/discoverapi.go @@ -26,8 +26,9 @@ const ( // NodeDiscoveryData represents the structure backing the node discovery data json string type NodeDiscoveryData struct { - Address string - Self bool + Address string + BindAddress string + Self bool } // DatastoreConfigData is the data for the datastore update event message diff --git a/libnetwork/drivers/overlay/encryption.go b/libnetwork/drivers/overlay/encryption.go index 36d2a2f2b7..c3a9e7991b 100644 --- a/libnetwork/drivers/overlay/encryption.go +++ b/libnetwork/drivers/overlay/encryption.go @@ -8,12 +8,13 @@ import ( "sync" "syscall" + "strconv" + log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" - "strconv" ) const ( @@ -85,6 +86,7 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal } lIP := types.GetMinimalIP(net.ParseIP(d.bindAddress)) + aIP := types.GetMinimalIP(net.ParseIP(d.advertiseAddress)) nodes := map[string]net.IP{} switch { @@ -107,7 +109,7 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal if add { for _, rIP := range nodes { - if err := setupEncryption(lIP, rIP, vxlanID, d.secMap, d.keys); err != nil { + if err := setupEncryption(lIP, aIP, rIP, vxlanID, d.secMap, d.keys); err != nil { log.Warnf("Failed to program network encryption between %s and %s: %v", lIP, rIP, err) } } @@ -122,7 +124,7 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal return nil } -func setupEncryption(localIP, remoteIP net.IP, vni uint32, em *encrMap, keys []*key) error { +func setupEncryption(localIP, advIP, remoteIP net.IP, vni uint32, em *encrMap, keys []*key) error { log.Debugf("Programming encryption for vxlan %d between %s and %s", vni, localIP, remoteIP) rIPs := remoteIP.String() @@ -134,7 +136,7 @@ func setupEncryption(localIP, remoteIP net.IP, vni uint32, em *encrMap, keys []* } for i, k := range keys { - spis := &spi{buildSPI(localIP, remoteIP, k.tag), buildSPI(remoteIP, localIP, k.tag)} + spis := &spi{buildSPI(advIP, remoteIP, k.tag), buildSPI(remoteIP, advIP, k.tag)} dir := reverse if i == 0 { dir = bidir diff --git a/libnetwork/drivers/overlay/joinleave.go b/libnetwork/drivers/overlay/joinleave.go index 1f21ee9b0d..b609e2b6f8 100644 --- a/libnetwork/drivers/overlay/joinleave.go +++ b/libnetwork/drivers/overlay/joinleave.go @@ -119,7 +119,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, } d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, - net.ParseIP(d.bindAddress), true) + net.ParseIP(d.advertiseAddress), true) if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil { log.Warn(err) @@ -128,7 +128,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, buf, err := proto.Marshal(&PeerRecord{ EndpointIP: ep.addr.String(), EndpointMAC: ep.mac.String(), - TunnelEndpointIP: d.bindAddress, + TunnelEndpointIP: d.advertiseAddress, }) if err != nil { return err @@ -159,7 +159,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri // Ignore local peers. We already know about them and they // should not be added to vxlan fdb. - if peer.TunnelEndpointIP == d.bindAddress { + if peer.TunnelEndpointIP == d.advertiseAddress { return } diff --git a/libnetwork/drivers/overlay/ov_serf.go b/libnetwork/drivers/overlay/ov_serf.go index bb60d95c2a..53c59b4df2 100644 --- a/libnetwork/drivers/overlay/ov_serf.go +++ b/libnetwork/drivers/overlay/ov_serf.go @@ -40,7 +40,7 @@ func (d *driver) serfInit() error { config := serf.DefaultConfig() config.Init() - config.MemberlistConfig.BindAddr = d.bindAddress + config.MemberlistConfig.BindAddr = d.advertiseAddress d.eventCh = make(chan serf.Event, 4) config.EventCh = d.eventCh diff --git a/libnetwork/drivers/overlay/overlay.go b/libnetwork/drivers/overlay/overlay.go index 88f8725e71..9a946a8180 100644 --- a/libnetwork/drivers/overlay/overlay.go +++ b/libnetwork/drivers/overlay/overlay.go @@ -31,22 +31,23 @@ const ( var initVxlanIdm = make(chan (bool), 1) type driver struct { - eventCh chan serf.Event - notifyCh chan ovNotify - exitCh chan chan struct{} - bindAddress string - neighIP string - config map[string]interface{} - peerDb peerNetworkMap - secMap *encrMap - serfInstance *serf.Serf - networks networkTable - store datastore.DataStore - localStore datastore.DataStore - vxlanIdm *idm.Idm - once sync.Once - joinOnce sync.Once - keys []*key + eventCh chan serf.Event + notifyCh chan ovNotify + exitCh chan chan struct{} + bindAddress string + advertiseAddress string + neighIP string + config map[string]interface{} + peerDb peerNetworkMap + secMap *encrMap + serfInstance *serf.Serf + networks networkTable + store datastore.DataStore + localStore datastore.DataStore + vxlanIdm *idm.Idm + once sync.Once + joinOnce sync.Once + keys []*key sync.Mutex } @@ -144,7 +145,7 @@ func (d *driver) restoreEndpoints() error { } n.incEndpointCount() - d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true) + d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true) } return nil } @@ -215,20 +216,25 @@ func validateSelf(node string) error { return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String()) } -func (d *driver) nodeJoin(node string, self bool) { +func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) { if self && !d.isSerfAlive() { - if err := validateSelf(node); err != nil { - logrus.Errorf("%s", err.Error()) - } d.Lock() - d.bindAddress = node + d.advertiseAddress = advertiseAddress + d.bindAddress = bindAddress d.Unlock() // If there is no cluster store there is no need to start serf. if d.store != nil { + if err := validateSelf(advertiseAddress); err != nil { + logrus.Warnf("%s", err.Error()) + } err := d.serfInit() if err != nil { logrus.Errorf("initializing serf instance failed: %v", err) + d.Lock() + d.advertiseAddress = "" + d.bindAddress = "" + d.Unlock() return } } @@ -236,7 +242,7 @@ func (d *driver) nodeJoin(node string, self bool) { d.Lock() if !self { - d.neighIP = node + d.neighIP = advertiseAddress } neighIP := d.neighIP d.Unlock() @@ -250,7 +256,7 @@ func (d *driver) nodeJoin(node string, self bool) { } }) if err != nil { - logrus.Errorf("joining serf neighbor %s failed: %v", node, err) + logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err) d.Lock() d.joinOnce = sync.Once{} d.Unlock() @@ -290,7 +296,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) if !ok || nodeData.Address == "" { return fmt.Errorf("invalid discovery data") } - d.nodeJoin(nodeData.Address, nodeData.Self) + d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self) case discoverapi.DatastoreConfig: if d.store != nil { return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already") diff --git a/libnetwork/networkdb/cluster.go b/libnetwork/networkdb/cluster.go index baf72bf014..1424303e01 100644 --- a/libnetwork/networkdb/cluster.go +++ b/libnetwork/networkdb/cluster.go @@ -81,7 +81,7 @@ func (nDB *NetworkDB) RemoveKey(key []byte) { func (nDB *NetworkDB) clusterInit() error { config := memberlist.DefaultLANConfig() config.Name = nDB.config.NodeName - config.BindAddr = nDB.config.BindAddr + config.AdvertiseAddr = nDB.config.AdvertiseAddr if nDB.config.BindPort != 0 { config.BindPort = nDB.config.BindPort diff --git a/libnetwork/networkdb/networkdb.go b/libnetwork/networkdb/networkdb.go index 8676986e7d..20bdc216ff 100644 --- a/libnetwork/networkdb/networkdb.go +++ b/libnetwork/networkdb/networkdb.go @@ -107,9 +107,9 @@ type Config struct { // NodeName is the cluster wide unique name for this node. NodeName string - // BindAddr is the local node's IP address that we bind to for + // AdvertiseAddr is the node's IP address that we advertise for // cluster communication. - BindAddr string + AdvertiseAddr string // BindPort is the local node's port to which we bind to for // cluster communication.