From 9a2464f4369679b46dc9b0217b57c5b151eaa62f Mon Sep 17 00:00:00 2001 From: Chris Telfer Date: Fri, 7 Sep 2018 09:48:05 -0400 Subject: [PATCH 1/2] Set east-west load balancing to use direct routing Modify the loadbalancing for east-west traffic to use direct routing rather than NAT and update tasks to use direct service return under linux. This avoids hiding the source address of the sender and improves the performance in single-client/single-server tests. Signed-off-by: Chris Telfer --- libnetwork/ipvs/constants.go | 20 ++++++++++++++++++++ libnetwork/osl/namespace_linux.go | 30 ++++++++++++++++++++++++++++++ libnetwork/osl/sandbox.go | 4 ++++ libnetwork/sandbox.go | 28 +++++++++++++++++++++++++--- libnetwork/service_linux.go | 24 ++++++++++++++++++------ 5 files changed, 97 insertions(+), 9 deletions(-) diff --git a/libnetwork/ipvs/constants.go b/libnetwork/ipvs/constants.go index d36bec0e80..b6b7f2bb5e 100644 --- a/libnetwork/ipvs/constants.go +++ b/libnetwork/ipvs/constants.go @@ -145,3 +145,23 @@ const ( // addresses. SourceHashing = "sh" ) + +const ( + // ConnFwdMask is a mask for the fwd methods + ConnFwdMask = 0x0007 + + // ConnFwdMasq denotes forwarding via masquerading/NAT + ConnFwdMasq = 0x0000 + + // ConnFwdLocalNode denotes forwarding to a local node + ConnFwdLocalNode = 0x0001 + + // ConnFwdTunnel denotes forwarding via a tunnel + ConnFwdTunnel = 0x0002 + + // ConnFwdDirectRoute denotes forwarding via direct routing + ConnFwdDirectRoute = 0x0003 + + // ConnFwdBypass denotes forwarding while bypassing the cache + ConnFwdBypass = 0x0004 +) diff --git a/libnetwork/osl/namespace_linux.go b/libnetwork/osl/namespace_linux.go index abb748b567..3879b03444 100644 --- a/libnetwork/osl/namespace_linux.go +++ b/libnetwork/osl/namespace_linux.go @@ -384,6 +384,36 @@ func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error { return n.nlHandle.AddrDel(iface, &netlink.Addr{IPNet: ip}) } +func (n *networkNamespace) DisableARPForVIP(srcName string) (Err error) { + dstName := "" + for _, i := range n.Interfaces() { + if i.SrcName() == srcName { + dstName = i.DstName() + break + } + } + if dstName == "" { + return fmt.Errorf("failed to find interface %s in sandbox", srcName) + } + + err := n.InvokeFunc(func() { + path := filepath.Join("/proc/sys/net/ipv4/conf", dstName, "arp_ignore") + if err := ioutil.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil { + Err = fmt.Errorf("Failed to set %s to 1: %v", path, err) + return + } + path = filepath.Join("/proc/sys/net/ipv4/conf", dstName, "arp_announce") + if err := ioutil.WriteFile(path, []byte{'2', '\n'}, 0644); err != nil { + Err = fmt.Errorf("Failed to set %s to 2: %v", path, err) + return + } + }) + if err != nil { + return err + } + return +} + func (n *networkNamespace) InvokeFunc(f func()) error { return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error { f() diff --git a/libnetwork/osl/sandbox.go b/libnetwork/osl/sandbox.go index 5019e068df..198cf641a1 100644 --- a/libnetwork/osl/sandbox.go +++ b/libnetwork/osl/sandbox.go @@ -51,6 +51,10 @@ type Sandbox interface { // RemoveAliasIP removes the passed IP address from the named interface RemoveAliasIP(ifName string, ip *net.IPNet) error + // DisableARPForVIP disables ARP replies and requests for VIP addresses + // on a particular interface + DisableARPForVIP(ifName string) error + // Add a static route to the sandbox. AddStaticRoute(*types.StaticRoute) error diff --git a/libnetwork/sandbox.go b/libnetwork/sandbox.go index d98f359a7d..185b0adbdd 100644 --- a/libnetwork/sandbox.go +++ b/libnetwork/sandbox.go @@ -730,7 +730,7 @@ func (sb *sandbox) DisableService() (err error) { return nil } -func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { +func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint, ingress bool) { for _, i := range osSbox.Info().Interfaces() { // Only remove the interfaces owned by this endpoint from the sandbox. if ep.hasInterface(i.SrcName()) { @@ -742,8 +742,16 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { ep.Lock() joinInfo := ep.joinInfo + vip := ep.virtualIP ep.Unlock() + if len(vip) > 0 && !ingress { + ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)} + if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil { + logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet) + } + } + if joinInfo == nil { return } @@ -760,6 +768,7 @@ func (sb *sandbox) releaseOSSbox() { sb.Lock() osSbox := sb.osSbox sb.osSbox = nil + ingress := sb.ingress sb.Unlock() if osSbox == nil { @@ -767,7 +776,7 @@ func (sb *sandbox) releaseOSSbox() { } for _, ep := range sb.getConnectedEndpoints() { - releaseOSSboxResources(osSbox, ep) + releaseOSSboxResources(osSbox, ep, ingress) } osSbox.Destroy() @@ -854,6 +863,18 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil { return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err) } + + if len(ep.virtualIP) > 0 && !sb.ingress { + if sb.loadBalancerNID == "" { + if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil { + return fmt.Errorf("failed disable ARP for VIP: %v", err) + } + } + ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)} + if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil { + return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err) + } + } } if joinInfo != nil { @@ -904,9 +925,10 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { sb.Lock() osSbox := sb.osSbox inDelete := sb.inDelete + ingress := sb.ingress sb.Unlock() if osSbox != nil { - releaseOSSboxResources(osSbox, ep) + releaseOSSboxResources(osSbox, ep, ingress) } sb.Lock() diff --git a/libnetwork/service_linux.go b/libnetwork/service_linux.go index 3d61adeb67..5e9e0e03e0 100644 --- a/libnetwork/service_linux.go +++ b/libnetwork/service_linux.go @@ -142,7 +142,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) { } logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID()) - if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false); err != nil { + if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.ingress); err != nil { logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err) return } @@ -158,6 +158,9 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) { Address: ip, Weight: 1, } + if !n.ingress { + d.ConnectionFlags = ipvs.ConnFwdDirectRoute + } // Remove the sched name before using the service to add // destination. @@ -203,6 +206,9 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR Address: ip, Weight: 1, } + if !n.ingress { + d.ConnectionFlags = ipvs.ConnFwdDirectRoute + } if fullRemove { if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT { @@ -231,7 +237,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR } } - if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true); err != nil { + if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.ingress); err != nil { logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err) } @@ -566,7 +572,7 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) { // Invoke fwmarker reexec routine to mark vip destined packets with // the passed firewall mark. -func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool) error { +func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, isIngress bool) error { var ingressPortsFile string if len(ingressPorts) != 0 { @@ -584,9 +590,14 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port addDelOpt = "-D" } + isIngressOpt := "false" + if isIngress { + isIngressOpt = "true" + } + cmd := &exec.Cmd{ Path: reexec.Self(), - Args: append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String()), + Args: append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String(), isIngressOpt), Stdout: os.Stdout, Stderr: os.Stderr, } @@ -603,7 +614,7 @@ func fwMarker() { runtime.LockOSThread() defer runtime.UnlockOSThread() - if len(os.Args) < 7 { + if len(os.Args) < 8 { logrus.Error("invalid number of arguments..") os.Exit(1) } @@ -645,7 +656,8 @@ func fwMarker() { os.Exit(5) } - if addDelOpt == "-A" { + isIngressOpt := os.Args[7] + if addDelOpt == "-A" && isIngressOpt == "true" { eIP, subnet, err := net.ParseCIDR(os.Args[6]) if err != nil { logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err) From 013ca3bdf84182194d12c9cf637c33a82755813d Mon Sep 17 00:00:00 2001 From: Chris Telfer Date: Tue, 9 Oct 2018 10:04:31 -0400 Subject: [PATCH 2/2] Make DSR an overlay-specific driver "option" Allow DSR to be a configurable option through a generic option to the overlay driver. On the one hand this approach makes sense insofar as only overlay networks can currently perform load balancing. On the other hand, this approach has several issues. First, should we create another type of swarm scope network, this will prevent it working. Second, the service core code is separate from the driver code and the driver code can't influence the core data structures. So the driver code can't set this option itself. Therefore, implementing in this way requires some hack code to test for this option in controller.NewNetwork. A more correct approach would be to make this a generic option for any network. Then the driver could ignore, reject or be unaware of the option depending on the chosen model. This would require changes to: * libnetwork - naturally * the docker API - to carry the option * swarmkit - to propagate the option * the docker CLI - to support the option * moby - to translate the API option into a libnetwork option Given the urgency of requests to address this issue, this approach will be saved for a future iteration. Signed-off-by: Chris Telfer --- libnetwork/controller.go | 37 ++++++++++++----- libnetwork/network.go | 81 +++++++++++++++++++++---------------- libnetwork/sandbox.go | 14 +++---- libnetwork/service_linux.go | 21 ++++------ 4 files changed, 90 insertions(+), 63 deletions(-) diff --git a/libnetwork/controller.go b/libnetwork/controller.go index f956ddb280..2896011dbf 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -700,6 +700,9 @@ func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, return nil } +// XXX This should be made driver agnostic. See comment below. +const overlayDSROptionString = "dsr" + // NewNetwork creates a new network of the specified network type. The options // are network specific and modeled in a generic way. func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) { @@ -723,15 +726,16 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... defaultIpam := defaultIpamForNetworkType(networkType) // Construct the network object network := &network{ - name: name, - networkType: networkType, - generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)}, - ipamType: defaultIpam, - id: id, - created: time.Now(), - ctrlr: c, - persist: true, - drvOnce: &sync.Once{}, + name: name, + networkType: networkType, + generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)}, + ipamType: defaultIpam, + id: id, + created: time.Now(), + ctrlr: c, + persist: true, + drvOnce: &sync.Once{}, + loadBalancerMode: loadBalancerModeDefault, } network.processOptions(options...) @@ -829,6 +833,21 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... } }() + // XXX If the driver type is "overlay" check the options for DSR + // being set. If so, set the network's load balancing mode to DSR. + // This should really be done in a network option, but due to + // time pressure to get this in without adding changes to moby, + // swarm and CLI, it is being implemented as a driver-specific + // option. Unfortunately, drivers can't influence the core + // "libnetwork.network" data type. Hence we need this hack code + // to implement in this manner. + if gval, ok := network.generic[netlabel.GenericData]; ok && network.networkType == "overlay" { + optMap := gval.(map[string]string) + if _, ok := optMap[overlayDSROptionString]; ok { + network.loadBalancerMode = loadBalancerModeDSR + } + } + addToStore: // First store the endpoint count, then the network. To avoid to // end up with a datastore containing a network and not an epCnt, diff --git a/libnetwork/network.go b/libnetwork/network.go index 052aa8febe..0a4a2277b0 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -199,43 +199,50 @@ func (i *IpamInfo) UnmarshalJSON(data []byte) error { } type network struct { - ctrlr *controller - name string - networkType string - id string - created time.Time - scope string // network data scope - labels map[string]string - ipamType string - ipamOptions map[string]string - addrSpace string - ipamV4Config []*IpamConf - ipamV6Config []*IpamConf - ipamV4Info []*IpamInfo - ipamV6Info []*IpamInfo - enableIPv6 bool - postIPv6 bool - epCnt *endpointCnt - generic options.Generic - dbIndex uint64 - dbExists bool - persist bool - stopWatchCh chan struct{} - drvOnce *sync.Once - resolverOnce sync.Once - resolver []Resolver - internal bool - attachable bool - inDelete bool - ingress bool - driverTables []networkDBTable - dynamic bool - configOnly bool - configFrom string - loadBalancerIP net.IP + ctrlr *controller + name string + networkType string + id string + created time.Time + scope string // network data scope + labels map[string]string + ipamType string + ipamOptions map[string]string + addrSpace string + ipamV4Config []*IpamConf + ipamV6Config []*IpamConf + ipamV4Info []*IpamInfo + ipamV6Info []*IpamInfo + enableIPv6 bool + postIPv6 bool + epCnt *endpointCnt + generic options.Generic + dbIndex uint64 + dbExists bool + persist bool + stopWatchCh chan struct{} + drvOnce *sync.Once + resolverOnce sync.Once + resolver []Resolver + internal bool + attachable bool + inDelete bool + ingress bool + driverTables []networkDBTable + dynamic bool + configOnly bool + configFrom string + loadBalancerIP net.IP + loadBalancerMode string sync.Mutex } +const ( + loadBalancerModeNAT = "NAT" + loadBalancerModeDSR = "DSR" + loadBalancerModeDefault = loadBalancerModeNAT +) + func (n *network) Name() string { n.Lock() defer n.Unlock() @@ -475,6 +482,7 @@ func (n *network) CopyTo(o datastore.KVObject) error { dstN.configOnly = n.configOnly dstN.configFrom = n.configFrom dstN.loadBalancerIP = n.loadBalancerIP + dstN.loadBalancerMode = n.loadBalancerMode // copy labels if dstN.labels == nil { @@ -592,6 +600,7 @@ func (n *network) MarshalJSON() ([]byte, error) { netMap["configOnly"] = n.configOnly netMap["configFrom"] = n.configFrom netMap["loadBalancerIP"] = n.loadBalancerIP + netMap["loadBalancerMode"] = n.loadBalancerMode return json.Marshal(netMap) } @@ -705,6 +714,10 @@ func (n *network) UnmarshalJSON(b []byte) (err error) { if v, ok := netMap["loadBalancerIP"]; ok { n.loadBalancerIP = net.ParseIP(v.(string)) } + n.loadBalancerMode = loadBalancerModeDefault + if v, ok := netMap["loadBalancerMode"]; ok { + n.loadBalancerMode = v.(string) + } // Reconcile old networks with the recently added `--ipv6` flag if !n.enableIPv6 { n.enableIPv6 = len(n.ipamV6Info) > 0 diff --git a/libnetwork/sandbox.go b/libnetwork/sandbox.go index 185b0adbdd..03c9215786 100644 --- a/libnetwork/sandbox.go +++ b/libnetwork/sandbox.go @@ -730,7 +730,7 @@ func (sb *sandbox) DisableService() (err error) { return nil } -func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint, ingress bool) { +func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { for _, i := range osSbox.Info().Interfaces() { // Only remove the interfaces owned by this endpoint from the sandbox. if ep.hasInterface(i.SrcName()) { @@ -743,9 +743,10 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint, ingress bool) { ep.Lock() joinInfo := ep.joinInfo vip := ep.virtualIP + lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR ep.Unlock() - if len(vip) > 0 && !ingress { + if len(vip) > 0 && lbModeIsDSR { ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)} if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil { logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet) @@ -768,7 +769,6 @@ func (sb *sandbox) releaseOSSbox() { sb.Lock() osSbox := sb.osSbox sb.osSbox = nil - ingress := sb.ingress sb.Unlock() if osSbox == nil { @@ -776,7 +776,7 @@ func (sb *sandbox) releaseOSSbox() { } for _, ep := range sb.getConnectedEndpoints() { - releaseOSSboxResources(osSbox, ep, ingress) + releaseOSSboxResources(osSbox, ep) } osSbox.Destroy() @@ -840,6 +840,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { ep.Lock() joinInfo := ep.joinInfo i := ep.iface + lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR ep.Unlock() if ep.needResolver() { @@ -864,7 +865,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err) } - if len(ep.virtualIP) > 0 && !sb.ingress { + if len(ep.virtualIP) > 0 && lbModeIsDSR { if sb.loadBalancerNID == "" { if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil { return fmt.Errorf("failed disable ARP for VIP: %v", err) @@ -925,10 +926,9 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { sb.Lock() osSbox := sb.osSbox inDelete := sb.inDelete - ingress := sb.ingress sb.Unlock() if osSbox != nil { - releaseOSSboxResources(osSbox, ep, ingress) + releaseOSSboxResources(osSbox, ep) } sb.Lock() diff --git a/libnetwork/service_linux.go b/libnetwork/service_linux.go index 5e9e0e03e0..451f760b61 100644 --- a/libnetwork/service_linux.go +++ b/libnetwork/service_linux.go @@ -142,7 +142,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) { } logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID()) - if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.ingress); err != nil { + if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil { logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err) return } @@ -158,7 +158,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) { Address: ip, Weight: 1, } - if !n.ingress { + if n.loadBalancerMode == loadBalancerModeDSR { d.ConnectionFlags = ipvs.ConnFwdDirectRoute } @@ -206,7 +206,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR Address: ip, Weight: 1, } - if !n.ingress { + if n.loadBalancerMode == loadBalancerModeDSR { d.ConnectionFlags = ipvs.ConnFwdDirectRoute } @@ -237,7 +237,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR } } - if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.ingress); err != nil { + if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil { logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err) } @@ -572,7 +572,7 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) { // Invoke fwmarker reexec routine to mark vip destined packets with // the passed firewall mark. -func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, isIngress bool) error { +func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error { var ingressPortsFile string if len(ingressPorts) != 0 { @@ -590,14 +590,9 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port addDelOpt = "-D" } - isIngressOpt := "false" - if isIngress { - isIngressOpt = "true" - } - cmd := &exec.Cmd{ Path: reexec.Self(), - Args: append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String(), isIngressOpt), + Args: append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String(), lbMode), Stdout: os.Stdout, Stderr: os.Stderr, } @@ -656,8 +651,8 @@ func fwMarker() { os.Exit(5) } - isIngressOpt := os.Args[7] - if addDelOpt == "-A" && isIngressOpt == "true" { + lbMode := os.Args[7] + if addDelOpt == "-A" && lbMode == loadBalancerModeNAT { eIP, subnet, err := net.ParseCIDR(os.Args[6]) if err != nil { logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)