From 81296dda15e79f09ee3c20000c71b287c5b54d98 Mon Sep 17 00:00:00 2001 From: dhilipkumars Date: Sat, 20 May 2017 23:09:30 +0530 Subject: [PATCH] Update GetService(*Service) api. Update UT with new apis and remove dependency from ipvsadm Signed-off-by: dhilipkumars --- libnetwork/ipvs/ipvs.go | 44 +++------ libnetwork/ipvs/ipvs_test.go | 167 +++++++++++++++----------------- libnetwork/ipvs/netlink.go | 179 ++++++++++++++++++++++------------- 3 files changed, 204 insertions(+), 186 deletions(-) diff --git a/libnetwork/ipvs/ipvs.go b/libnetwork/ipvs/ipvs.go index e442ecb2e9..11d4c297de 100644 --- a/libnetwork/ipvs/ipvs.go +++ b/libnetwork/ipvs/ipvs.go @@ -6,6 +6,7 @@ import ( "net" "syscall" + "fmt" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" ) @@ -28,6 +29,7 @@ type Service struct { Stats SvcStats } +// SvcStats defines an IPVS service statistics type SvcStats struct { Connections uint32 PacketsIn uint32 @@ -132,44 +134,28 @@ func (i *Handle) DelDestination(s *Service, d *Destination) error { return i.doCmd(s, d, ipvsCmdDelDest) } -// GetServices returns an array of services configured at the kernel +// GetServices returns an array of services configured on the Node func (i *Handle) GetServices() ([]*Service, error) { - var res []*Service - //var emptySrv Service - //var emptyDest Destination - - msgs, err := i.doCmdwithResponse(nil, nil, ipvsCmdGetService) - if err != nil { - return nil, err - } - - for _, msg := range msgs { - srv, err := i.ParseService(msg) - if err != nil { - return res, err - } - res = append(res, srv) - } - - return res, nil + return i.doGetServicesCmd(nil) } -// GetDestinations returns an array of Destinations configured for this +// GetDestinations returns an array of Destinations configured for this Service func (i *Handle) GetDestinations(s *Service) ([]*Destination, error) { + return i.doGetDestinationsCmd(s, nil) +} - var res []*Destination +//GetService gets details of a specific IPVS services, useful in updating statisics etc., +func (i *Handle) GetService(s *Service) (*Service, error) { - msgs, err := i.doCmdwithResponse(s, nil, ipvsCmdGetDest) + res, err := i.doGetServicesCmd(s) if err != nil { return nil, err } - for _, msg := range msgs { - dest, err := i.ParseDestination(msg) - if err != nil { - return res, err - } - res = append(res, dest) + //We are looking for exactly one service otherwise error out + if len(res) != 1 { + return nil, fmt.Errorf("Expected only one service obtained=%d", len(res)) } - return res, nil + + return res[0], nil } diff --git a/libnetwork/ipvs/ipvs_test.go b/libnetwork/ipvs/ipvs_test.go index 62c9a8f9cf..adb8157afb 100644 --- a/libnetwork/ipvs/ipvs_test.go +++ b/libnetwork/ipvs/ipvs_test.go @@ -3,10 +3,7 @@ package ipvs import ( - "fmt" "net" - "os/exec" - "strings" "syscall" "testing" @@ -44,75 +41,73 @@ var ( } ) -func checkDestination(t *testing.T, checkPresent bool, protocol, serviceAddress, realAddress, fwdMethod string) { - var ( - realServerStart bool - realServers []string - ) +func lookupFwMethod(fwMethod uint32) string { - out, err := exec.Command("ipvsadm", "-Ln").CombinedOutput() - require.NoError(t, err) - - for _, o := range strings.Split(string(out), "\n") { - cmpStr := serviceAddress - if protocol == "FWM" { - cmpStr = " " + cmpStr - } - - if strings.Contains(o, cmpStr) { - realServerStart = true - continue - } - - if realServerStart { - if !strings.Contains(o, "->") { - break - } - - realServers = append(realServers, o) - } - } - - for _, r := range realServers { - if strings.Contains(r, realAddress) { - parts := strings.Fields(r) - assert.Equal(t, fwdMethod, parts[2]) - return - } - } - - if checkPresent { - t.Fatalf("Did not find the destination %s fwdMethod %s in ipvs output", realAddress, fwdMethod) + switch fwMethod { + case ConnectionFlagMasq: + return fwdMethodStrings[0] + case ConnectionFlagTunnel: + return fwdMethodStrings[1] + case ConnectionFlagDirectRoute: + return fwdMethodStrings[2] } + return "" } -func checkService(t *testing.T, checkPresent bool, protocol, schedMethod, serviceAddress string) { - out, err := exec.Command("ipvsadm", "-Ln").CombinedOutput() +func checkDestination(t *testing.T, i *Handle, s *Service, d *Destination, checkPresent bool) { + dstFound := false + + dstArray, err := i.GetDestinations(s) require.NoError(t, err) - for _, o := range strings.Split(string(out), "\n") { - cmpStr := serviceAddress - if protocol == "FWM" { - cmpStr = " " + cmpStr - } - - if strings.Contains(o, cmpStr) { - parts := strings.Split(o, " ") - assert.Equal(t, protocol, parts[0]) - assert.Equal(t, serviceAddress, parts[2]) - assert.Equal(t, schedMethod, parts[3]) - - if !checkPresent { - t.Fatalf("Did not expect the service %s in ipvs output", serviceAddress) - } - - return + for _, dst := range dstArray { + if dst.Address.String() == d.Address.String() && dst.Port == d.Port && lookupFwMethod(dst.ConnectionFlags) == lookupFwMethod(d.ConnectionFlags) { + dstFound = true + break } } - if checkPresent { - t.Fatalf("Did not find the service %s in ipvs output", serviceAddress) + switch checkPresent { + case true: //The test expects the service to be present + if !dstFound { + + t.Fatalf("Did not find the service %s in ipvs output", d.Address.String()) + } + case false: //The test expects that the service should not be present + if dstFound { + t.Fatalf("Did not find the destination %s fwdMethod %s in ipvs output", d.Address.String(), lookupFwMethod(d.ConnectionFlags)) + } } + +} + +func checkService(t *testing.T, i *Handle, s *Service, checkPresent bool) { + + svcArray, err := i.GetServices() + require.NoError(t, err) + + svcFound := false + + for _, svc := range svcArray { + + if svc.Protocol == s.Protocol && svc.Address.String() == s.Address.String() && svc.Port == s.Port { + svcFound = true + break + } + } + + switch checkPresent { + case true: //The test expects the service to be present + if !svcFound { + + t.Fatalf("Did not find the service %s in ipvs output", s.Address.String()) + } + case false: //The test expects that the service should not be present + if svcFound { + t.Fatalf("Did not expect the service %s in ipvs output", s.Address.String()) + } + } + } func TestGetFamily(t *testing.T) { @@ -137,7 +132,6 @@ func TestService(t *testing.T) { for _, protocol := range protocols { for _, schedMethod := range schedMethods { - var serviceAddress string s := Service{ AddressFamily: nl.FAMILY_V4, @@ -147,24 +141,20 @@ func TestService(t *testing.T) { switch protocol { case "FWM": s.FWMark = 1234 - serviceAddress = fmt.Sprintf("%d", 1234) case "TCP": s.Protocol = syscall.IPPROTO_TCP s.Port = 80 s.Address = net.ParseIP("1.2.3.4") s.Netmask = 0xFFFFFFFF - serviceAddress = "1.2.3.4:80" case "UDP": s.Protocol = syscall.IPPROTO_UDP s.Port = 53 s.Address = net.ParseIP("2.3.4.5") - serviceAddress = "2.3.4.5:53" } err := i.NewService(&s) assert.NoError(t, err) - checkService(t, true, protocol, schedMethod, serviceAddress) - var lastMethod string + checkService(t, i, &s, true) for _, updateSchedMethod := range schedMethods { if updateSchedMethod == schedMethod { continue @@ -173,13 +163,18 @@ func TestService(t *testing.T) { s.SchedName = updateSchedMethod err = i.UpdateService(&s) assert.NoError(t, err) - checkService(t, true, protocol, updateSchedMethod, serviceAddress) - lastMethod = updateSchedMethod + checkService(t, i, &s, true) + + scopy, err := i.GetService(&s) + assert.NoError(t, err) + assert.Equal(t, (*scopy).Address.String(), s.Address.String()) + assert.Equal(t, (*scopy).Port, s.Port) + assert.Equal(t, (*scopy).Protocol, s.Protocol) } err = i.DelService(&s) assert.NoError(t, err) - checkService(t, false, protocol, lastMethod, serviceAddress) + checkService(t, i, &s, false) } } @@ -220,7 +215,6 @@ func TestDestination(t *testing.T) { require.NoError(t, err) for _, protocol := range protocols { - var serviceAddress string s := Service{ AddressFamily: nl.FAMILY_V4, @@ -230,26 +224,23 @@ func TestDestination(t *testing.T) { switch protocol { case "FWM": s.FWMark = 1234 - serviceAddress = fmt.Sprintf("%d", 1234) case "TCP": s.Protocol = syscall.IPPROTO_TCP s.Port = 80 s.Address = net.ParseIP("1.2.3.4") s.Netmask = 0xFFFFFFFF - serviceAddress = "1.2.3.4:80" case "UDP": s.Protocol = syscall.IPPROTO_UDP s.Port = 53 s.Address = net.ParseIP("2.3.4.5") - serviceAddress = "2.3.4.5:53" } err := i.NewService(&s) assert.NoError(t, err) - checkService(t, true, protocol, RoundRobin, serviceAddress) + checkService(t, i, &s, true) s.SchedName = "" - for j, fwdMethod := range fwdMethods { + for _, fwdMethod := range fwdMethods { d1 := Destination{ AddressFamily: nl.FAMILY_V4, Address: net.ParseIP("10.1.1.2"), @@ -258,10 +249,9 @@ func TestDestination(t *testing.T) { ConnectionFlags: fwdMethod, } - realAddress := "10.1.1.2:5000" err := i.NewDestination(&s, &d1) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j]) + checkDestination(t, i, &s, &d1, true) d2 := Destination{ AddressFamily: nl.FAMILY_V4, Address: net.ParseIP("10.1.1.3"), @@ -270,10 +260,9 @@ func TestDestination(t *testing.T) { ConnectionFlags: fwdMethod, } - realAddress = "10.1.1.3:5000" err = i.NewDestination(&s, &d2) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j]) + checkDestination(t, i, &s, &d2, true) d3 := Destination{ AddressFamily: nl.FAMILY_V4, @@ -283,32 +272,28 @@ func TestDestination(t *testing.T) { ConnectionFlags: fwdMethod, } - realAddress = "10.1.1.4:5000" err = i.NewDestination(&s, &d3) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j]) + checkDestination(t, i, &s, &d3, true) - for m, updateFwdMethod := range fwdMethods { + for _, updateFwdMethod := range fwdMethods { if updateFwdMethod == fwdMethod { continue } d1.ConnectionFlags = updateFwdMethod - realAddress = "10.1.1.2:5000" err = i.UpdateDestination(&s, &d1) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m]) + checkDestination(t, i, &s, &d1, true) d2.ConnectionFlags = updateFwdMethod - realAddress = "10.1.1.3:5000" err = i.UpdateDestination(&s, &d2) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m]) + checkDestination(t, i, &s, &d2, true) d3.ConnectionFlags = updateFwdMethod - realAddress = "10.1.1.4:5000" err = i.UpdateDestination(&s, &d3) assert.NoError(t, err) - checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m]) + checkDestination(t, i, &s, &d3, true) } err = i.DelDestination(&s, &d1) @@ -317,6 +302,8 @@ func TestDestination(t *testing.T) { assert.NoError(t, err) err = i.DelDestination(&s, &d3) assert.NoError(t, err) + checkDestination(t, i, &s, &d3, false) + } } } diff --git a/libnetwork/ipvs/netlink.go b/libnetwork/ipvs/netlink.go index 67f2eec3cb..365de3974a 100644 --- a/libnetwork/ipvs/netlink.go +++ b/libnetwork/ipvs/netlink.go @@ -19,7 +19,7 @@ import ( "github.com/vishvananda/netns" ) -//For Quick Reference IPVS related netlink message is described below. +//For Quick Reference IPVS related netlink message is described at the end of this file. var ( native = nl.NativeEndian() @@ -97,7 +97,6 @@ func fillService(s *Service) nl.NetlinkRequestData { mask: 0xFFFFFFFF, } nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFlags, f.Serialize()) - nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrTimeout, nl.Uint32Attr(s.Timeout)) nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrNetmask, nl.Uint32Attr(s.Netmask)) return cmdAttr @@ -129,13 +128,13 @@ func (i *Handle) doCmdwithResponse(s *Service, d *Destination, cmd uint8) ([][]b req.AddData(nl.NewRtAttr(ipvsCmdAttrService, nil)) //Add a dummy attribute } else { req.AddData(fillService(s)) - } if d == nil { if cmd == ipvsCmdGetDest { req.Flags |= syscall.NLM_F_DUMP } + } else { req.AddData(fillDestinaton(d)) } @@ -274,7 +273,8 @@ func parseIP(ip []byte, family uint16) (net.IP, error) { return resIP, nil } -func parseStats(msg []byte) (SvcStats, error) { +//parseStats +func assembleStats(msg []byte) (SvcStats, error) { var s SvcStats @@ -314,7 +314,7 @@ func parseStats(msg []byte) (SvcStats, error) { //assembleService assembles a services back from a hain of netlink attributes func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) { - var svc Service + var s Service for _, attr := range attrs { @@ -323,37 +323,87 @@ func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) { switch attrType { case ipvsSvcAttrAddressFamily: - svc.AddressFamily = native.Uint16(attr.Value) + s.AddressFamily = native.Uint16(attr.Value) case ipvsSvcAttrProtocol: - svc.Protocol = native.Uint16(attr.Value) + s.Protocol = native.Uint16(attr.Value) case ipvsSvcAttrAddress: - ip, err := parseIP(attr.Value, svc.AddressFamily) + ip, err := parseIP(attr.Value, s.AddressFamily) if err != nil { return nil, err } - svc.Address = ip + s.Address = ip case ipvsSvcAttrPort: - svc.Port = binary.BigEndian.Uint16(attr.Value) + s.Port = binary.BigEndian.Uint16(attr.Value) case ipvsSvcAttrFWMark: - svc.FWMark = native.Uint32(attr.Value) + s.FWMark = native.Uint32(attr.Value) case ipvsSvcAttrSchedName: - svc.SchedName = string(attr.Value) + s.SchedName = nl.BytesToString(attr.Value) case ipvsSvcAttrFlags: - svc.Flags = native.Uint32(attr.Value) + s.Flags = native.Uint32(attr.Value) case ipvsSvcAttrTimeout: - svc.Timeout = native.Uint32(attr.Value) + s.Timeout = native.Uint32(attr.Value) case ipvsSvcAttrNetmask: - svc.Timeout = native.Uint32(attr.Value) + s.Netmask = native.Uint32(attr.Value) case ipvsSvcAttrStats: - stats, err := parseStats(attr.Value) + stats, err := assembleStats(attr.Value) if err != nil { return nil, err } - svc.Stats = stats + s.Stats = stats } } - return &svc, nil + return &s, nil +} + +//parseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise +func (i *Handle) parseService(msg []byte) (*Service, error) { + + var s *Service + + //Remove General header for this message and parse the NetLink message + hdr := deserializeGenlMsg(msg) + NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) + if err != nil { + return nil, err + } + if len(NetLinkAttrs) == 0 { + return nil, fmt.Errorf("Error No valid net link message found while Parsing service record") + } + + //Now Parse and get IPVS related attributes messages packed in this message. + ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) + if err != nil { + return nil, err + } + + //Assemble all the IPVS related attribute messages and create a service record + s, err = assembleService(ipvsAttrs) + if err != nil { + return nil, err + } + + return s, nil +} + +//doGetServicesCmd a wrapper which could be used commonly for both GetServices() and GetService(*Service) +func (i *Handle) doGetServicesCmd(svc *Service) ([]*Service, error) { + var res []*Service + + msgs, err := i.doCmdwithResponse(svc, nil, ipvsCmdGetService) + if err != nil { + return nil, err + } + + for _, msg := range msgs { + srv, err := i.parseService(msg) + if err != nil { + return res, err + } + res = append(res, srv) + } + + return res, nil } func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) { @@ -374,6 +424,7 @@ func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) case ipvsDestAttrPort: d.Port = binary.BigEndian.Uint16(attr.Value) case ipvsDestAttrForwardingMethod: + d.ConnectionFlags = native.Uint32(attr.Value) case ipvsDestAttrWeight: d.Weight = int(native.Uint16(attr.Value)) case ipvsDestAttrUpperThreshold: @@ -384,42 +435,11 @@ func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) d.AddressFamily = native.Uint16(attr.Value) } } - return &d, nil } -//ParseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise -func (i *Handle) ParseService(msg []byte) (*Service, error) { - - var svc *Service - - //Remove General header for this message and parse the NetLink message - hdr := deserializeGenlMsg(msg) - NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) - if err != nil { - return nil, err - } - if len(NetLinkAttrs) == 0 { - return nil, fmt.Errorf("Error No valid net link message found while Parsing service record") - } - - //Now parse the smaller messages and get a list of attributes to construct a valid service - ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) - if err != nil { - return nil, err - } - - //Assemble netlink attributes and create a service record - svc, err = assembleService(ipvsAttrs) - if err != nil { - return nil, err - } - - return svc, nil -} - -//ParseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise -func (i *Handle) ParseDestination(msg []byte) (*Destination, error) { +//parseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise +func (i *Handle) parseDestination(msg []byte) (*Destination, error) { var dst *Destination //Remove General header for this message @@ -432,7 +452,7 @@ func (i *Handle) ParseDestination(msg []byte) (*Destination, error) { return nil, fmt.Errorf("Error No valid net link message found while Parsing service record") } - //Convert rest of the messages to Attribute Array + //Now Parse and get IPVS related attributes messages packed in this message. ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) if err != nil { return nil, err @@ -447,29 +467,54 @@ func (i *Handle) ParseDestination(msg []byte) (*Destination, error) { return dst, nil } +// doGetDestinationsCmd a wrapper function to be used by GetDestinations and GetDestination(d) apis +func (i *Handle) doGetDestinationsCmd(s *Service, d *Destination) ([]*Destination, error) { + + var res []*Destination + + msgs, err := i.doCmdwithResponse(s, d, ipvsCmdGetDest) + if err != nil { + return nil, err + } + + for _, msg := range msgs { + dest, err := i.parseDestination(msg) + if err != nil { + return res, err + } + res = append(res, dest) + } + return res, nil +} + //IPVS related netlink message format explained -//and unpacks these messages -/* EACH NETLINK REQUEST / RESPONSE WILL LOOK LIKE THIS when returned from +/* EACH NETLINK MSG is of the below format, this is what we will receive from execute() api. + If we have multiple netlink objects to process like GetServices() etc., execute() will + supply an array of this below object + NETLINK MSG |-----------------------------------| 0 1 2 3 -|--------|--------|--------|--------| -| CMD ID | VER | RESERVED | -|-----------------------------------| -| MSG LEN | MSG TYPE | -|-----------------------------------| -| | -| | -| []byte IPVS MSG PADDED BY 4 BYTES | -| | -| | -|-----------------------------------| +|--------|--------|--------|--------| - +| CMD ID | VER | RESERVED | |==> General Message Header represented by genlMsgHdr +|-----------------------------------| - +| ATTR LEN | ATTR TYPE | | +|-----------------------------------| | +| | | +| VALUE | | +| []byte Array of IPVS MSG | |==> Attribute Message represented by syscall.NetlinkRouteAttr +| PADDED BY 4 BYTES | | +| | | +|-----------------------------------| - -A response from the IPVS module is usually - + Once We strip genlMsgHdr from above NETLINK MSG, we should parse the VALUE. + VALUE will have an array of netlink attributes (syscall.NetlinkRouteAttr) such that each attribute will + represent a "Service" or "Destination" object's field. If we assemble these attributes we can construct + Service or Destination. + IPVS MSG |-----------------------------------| 0 1 2 3 |--------|--------|--------|--------|