1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Update GetService(*Service) api. Update UT with new apis and remove dependency from ipvsadm

Signed-off-by: dhilipkumars <dhilip.kumar.s@huawei.com>
This commit is contained in:
dhilipkumars 2017-05-20 23:09:30 +05:30
parent 000775b918
commit 81296dda15
3 changed files with 204 additions and 186 deletions

View file

@ -6,6 +6,7 @@ import (
"net"
"syscall"
"fmt"
"github.com/vishvananda/netlink/nl"
"github.com/vishvananda/netns"
)
@ -28,6 +29,7 @@ type Service struct {
Stats SvcStats
}
// SvcStats defines an IPVS service statistics
type SvcStats struct {
Connections uint32
PacketsIn uint32
@ -132,44 +134,28 @@ func (i *Handle) DelDestination(s *Service, d *Destination) error {
return i.doCmd(s, d, ipvsCmdDelDest)
}
// GetServices returns an array of services configured at the kernel
// GetServices returns an array of services configured on the Node
func (i *Handle) GetServices() ([]*Service, error) {
var res []*Service
//var emptySrv Service
//var emptyDest Destination
msgs, err := i.doCmdwithResponse(nil, nil, ipvsCmdGetService)
if err != nil {
return nil, err
}
for _, msg := range msgs {
srv, err := i.ParseService(msg)
if err != nil {
return res, err
}
res = append(res, srv)
}
return res, nil
return i.doGetServicesCmd(nil)
}
// GetDestinations returns an array of Destinations configured for this
// GetDestinations returns an array of Destinations configured for this Service
func (i *Handle) GetDestinations(s *Service) ([]*Destination, error) {
return i.doGetDestinationsCmd(s, nil)
}
var res []*Destination
//GetService gets details of a specific IPVS services, useful in updating statisics etc.,
func (i *Handle) GetService(s *Service) (*Service, error) {
msgs, err := i.doCmdwithResponse(s, nil, ipvsCmdGetDest)
res, err := i.doGetServicesCmd(s)
if err != nil {
return nil, err
}
for _, msg := range msgs {
dest, err := i.ParseDestination(msg)
if err != nil {
return res, err
}
res = append(res, dest)
//We are looking for exactly one service otherwise error out
if len(res) != 1 {
return nil, fmt.Errorf("Expected only one service obtained=%d", len(res))
}
return res, nil
return res[0], nil
}

View file

@ -3,10 +3,7 @@
package ipvs
import (
"fmt"
"net"
"os/exec"
"strings"
"syscall"
"testing"
@ -44,75 +41,73 @@ var (
}
)
func checkDestination(t *testing.T, checkPresent bool, protocol, serviceAddress, realAddress, fwdMethod string) {
var (
realServerStart bool
realServers []string
)
func lookupFwMethod(fwMethod uint32) string {
out, err := exec.Command("ipvsadm", "-Ln").CombinedOutput()
require.NoError(t, err)
for _, o := range strings.Split(string(out), "\n") {
cmpStr := serviceAddress
if protocol == "FWM" {
cmpStr = " " + cmpStr
}
if strings.Contains(o, cmpStr) {
realServerStart = true
continue
}
if realServerStart {
if !strings.Contains(o, "->") {
break
}
realServers = append(realServers, o)
}
}
for _, r := range realServers {
if strings.Contains(r, realAddress) {
parts := strings.Fields(r)
assert.Equal(t, fwdMethod, parts[2])
return
}
}
if checkPresent {
t.Fatalf("Did not find the destination %s fwdMethod %s in ipvs output", realAddress, fwdMethod)
switch fwMethod {
case ConnectionFlagMasq:
return fwdMethodStrings[0]
case ConnectionFlagTunnel:
return fwdMethodStrings[1]
case ConnectionFlagDirectRoute:
return fwdMethodStrings[2]
}
return ""
}
func checkService(t *testing.T, checkPresent bool, protocol, schedMethod, serviceAddress string) {
out, err := exec.Command("ipvsadm", "-Ln").CombinedOutput()
func checkDestination(t *testing.T, i *Handle, s *Service, d *Destination, checkPresent bool) {
dstFound := false
dstArray, err := i.GetDestinations(s)
require.NoError(t, err)
for _, o := range strings.Split(string(out), "\n") {
cmpStr := serviceAddress
if protocol == "FWM" {
cmpStr = " " + cmpStr
}
if strings.Contains(o, cmpStr) {
parts := strings.Split(o, " ")
assert.Equal(t, protocol, parts[0])
assert.Equal(t, serviceAddress, parts[2])
assert.Equal(t, schedMethod, parts[3])
if !checkPresent {
t.Fatalf("Did not expect the service %s in ipvs output", serviceAddress)
}
return
for _, dst := range dstArray {
if dst.Address.String() == d.Address.String() && dst.Port == d.Port && lookupFwMethod(dst.ConnectionFlags) == lookupFwMethod(d.ConnectionFlags) {
dstFound = true
break
}
}
if checkPresent {
t.Fatalf("Did not find the service %s in ipvs output", serviceAddress)
switch checkPresent {
case true: //The test expects the service to be present
if !dstFound {
t.Fatalf("Did not find the service %s in ipvs output", d.Address.String())
}
case false: //The test expects that the service should not be present
if dstFound {
t.Fatalf("Did not find the destination %s fwdMethod %s in ipvs output", d.Address.String(), lookupFwMethod(d.ConnectionFlags))
}
}
}
func checkService(t *testing.T, i *Handle, s *Service, checkPresent bool) {
svcArray, err := i.GetServices()
require.NoError(t, err)
svcFound := false
for _, svc := range svcArray {
if svc.Protocol == s.Protocol && svc.Address.String() == s.Address.String() && svc.Port == s.Port {
svcFound = true
break
}
}
switch checkPresent {
case true: //The test expects the service to be present
if !svcFound {
t.Fatalf("Did not find the service %s in ipvs output", s.Address.String())
}
case false: //The test expects that the service should not be present
if svcFound {
t.Fatalf("Did not expect the service %s in ipvs output", s.Address.String())
}
}
}
func TestGetFamily(t *testing.T) {
@ -137,7 +132,6 @@ func TestService(t *testing.T) {
for _, protocol := range protocols {
for _, schedMethod := range schedMethods {
var serviceAddress string
s := Service{
AddressFamily: nl.FAMILY_V4,
@ -147,24 +141,20 @@ func TestService(t *testing.T) {
switch protocol {
case "FWM":
s.FWMark = 1234
serviceAddress = fmt.Sprintf("%d", 1234)
case "TCP":
s.Protocol = syscall.IPPROTO_TCP
s.Port = 80
s.Address = net.ParseIP("1.2.3.4")
s.Netmask = 0xFFFFFFFF
serviceAddress = "1.2.3.4:80"
case "UDP":
s.Protocol = syscall.IPPROTO_UDP
s.Port = 53
s.Address = net.ParseIP("2.3.4.5")
serviceAddress = "2.3.4.5:53"
}
err := i.NewService(&s)
assert.NoError(t, err)
checkService(t, true, protocol, schedMethod, serviceAddress)
var lastMethod string
checkService(t, i, &s, true)
for _, updateSchedMethod := range schedMethods {
if updateSchedMethod == schedMethod {
continue
@ -173,13 +163,18 @@ func TestService(t *testing.T) {
s.SchedName = updateSchedMethod
err = i.UpdateService(&s)
assert.NoError(t, err)
checkService(t, true, protocol, updateSchedMethod, serviceAddress)
lastMethod = updateSchedMethod
checkService(t, i, &s, true)
scopy, err := i.GetService(&s)
assert.NoError(t, err)
assert.Equal(t, (*scopy).Address.String(), s.Address.String())
assert.Equal(t, (*scopy).Port, s.Port)
assert.Equal(t, (*scopy).Protocol, s.Protocol)
}
err = i.DelService(&s)
assert.NoError(t, err)
checkService(t, false, protocol, lastMethod, serviceAddress)
checkService(t, i, &s, false)
}
}
@ -220,7 +215,6 @@ func TestDestination(t *testing.T) {
require.NoError(t, err)
for _, protocol := range protocols {
var serviceAddress string
s := Service{
AddressFamily: nl.FAMILY_V4,
@ -230,26 +224,23 @@ func TestDestination(t *testing.T) {
switch protocol {
case "FWM":
s.FWMark = 1234
serviceAddress = fmt.Sprintf("%d", 1234)
case "TCP":
s.Protocol = syscall.IPPROTO_TCP
s.Port = 80
s.Address = net.ParseIP("1.2.3.4")
s.Netmask = 0xFFFFFFFF
serviceAddress = "1.2.3.4:80"
case "UDP":
s.Protocol = syscall.IPPROTO_UDP
s.Port = 53
s.Address = net.ParseIP("2.3.4.5")
serviceAddress = "2.3.4.5:53"
}
err := i.NewService(&s)
assert.NoError(t, err)
checkService(t, true, protocol, RoundRobin, serviceAddress)
checkService(t, i, &s, true)
s.SchedName = ""
for j, fwdMethod := range fwdMethods {
for _, fwdMethod := range fwdMethods {
d1 := Destination{
AddressFamily: nl.FAMILY_V4,
Address: net.ParseIP("10.1.1.2"),
@ -258,10 +249,9 @@ func TestDestination(t *testing.T) {
ConnectionFlags: fwdMethod,
}
realAddress := "10.1.1.2:5000"
err := i.NewDestination(&s, &d1)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j])
checkDestination(t, i, &s, &d1, true)
d2 := Destination{
AddressFamily: nl.FAMILY_V4,
Address: net.ParseIP("10.1.1.3"),
@ -270,10 +260,9 @@ func TestDestination(t *testing.T) {
ConnectionFlags: fwdMethod,
}
realAddress = "10.1.1.3:5000"
err = i.NewDestination(&s, &d2)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j])
checkDestination(t, i, &s, &d2, true)
d3 := Destination{
AddressFamily: nl.FAMILY_V4,
@ -283,32 +272,28 @@ func TestDestination(t *testing.T) {
ConnectionFlags: fwdMethod,
}
realAddress = "10.1.1.4:5000"
err = i.NewDestination(&s, &d3)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[j])
checkDestination(t, i, &s, &d3, true)
for m, updateFwdMethod := range fwdMethods {
for _, updateFwdMethod := range fwdMethods {
if updateFwdMethod == fwdMethod {
continue
}
d1.ConnectionFlags = updateFwdMethod
realAddress = "10.1.1.2:5000"
err = i.UpdateDestination(&s, &d1)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m])
checkDestination(t, i, &s, &d1, true)
d2.ConnectionFlags = updateFwdMethod
realAddress = "10.1.1.3:5000"
err = i.UpdateDestination(&s, &d2)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m])
checkDestination(t, i, &s, &d2, true)
d3.ConnectionFlags = updateFwdMethod
realAddress = "10.1.1.4:5000"
err = i.UpdateDestination(&s, &d3)
assert.NoError(t, err)
checkDestination(t, true, protocol, serviceAddress, realAddress, fwdMethodStrings[m])
checkDestination(t, i, &s, &d3, true)
}
err = i.DelDestination(&s, &d1)
@ -317,6 +302,8 @@ func TestDestination(t *testing.T) {
assert.NoError(t, err)
err = i.DelDestination(&s, &d3)
assert.NoError(t, err)
checkDestination(t, i, &s, &d3, false)
}
}
}

View file

@ -19,7 +19,7 @@ import (
"github.com/vishvananda/netns"
)
//For Quick Reference IPVS related netlink message is described below.
//For Quick Reference IPVS related netlink message is described at the end of this file.
var (
native = nl.NativeEndian()
@ -97,7 +97,6 @@ func fillService(s *Service) nl.NetlinkRequestData {
mask: 0xFFFFFFFF,
}
nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFlags, f.Serialize())
nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrTimeout, nl.Uint32Attr(s.Timeout))
nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrNetmask, nl.Uint32Attr(s.Netmask))
return cmdAttr
@ -129,13 +128,13 @@ func (i *Handle) doCmdwithResponse(s *Service, d *Destination, cmd uint8) ([][]b
req.AddData(nl.NewRtAttr(ipvsCmdAttrService, nil)) //Add a dummy attribute
} else {
req.AddData(fillService(s))
}
if d == nil {
if cmd == ipvsCmdGetDest {
req.Flags |= syscall.NLM_F_DUMP
}
} else {
req.AddData(fillDestinaton(d))
}
@ -274,7 +273,8 @@ func parseIP(ip []byte, family uint16) (net.IP, error) {
return resIP, nil
}
func parseStats(msg []byte) (SvcStats, error) {
//parseStats
func assembleStats(msg []byte) (SvcStats, error) {
var s SvcStats
@ -314,7 +314,7 @@ func parseStats(msg []byte) (SvcStats, error) {
//assembleService assembles a services back from a hain of netlink attributes
func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) {
var svc Service
var s Service
for _, attr := range attrs {
@ -323,37 +323,87 @@ func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) {
switch attrType {
case ipvsSvcAttrAddressFamily:
svc.AddressFamily = native.Uint16(attr.Value)
s.AddressFamily = native.Uint16(attr.Value)
case ipvsSvcAttrProtocol:
svc.Protocol = native.Uint16(attr.Value)
s.Protocol = native.Uint16(attr.Value)
case ipvsSvcAttrAddress:
ip, err := parseIP(attr.Value, svc.AddressFamily)
ip, err := parseIP(attr.Value, s.AddressFamily)
if err != nil {
return nil, err
}
svc.Address = ip
s.Address = ip
case ipvsSvcAttrPort:
svc.Port = binary.BigEndian.Uint16(attr.Value)
s.Port = binary.BigEndian.Uint16(attr.Value)
case ipvsSvcAttrFWMark:
svc.FWMark = native.Uint32(attr.Value)
s.FWMark = native.Uint32(attr.Value)
case ipvsSvcAttrSchedName:
svc.SchedName = string(attr.Value)
s.SchedName = nl.BytesToString(attr.Value)
case ipvsSvcAttrFlags:
svc.Flags = native.Uint32(attr.Value)
s.Flags = native.Uint32(attr.Value)
case ipvsSvcAttrTimeout:
svc.Timeout = native.Uint32(attr.Value)
s.Timeout = native.Uint32(attr.Value)
case ipvsSvcAttrNetmask:
svc.Timeout = native.Uint32(attr.Value)
s.Netmask = native.Uint32(attr.Value)
case ipvsSvcAttrStats:
stats, err := parseStats(attr.Value)
stats, err := assembleStats(attr.Value)
if err != nil {
return nil, err
}
svc.Stats = stats
s.Stats = stats
}
}
return &svc, nil
return &s, nil
}
//parseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise
func (i *Handle) parseService(msg []byte) (*Service, error) {
var s *Service
//Remove General header for this message and parse the NetLink message
hdr := deserializeGenlMsg(msg)
NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():])
if err != nil {
return nil, err
}
if len(NetLinkAttrs) == 0 {
return nil, fmt.Errorf("Error No valid net link message found while Parsing service record")
}
//Now Parse and get IPVS related attributes messages packed in this message.
ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value)
if err != nil {
return nil, err
}
//Assemble all the IPVS related attribute messages and create a service record
s, err = assembleService(ipvsAttrs)
if err != nil {
return nil, err
}
return s, nil
}
//doGetServicesCmd a wrapper which could be used commonly for both GetServices() and GetService(*Service)
func (i *Handle) doGetServicesCmd(svc *Service) ([]*Service, error) {
var res []*Service
msgs, err := i.doCmdwithResponse(svc, nil, ipvsCmdGetService)
if err != nil {
return nil, err
}
for _, msg := range msgs {
srv, err := i.parseService(msg)
if err != nil {
return res, err
}
res = append(res, srv)
}
return res, nil
}
func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) {
@ -374,6 +424,7 @@ func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error)
case ipvsDestAttrPort:
d.Port = binary.BigEndian.Uint16(attr.Value)
case ipvsDestAttrForwardingMethod:
d.ConnectionFlags = native.Uint32(attr.Value)
case ipvsDestAttrWeight:
d.Weight = int(native.Uint16(attr.Value))
case ipvsDestAttrUpperThreshold:
@ -384,42 +435,11 @@ func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error)
d.AddressFamily = native.Uint16(attr.Value)
}
}
return &d, nil
}
//ParseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise
func (i *Handle) ParseService(msg []byte) (*Service, error) {
var svc *Service
//Remove General header for this message and parse the NetLink message
hdr := deserializeGenlMsg(msg)
NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():])
if err != nil {
return nil, err
}
if len(NetLinkAttrs) == 0 {
return nil, fmt.Errorf("Error No valid net link message found while Parsing service record")
}
//Now parse the smaller messages and get a list of attributes to construct a valid service
ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value)
if err != nil {
return nil, err
}
//Assemble netlink attributes and create a service record
svc, err = assembleService(ipvsAttrs)
if err != nil {
return nil, err
}
return svc, nil
}
//ParseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise
func (i *Handle) ParseDestination(msg []byte) (*Destination, error) {
//parseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise
func (i *Handle) parseDestination(msg []byte) (*Destination, error) {
var dst *Destination
//Remove General header for this message
@ -432,7 +452,7 @@ func (i *Handle) ParseDestination(msg []byte) (*Destination, error) {
return nil, fmt.Errorf("Error No valid net link message found while Parsing service record")
}
//Convert rest of the messages to Attribute Array
//Now Parse and get IPVS related attributes messages packed in this message.
ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value)
if err != nil {
return nil, err
@ -447,29 +467,54 @@ func (i *Handle) ParseDestination(msg []byte) (*Destination, error) {
return dst, nil
}
// doGetDestinationsCmd a wrapper function to be used by GetDestinations and GetDestination(d) apis
func (i *Handle) doGetDestinationsCmd(s *Service, d *Destination) ([]*Destination, error) {
var res []*Destination
msgs, err := i.doCmdwithResponse(s, d, ipvsCmdGetDest)
if err != nil {
return nil, err
}
for _, msg := range msgs {
dest, err := i.parseDestination(msg)
if err != nil {
return res, err
}
res = append(res, dest)
}
return res, nil
}
//IPVS related netlink message format explained
//and unpacks these messages
/* EACH NETLINK REQUEST / RESPONSE WILL LOOK LIKE THIS when returned from
/* EACH NETLINK MSG is of the below format, this is what we will receive from execute() api.
If we have multiple netlink objects to process like GetServices() etc., execute() will
supply an array of this below object
NETLINK MSG
|-----------------------------------|
0 1 2 3
|--------|--------|--------|--------|
| CMD ID | VER | RESERVED |
|-----------------------------------|
| MSG LEN | MSG TYPE |
|-----------------------------------|
| |
| |
| []byte IPVS MSG PADDED BY 4 BYTES |
| |
| |
|-----------------------------------|
|--------|--------|--------|--------| -
| CMD ID | VER | RESERVED | |==> General Message Header represented by genlMsgHdr
|-----------------------------------| -
| ATTR LEN | ATTR TYPE | |
|-----------------------------------| |
| | |
| VALUE | |
| []byte Array of IPVS MSG | |==> Attribute Message represented by syscall.NetlinkRouteAttr
| PADDED BY 4 BYTES | |
| | |
|-----------------------------------| -
A response from the IPVS module is usually
Once We strip genlMsgHdr from above NETLINK MSG, we should parse the VALUE.
VALUE will have an array of netlink attributes (syscall.NetlinkRouteAttr) such that each attribute will
represent a "Service" or "Destination" object's field. If we assemble these attributes we can construct
Service or Destination.
IPVS MSG
|-----------------------------------|
0 1 2 3
|--------|--------|--------|--------|