From 000775b918de1110369d33562686f60fd3cbc248 Mon Sep 17 00:00:00 2001 From: dhilipkumars Date: Sat, 20 May 2017 01:48:52 +0530 Subject: [PATCH] Initial Checkin GetServices and GetDestinations Signed-off-by: dhilipkumars --- libnetwork/ipvs/constants.go | 17 +++ libnetwork/ipvs/ipvs.go | 56 +++++++ libnetwork/ipvs/netlink.go | 284 ++++++++++++++++++++++++++++++++++- 3 files changed, 349 insertions(+), 8 deletions(-) diff --git a/libnetwork/ipvs/constants.go b/libnetwork/ipvs/constants.go index 103e71a37c..d36bec0e80 100644 --- a/libnetwork/ipvs/constants.go +++ b/libnetwork/ipvs/constants.go @@ -85,6 +85,23 @@ const ( ipvsDestAttrInactiveConnections ipvsDestAttrPersistentConnections ipvsDestAttrStats + ipvsDestAttrAddressFamily +) + +// IPVS Svc Statistics constancs + +const ( + ipvsSvcStatsUnspec int = iota + ipvsSvcStatsConns + ipvsSvcStatsPktsIn + ipvsSvcStatsPktsOut + ipvsSvcStatsBytesIn + ipvsSvcStatsBytesOut + ipvsSvcStatsCPS + ipvsSvcStatsPPSIn + ipvsSvcStatsPPSOut + ipvsSvcStatsBPSIn + ipvsSvcStatsBPSOut ) // Destination forwarding methods diff --git a/libnetwork/ipvs/ipvs.go b/libnetwork/ipvs/ipvs.go index 266cc24dbe..e442ecb2e9 100644 --- a/libnetwork/ipvs/ipvs.go +++ b/libnetwork/ipvs/ipvs.go @@ -25,6 +25,20 @@ type Service struct { Netmask uint32 AddressFamily uint16 PEName string + Stats SvcStats +} + +type SvcStats struct { + Connections uint32 + PacketsIn uint32 + PacketsOut uint32 + BytesIn uint64 + BytesOut uint64 + CPS uint32 + BPSOut uint32 + PPSIn uint32 + PPSOut uint32 + BPSIn uint32 } // Destination defines an IPVS destination (real server) in its @@ -117,3 +131,45 @@ func (i *Handle) UpdateDestination(s *Service, d *Destination) error { func (i *Handle) DelDestination(s *Service, d *Destination) error { return i.doCmd(s, d, ipvsCmdDelDest) } + +// GetServices returns an array of services configured at the kernel +func (i *Handle) GetServices() ([]*Service, error) { + var res []*Service + //var emptySrv Service + //var emptyDest Destination + + msgs, err := i.doCmdwithResponse(nil, nil, ipvsCmdGetService) + if err != nil { + return nil, err + } + + for _, msg := range msgs { + srv, err := i.ParseService(msg) + if err != nil { + return res, err + } + res = append(res, srv) + } + + return res, nil +} + +// GetDestinations returns an array of Destinations configured for this +func (i *Handle) GetDestinations(s *Service) ([]*Destination, error) { + + var res []*Destination + + msgs, err := i.doCmdwithResponse(s, nil, ipvsCmdGetDest) + if err != nil { + return nil, err + } + + for _, msg := range msgs { + dest, err := i.ParseDestination(msg) + if err != nil { + return res, err + } + res = append(res, dest) + } + return res, nil +} diff --git a/libnetwork/ipvs/netlink.go b/libnetwork/ipvs/netlink.go index 635606dacd..67f2eec3cb 100644 --- a/libnetwork/ipvs/netlink.go +++ b/libnetwork/ipvs/netlink.go @@ -19,6 +19,8 @@ import ( "github.com/vishvananda/netns" ) +//For Quick Reference IPVS related netlink message is described below. + var ( native = nl.NativeEndian() ipvsFamily int @@ -72,6 +74,7 @@ func setup() { func fillService(s *Service) nl.NetlinkRequestData { cmdAttr := nl.NewRtAttr(ipvsCmdAttrService, nil) + nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrAddressFamily, nl.Uint16Attr(s.AddressFamily)) if s.FWMark != 0 { nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFWMark, nl.Uint32Attr(s.FWMark)) @@ -89,12 +92,12 @@ func fillService(s *Service) nl.NetlinkRequestData { if s.PEName != "" { nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrPEName, nl.ZeroTerminated(s.PEName)) } - f := &ipvsFlags{ flags: s.Flags, mask: 0xFFFFFFFF, } nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFlags, f.Serialize()) + nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrTimeout, nl.Uint32Attr(s.Timeout)) nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrNetmask, nl.Uint32Attr(s.Netmask)) return cmdAttr @@ -117,20 +120,38 @@ func fillDestinaton(d *Destination) nl.NetlinkRequestData { return cmdAttr } -func (i *Handle) doCmd(s *Service, d *Destination, cmd uint8) error { +func (i *Handle) doCmdwithResponse(s *Service, d *Destination, cmd uint8) ([][]byte, error) { req := newIPVSRequest(cmd) req.Seq = atomic.AddUint32(&i.seq, 1) - req.AddData(fillService(s)) - if d != nil { + if s == nil { + req.Flags |= syscall.NLM_F_DUMP //Flag to dump all messages + req.AddData(nl.NewRtAttr(ipvsCmdAttrService, nil)) //Add a dummy attribute + } else { + req.AddData(fillService(s)) + + } + + if d == nil { + if cmd == ipvsCmdGetDest { + req.Flags |= syscall.NLM_F_DUMP + } + } else { req.AddData(fillDestinaton(d)) } - if _, err := execute(i.sock, req, 0); err != nil { - return err + res, err := execute(i.sock, req, 0) + if err != nil { + return [][]byte{}, err } - return nil + return res, nil +} + +func (i *Handle) doCmd(s *Service, d *Destination, cmd uint8) error { + _, err := i.doCmdwithResponse(s, d, cmd) + + return err } func getIPVSFamily() (int, error) { @@ -171,7 +192,6 @@ func rawIPData(ip net.IP) []byte { if family == nl.FAMILY_V4 { return ip.To4() } - return ip } @@ -208,6 +228,7 @@ done: return nil, err } for _, m := range msgs { + if m.Header.Seq != req.Seq { continue } @@ -217,6 +238,7 @@ done: if m.Header.Type == syscall.NLMSG_DONE { break done } + if m.Header.Type == syscall.NLMSG_ERROR { error := int32(native.Uint32(m.Data[0:4])) if error == 0 { @@ -235,3 +257,249 @@ done: } return res, nil } + +func parseIP(ip []byte, family uint16) (net.IP, error) { + + var resIP net.IP + + switch family { + case syscall.AF_INET: + resIP = (net.IP)(ip[:4]) + case syscall.AF_INET6: + resIP = (net.IP)(ip[:16]) + default: + return resIP, fmt.Errorf("parseIP Error ip=%v", ip) + + } + return resIP, nil +} + +func parseStats(msg []byte) (SvcStats, error) { + + var s SvcStats + + attrs, err := nl.ParseRouteAttr(msg) + if err != nil { + return s, err + } + + for _, attr := range attrs { + attrType := int(attr.Attr.Type) + switch attrType { + case ipvsSvcStatsConns: + s.Connections = native.Uint32(attr.Value) + case ipvsSvcStatsPktsIn: + s.PacketsIn = native.Uint32(attr.Value) + case ipvsSvcStatsPktsOut: + s.PacketsOut = native.Uint32(attr.Value) + case ipvsSvcStatsBytesIn: + s.BytesIn = native.Uint64(attr.Value) + case ipvsSvcStatsBytesOut: + s.BytesOut = native.Uint64(attr.Value) + case ipvsSvcStatsCPS: + s.CPS = native.Uint32(attr.Value) + case ipvsSvcStatsPPSIn: + s.PPSIn = native.Uint32(attr.Value) + case ipvsSvcStatsPPSOut: + s.PPSOut = native.Uint32(attr.Value) + case ipvsSvcStatsBPSIn: + s.BPSIn = native.Uint32(attr.Value) + case ipvsSvcStatsBPSOut: + s.BPSOut = native.Uint32(attr.Value) + } + } + return s, nil +} + +//assembleService assembles a services back from a hain of netlink attributes +func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) { + + var svc Service + + for _, attr := range attrs { + + attrType := int(attr.Attr.Type) + + switch attrType { + + case ipvsSvcAttrAddressFamily: + svc.AddressFamily = native.Uint16(attr.Value) + case ipvsSvcAttrProtocol: + svc.Protocol = native.Uint16(attr.Value) + case ipvsSvcAttrAddress: + ip, err := parseIP(attr.Value, svc.AddressFamily) + if err != nil { + return nil, err + } + svc.Address = ip + case ipvsSvcAttrPort: + svc.Port = binary.BigEndian.Uint16(attr.Value) + case ipvsSvcAttrFWMark: + svc.FWMark = native.Uint32(attr.Value) + case ipvsSvcAttrSchedName: + svc.SchedName = string(attr.Value) + case ipvsSvcAttrFlags: + svc.Flags = native.Uint32(attr.Value) + case ipvsSvcAttrTimeout: + svc.Timeout = native.Uint32(attr.Value) + case ipvsSvcAttrNetmask: + svc.Timeout = native.Uint32(attr.Value) + case ipvsSvcAttrStats: + stats, err := parseStats(attr.Value) + if err != nil { + return nil, err + } + svc.Stats = stats + } + + } + return &svc, nil +} + +func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) { + + var d Destination + + for _, attr := range attrs { + + attrType := int(attr.Attr.Type) + + switch attrType { + case ipvsDestAttrAddress: + ip, err := parseIP(attr.Value, syscall.AF_INET) + if err != nil { + return nil, err + } + d.Address = ip + case ipvsDestAttrPort: + d.Port = binary.BigEndian.Uint16(attr.Value) + case ipvsDestAttrForwardingMethod: + case ipvsDestAttrWeight: + d.Weight = int(native.Uint16(attr.Value)) + case ipvsDestAttrUpperThreshold: + d.UpperThreshold = native.Uint32(attr.Value) + case ipvsDestAttrLowerThreshold: + d.LowerThreshold = native.Uint32(attr.Value) + case ipvsDestAttrAddressFamily: + d.AddressFamily = native.Uint16(attr.Value) + } + } + + return &d, nil +} + +//ParseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise +func (i *Handle) ParseService(msg []byte) (*Service, error) { + + var svc *Service + + //Remove General header for this message and parse the NetLink message + hdr := deserializeGenlMsg(msg) + NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) + if err != nil { + return nil, err + } + if len(NetLinkAttrs) == 0 { + return nil, fmt.Errorf("Error No valid net link message found while Parsing service record") + } + + //Now parse the smaller messages and get a list of attributes to construct a valid service + ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) + if err != nil { + return nil, err + } + + //Assemble netlink attributes and create a service record + svc, err = assembleService(ipvsAttrs) + if err != nil { + return nil, err + } + + return svc, nil +} + +//ParseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise +func (i *Handle) ParseDestination(msg []byte) (*Destination, error) { + var dst *Destination + + //Remove General header for this message + hdr := deserializeGenlMsg(msg) + NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) + if err != nil { + return nil, err + } + if len(NetLinkAttrs) == 0 { + return nil, fmt.Errorf("Error No valid net link message found while Parsing service record") + } + + //Convert rest of the messages to Attribute Array + ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) + if err != nil { + return nil, err + } + + //Assemble netlink attributes and create a Destination record + dst, err = assembleDestination(ipvsAttrs) + if err != nil { + return nil, err + } + + return dst, nil +} + +//IPVS related netlink message format explained +//and unpacks these messages + +/* EACH NETLINK REQUEST / RESPONSE WILL LOOK LIKE THIS when returned from + +|-----------------------------------| + 0 1 2 3 +|--------|--------|--------|--------| +| CMD ID | VER | RESERVED | +|-----------------------------------| +| MSG LEN | MSG TYPE | +|-----------------------------------| +| | +| | +| []byte IPVS MSG PADDED BY 4 BYTES | +| | +| | +|-----------------------------------| + + +A response from the IPVS module is usually + + +|-----------------------------------| + 0 1 2 3 +|--------|--------|--------|--------| +| ATTR LEN | ATTR TYPE | +|-----------------------------------| +| | +| | +| []byte IPVS ATTRIBUTE BY 4 BYTES | +| | +| | +|-----------------------------------| + NEXT ATTRIBUTE +|-----------------------------------| +| ATTR LEN | ATTR TYPE | +|-----------------------------------| +| | +| | +| []byte IPVS ATTRIBUTE BY 4 BYTES | +| | +| | +|-----------------------------------| + NEXT ATTRIBUTE +|-----------------------------------| +| ATTR LEN | ATTR TYPE | +|-----------------------------------| +| | +| | +| []byte IPVS ATTRIBUTE BY 4 BYTES | +| | +| | +|-----------------------------------| + +*/