From 7fc1795cdf844c9f6769ecb8bbd84eea1397860e Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 18 May 2018 14:10:14 -0700 Subject: [PATCH] Allows to set generic knobs on the Sandbox Refactor the ostweaks file to allows a more easy reuse Add a method on the osl.Sandbox interface to allow setting knobs on the sandbox Signed-off-by: Flavio Crisciani --- libnetwork/cmd/proxy/network_proxy_test.go | 5 +- libnetwork/controller.go | 5 ++ libnetwork/drivers/overlay/ostweaks_linux.go | 55 ++------------------ libnetwork/drvregistry/drvregistry_test.go | 7 ++- libnetwork/network.go | 3 +- libnetwork/networkdb/networkdb_test.go | 9 ++-- libnetwork/osl/kernel/knobs.go | 16 ++++++ libnetwork/osl/kernel/knobs_linux.go | 47 +++++++++++++++++ libnetwork/osl/kernel/knobs_linux_test.go | 33 ++++++++++++ libnetwork/osl/kernel/knobs_unsupported.go | 7 +++ libnetwork/osl/namespace_linux.go | 30 ++++++++--- libnetwork/osl/sandbox.go | 13 +++++ libnetwork/sandbox.go | 10 ++++ libnetwork/types/types_test.go | 8 +-- 14 files changed, 172 insertions(+), 76 deletions(-) create mode 100644 libnetwork/osl/kernel/knobs.go create mode 100644 libnetwork/osl/kernel/knobs_linux.go create mode 100644 libnetwork/osl/kernel/knobs_linux_test.go create mode 100644 libnetwork/osl/kernel/knobs_unsupported.go diff --git a/libnetwork/cmd/proxy/network_proxy_test.go b/libnetwork/cmd/proxy/network_proxy_test.go index 38b5dce69a..2d226504b6 100644 --- a/libnetwork/cmd/proxy/network_proxy_test.go +++ b/libnetwork/cmd/proxy/network_proxy_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "flag" "fmt" "io" "io/ioutil" @@ -12,10 +11,10 @@ import ( "time" "github.com/ishidawataru/sctp" + // this takes care of the incontainer flag + _ "github.com/docker/libnetwork/testutils" ) -var _ = flag.Bool("incontainer", false, "Indicates if the test is running in a container") - var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo") var testBufSize = len(testBuf) diff --git a/libnetwork/controller.go b/libnetwork/controller.go index b6c536629a..f3b6b02aba 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -1144,6 +1144,11 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S } } + if sb.osSbox != nil { + // Apply operating specific knobs on the load balancer sandbox + sb.osSbox.ApplyOSTweaks(sb.oslTypes) + } + c.Lock() c.sandboxes[sb.id] = sb c.Unlock() diff --git a/libnetwork/drivers/overlay/ostweaks_linux.go b/libnetwork/drivers/overlay/ostweaks_linux.go index 68f1ee9cee..de946a5846 100644 --- a/libnetwork/drivers/overlay/ostweaks_linux.go +++ b/libnetwork/drivers/overlay/ostweaks_linux.go @@ -1,72 +1,23 @@ package overlay import ( - "io/ioutil" - "path" "strconv" - "strings" - "github.com/sirupsen/logrus" + "github.com/docker/libnetwork/osl/kernel" ) -type conditionalCheck func(val1, val2 string) bool - -type osValue struct { - value string - checkFn conditionalCheck -} - -var osConfig = map[string]osValue{ +var ovConfig = map[string]*kernel.OSValue{ "net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher}, "net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher}, "net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher}, } -func propertyIsValid(val1, val2 string, check conditionalCheck) bool { - if check == nil || check(val1, val2) { - return true - } - return false -} - func checkHigher(val1, val2 string) bool { val1Int, _ := strconv.ParseInt(val1, 10, 32) val2Int, _ := strconv.ParseInt(val2, 10, 32) return val1Int < val2Int } -// writeSystemProperty writes the value to a path under /proc/sys as determined from the key. -// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward. -func writeSystemProperty(key, value string) error { - keyPath := strings.Replace(key, ".", "/", -1) - return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644) -} - -func readSystemProperty(key string) (string, error) { - keyPath := strings.Replace(key, ".", "/", -1) - value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath)) - if err != nil { - return "", err - } - return string(value), nil -} - func applyOStweaks() { - for k, v := range osConfig { - // read the existing property from disk - oldv, err := readSystemProperty(k) - if err != nil { - logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err) - continue - } - - if propertyIsValid(oldv, v.value, v.checkFn) { - // write new prop value to disk - if err := writeSystemProperty(k, v.value); err != nil { - logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err) - continue - } - logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv) - } - } + kernel.ApplyOSTweaks(ovConfig) } diff --git a/libnetwork/drvregistry/drvregistry_test.go b/libnetwork/drvregistry/drvregistry_test.go index 742bee788f..2251e03302 100644 --- a/libnetwork/drvregistry/drvregistry_test.go +++ b/libnetwork/drvregistry/drvregistry_test.go @@ -1,7 +1,6 @@ package drvregistry import ( - "flag" "sort" "testing" @@ -13,10 +12,10 @@ import ( nullIpam "github.com/docker/libnetwork/ipams/null" remoteIpam "github.com/docker/libnetwork/ipams/remote" "github.com/stretchr/testify/assert" -) -var runningInContainer = flag.Bool("incontainer", false, - "Indicates if the test is running in a container") + // this takes care of the incontainer flag + _ "github.com/docker/libnetwork/testutils" +) const mockDriverName = "mock-driver" diff --git a/libnetwork/network.go b/libnetwork/network.go index 6136611276..ab7b887db0 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -2125,7 +2125,8 @@ func (n *network) lbEndpointName() string { func (n *network) createLoadBalancerSandbox() (retErr error) { sandboxName := n.lbSandboxName() - sbOptions := []SandboxOption{} + // Mark the sandbox to be a load balancer + sbOptions := []SandboxOption{OptionLoadBalancer()} if n.ingress { sbOptions = append(sbOptions, OptionIngress()) } diff --git a/libnetwork/networkdb/networkdb_test.go b/libnetwork/networkdb/networkdb_test.go index fda6b71a65..5140a48771 100644 --- a/libnetwork/networkdb/networkdb_test.go +++ b/libnetwork/networkdb/networkdb_test.go @@ -1,7 +1,6 @@ package networkdb import ( - "flag" "fmt" "io/ioutil" "log" @@ -18,12 +17,12 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + // this takes care of the incontainer flag + _ "github.com/docker/libnetwork/testutils" ) -var ( - dbPort int32 = 10000 - runningInContainer = flag.Bool("incontainer", false, "Indicates if the test is running in a container") -) +var dbPort int32 = 10000 func TestMain(m *testing.M) { ioutil.WriteFile("/proc/sys/net/ipv6/conf/lo/disable_ipv6", []byte{'0', '\n'}, 0644) diff --git a/libnetwork/osl/kernel/knobs.go b/libnetwork/osl/kernel/knobs.go new file mode 100644 index 0000000000..5088f0e7f9 --- /dev/null +++ b/libnetwork/osl/kernel/knobs.go @@ -0,0 +1,16 @@ +package kernel + +type conditionalCheck func(val1, val2 string) bool + +// OSValue represents a tuple, value defired, check function when to apply the value +type OSValue struct { + Value string + CheckFn conditionalCheck +} + +func propertyIsValid(val1, val2 string, check conditionalCheck) bool { + if check == nil || check(val1, val2) { + return true + } + return false +} diff --git a/libnetwork/osl/kernel/knobs_linux.go b/libnetwork/osl/kernel/knobs_linux.go new file mode 100644 index 0000000000..964280650c --- /dev/null +++ b/libnetwork/osl/kernel/knobs_linux.go @@ -0,0 +1,47 @@ +package kernel + +import ( + "io/ioutil" + "path" + "strings" + + "github.com/sirupsen/logrus" +) + +// writeSystemProperty writes the value to a path under /proc/sys as determined from the key. +// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward. +func writeSystemProperty(key, value string) error { + keyPath := strings.Replace(key, ".", "/", -1) + return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644) +} + +// readSystemProperty reads the value from the path under /proc/sys and returns it +func readSystemProperty(key string) (string, error) { + keyPath := strings.Replace(key, ".", "/", -1) + value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath)) + if err != nil { + return "", err + } + return strings.TrimSpace(string(value)), nil +} + +// ApplyOSTweaks applies the configuration values passed as arguments +func ApplyOSTweaks(osConfig map[string]*OSValue) { + for k, v := range osConfig { + // read the existing property from disk + oldv, err := readSystemProperty(k) + if err != nil { + logrus.WithError(err).Errorf("error reading the kernel parameter %s", k) + continue + } + + if propertyIsValid(oldv, v.Value, v.CheckFn) { + // write new prop value to disk + if err := writeSystemProperty(k, v.Value); err != nil { + logrus.WithError(err).Errorf("error setting the kernel parameter %s = %s, (leaving as %s)", k, v.Value, oldv) + continue + } + logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.Value, oldv) + } + } +} diff --git a/libnetwork/osl/kernel/knobs_linux_test.go b/libnetwork/osl/kernel/knobs_linux_test.go new file mode 100644 index 0000000000..cfc7b6cbb3 --- /dev/null +++ b/libnetwork/osl/kernel/knobs_linux_test.go @@ -0,0 +1,33 @@ +package kernel + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + _ "github.com/docker/libnetwork/testutils" +) + +func TestReadWriteKnobs(t *testing.T) { + for _, k := range []string{ + "net.ipv4.neigh.default.gc_thresh1", + "net.ipv4.neigh.default.gc_thresh2", + "net.ipv4.neigh.default.gc_thresh3", + } { + // Check if the test is able to read the value + v, err := readSystemProperty(k) + if err != nil { + logrus.WithError(err).Warnf("Path %v not readable", k) + // the path is not there, skip this key + continue + } + // Test the write + assert.NoError(t, writeSystemProperty(k, "10000")) + newV, err := readSystemProperty(k) + assert.NoError(t, err) + assert.Equal(t, newV, "10000") + // Restore value + assert.NoError(t, writeSystemProperty(k, v)) + } +} diff --git a/libnetwork/osl/kernel/knobs_unsupported.go b/libnetwork/osl/kernel/knobs_unsupported.go new file mode 100644 index 0000000000..32f258f416 --- /dev/null +++ b/libnetwork/osl/kernel/knobs_unsupported.go @@ -0,0 +1,7 @@ +// +build !linux + +package kernel + +// ApplyOSTweaks applies the configuration values passed as arguments +func ApplyOSTweaks(osConfig map[string]*OSValue) { +} diff --git a/libnetwork/osl/namespace_linux.go b/libnetwork/osl/namespace_linux.go index a55932babe..996a250981 100644 --- a/libnetwork/osl/namespace_linux.go +++ b/libnetwork/osl/namespace_linux.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/ns" + "github.com/docker/libnetwork/osl/kernel" "github.com/docker/libnetwork/types" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" @@ -29,13 +30,18 @@ func init() { } var ( - once sync.Once - garbagePathMap = make(map[string]bool) - gpmLock sync.Mutex - gpmWg sync.WaitGroup - gpmCleanupPeriod = 60 * time.Second - gpmChan = make(chan chan struct{}) - prefix = defaultPrefix + once sync.Once + garbagePathMap = make(map[string]bool) + gpmLock sync.Mutex + gpmWg sync.WaitGroup + gpmCleanupPeriod = 60 * time.Second + gpmChan = make(chan chan struct{}) + prefix = defaultPrefix + loadBalancerConfig = map[string]*kernel.OSValue{ + // expires connection from the IPVS connection table when the backend is not available + // more info: https://github.com/torvalds/linux/blob/master/Documentation/networking/ipvs-sysctl.txt#L126:1 + "net.ipv4.vs.expire_nodest_conn": {"1", nil}, + } ) // The networkNamespace type is the linux implementation of the Sandbox @@ -630,3 +636,13 @@ func setIPv6(path, iface string, enable bool) error { } return nil } + +// ApplyOSTweaks applies linux configs on the sandbox +func (n *networkNamespace) ApplyOSTweaks(types []SandboxType) { + for _, t := range types { + switch t { + case SandboxTypeLoadBalancer: + kernel.ApplyOSTweaks(loadBalancerConfig) + } + } +} diff --git a/libnetwork/osl/sandbox.go b/libnetwork/osl/sandbox.go index 06149062fb..5019e068df 100644 --- a/libnetwork/osl/sandbox.go +++ b/libnetwork/osl/sandbox.go @@ -7,6 +7,16 @@ import ( "github.com/docker/libnetwork/types" ) +// SandboxType specify the time of the sandbox, this can be used to apply special configs +type SandboxType int + +const ( + // SandboxTypeIngress indicates that the sandbox is for the ingress + SandboxTypeIngress = iota + // SandboxTypeLoadBalancer indicates that the sandbox is a load balancer + SandboxTypeLoadBalancer = iota +) + // Sandbox represents a network sandbox, identified by a specific key. It // holds a list of Interfaces, routes etc, and more can be added dynamically. type Sandbox interface { @@ -70,6 +80,9 @@ type Sandbox interface { // restore sandbox Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error + + // ApplyOSTweaks applies operating system specific knobs on the sandbox + ApplyOSTweaks([]SandboxType) } // NeighborOptionSetter interface defines the option setter methods for interface options diff --git a/libnetwork/sandbox.go b/libnetwork/sandbox.go index abea13c076..0f9a53a367 100644 --- a/libnetwork/sandbox.go +++ b/libnetwork/sandbox.go @@ -83,6 +83,7 @@ type sandbox struct { inDelete bool ingress bool ndotsSet bool + oslTypes []osl.SandboxType // slice of properties of this sandbox sync.Mutex // This mutex is used to serialize service related operation for an endpoint // The lock is here because the endpoint is saved into the store so is not unique @@ -1162,6 +1163,15 @@ func OptionPortMapping(portBindings []types.PortBinding) SandboxOption { func OptionIngress() SandboxOption { return func(sb *sandbox) { sb.ingress = true + sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress) + } +} + +// OptionLoadBalancer function returns an option setter for marking a +// sandbox as a load balancer sandbox. +func OptionLoadBalancer() SandboxOption { + return func(sb *sandbox) { + sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer) } } diff --git a/libnetwork/types/types_test.go b/libnetwork/types/types_test.go index b487099079..29a283ed73 100644 --- a/libnetwork/types/types_test.go +++ b/libnetwork/types/types_test.go @@ -1,13 +1,13 @@ package types import ( - "flag" - "github.com/stretchr/testify/require" "net" "testing" -) -var runningInContainer = flag.Bool("incontainer", false, "Indicates if the test is running in a container") + "github.com/stretchr/testify/require" + + _ "github.com/docker/libnetwork/testutils" +) func TestTransportPortConv(t *testing.T) { sform := "tcp/23"