1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Merge pull request #37558 from selansen/master

Global Default Address Pool feature support
This commit is contained in:
Tibor Vass 2018-08-20 18:15:44 -07:00 committed by GitHub
commit 1800883bd1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 994 additions and 318 deletions

View file

@ -8768,14 +8768,28 @@ paths:
nodes in order to reach the containers running on this node. Using this parameter it is possible to nodes in order to reach the containers running on this node. Using this parameter it is possible to
separate the container data traffic from the management traffic of the cluster. separate the container data traffic from the management traffic of the cluster.
type: "string" type: "string"
DefaultAddrPool:
description: |
Default Address Pool specifies default subnet pools for global scope networks.
type: "array"
items:
type: "string"
example: ["10.10.0.0/16", "20.20.0.0/16"]
ForceNewCluster: ForceNewCluster:
description: "Force creation of a new swarm." description: "Force creation of a new swarm."
type: "boolean" type: "boolean"
SubnetSize:
description: |
SubnetSize specifies the subnet size of the networks created from the default subnet pool
type: "integer"
format: "uint32"
Spec: Spec:
$ref: "#/definitions/SwarmSpec" $ref: "#/definitions/SwarmSpec"
example: example:
ListenAddr: "0.0.0.0:2377" ListenAddr: "0.0.0.0:2377"
AdvertiseAddr: "192.168.1.1:2377" AdvertiseAddr: "192.168.1.1:2377"
DefaultAddrPool: ["10.10.0.0/8", "20.20.0.0/8"]
SubnetSize: 24
ForceNewCluster: false ForceNewCluster: false
Spec: Spec:
Orchestration: {} Orchestration: {}

View file

@ -1,6 +1,8 @@
package swarm // import "github.com/docker/docker/api/types/swarm" package swarm // import "github.com/docker/docker/api/types/swarm"
import "time" import (
"time"
)
// ClusterInfo represents info about the cluster for outputting in "info" // ClusterInfo represents info about the cluster for outputting in "info"
// it contains the same information as "Swarm", but without the JoinTokens // it contains the same information as "Swarm", but without the JoinTokens
@ -10,6 +12,8 @@ type ClusterInfo struct {
Spec Spec Spec Spec
TLSInfo TLSInfo TLSInfo TLSInfo
RootRotationInProgress bool RootRotationInProgress bool
DefaultAddrPool []string
SubnetSize uint32
} }
// Swarm represents a swarm. // Swarm represents a swarm.
@ -153,6 +157,8 @@ type InitRequest struct {
Spec Spec Spec Spec
AutoLockManagers bool AutoLockManagers bool
Availability NodeAvailability Availability NodeAvailability
DefaultAddrPool []string
SubnetSize uint32
} }
// JoinRequest is the request used to join a swarm. // JoinRequest is the request used to join a swarm.

View file

@ -40,6 +40,8 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm {
TrustRoot: string(c.RootCA.CACert), TrustRoot: string(c.RootCA.CACert),
}, },
RootRotationInProgress: c.RootCA.RootRotation != nil, RootRotationInProgress: c.RootCA.RootRotation != nil,
DefaultAddrPool: c.DefaultAddressPool,
SubnetSize: c.SubnetSize,
}, },
JoinTokens: types.JoinTokens{ JoinTokens: types.JoinTokens{
Worker: c.RootCA.JoinTokens.Worker, Worker: c.RootCA.JoinTokens.Worker,

View file

@ -3,6 +3,7 @@ package cluster // import "github.com/docker/docker/daemon/cluster"
import ( import (
"context" "context"
"fmt" "fmt"
"net"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings" "strings"
@ -52,6 +53,10 @@ type nodeStartConfig struct {
AdvertiseAddr string AdvertiseAddr string
// DataPathAddr is the address that has to be used for the data path // DataPathAddr is the address that has to be used for the data path
DataPathAddr string DataPathAddr string
// DefaultAddressPool contains list of subnets
DefaultAddressPool []string
// SubnetSize contains subnet size of DefaultAddressPool
SubnetSize uint32
// JoinInProgress is set to true if a join operation has started, but // JoinInProgress is set to true if a join operation has started, but
// not completed yet. // not completed yet.
JoinInProgress bool JoinInProgress bool
@ -110,6 +115,12 @@ func (n *nodeRunner) start(conf nodeStartConfig) error {
joinAddr = conf.RemoteAddr joinAddr = conf.RemoteAddr
} }
var defaultAddrPool []*net.IPNet
for _, address := range conf.DefaultAddressPool {
if _, b, err := net.ParseCIDR(address); err == nil {
defaultAddrPool = append(defaultAddrPool, b)
}
}
// Hostname is not set here. Instead, it is obtained from // Hostname is not set here. Instead, it is obtained from
// the node description that is reported periodically // the node description that is reported periodically
swarmnodeConfig := swarmnode.Config{ swarmnodeConfig := swarmnode.Config{
@ -117,6 +128,8 @@ func (n *nodeRunner) start(conf nodeStartConfig) error {
ListenControlAPI: control, ListenControlAPI: control,
ListenRemoteAPI: conf.ListenAddr, ListenRemoteAPI: conf.ListenAddr,
AdvertiseRemoteAPI: conf.AdvertiseAddr, AdvertiseRemoteAPI: conf.AdvertiseAddr,
DefaultAddrPool: defaultAddrPool,
SubnetSize: int(conf.SubnetSize),
JoinAddr: joinAddr, JoinAddr: joinAddr,
StateDir: n.cluster.root, StateDir: n.cluster.root,
JoinToken: conf.joinToken, JoinToken: conf.joinToken,

View file

@ -93,13 +93,15 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
} }
nr, err := c.newNodeRunner(nodeStartConfig{ nr, err := c.newNodeRunner(nodeStartConfig{
forceNewCluster: req.ForceNewCluster, forceNewCluster: req.ForceNewCluster,
autolock: req.AutoLockManagers, autolock: req.AutoLockManagers,
LocalAddr: localAddr, LocalAddr: localAddr,
ListenAddr: net.JoinHostPort(listenHost, listenPort), ListenAddr: net.JoinHostPort(listenHost, listenPort),
AdvertiseAddr: net.JoinHostPort(advertiseHost, advertisePort), AdvertiseAddr: net.JoinHostPort(advertiseHost, advertisePort),
DataPathAddr: dataPathAddr, DataPathAddr: dataPathAddr,
availability: req.Availability, DefaultAddressPool: req.DefaultAddrPool,
SubnetSize: req.SubnetSize,
availability: req.Availability,
}) })
if err != nil { if err != nil {
return "", err return "", err

View file

@ -21,6 +21,9 @@ keywords: "API, Docker, rcli, REST, documentation"
and `OperatingSystem` if the daemon was unable to obtain this information. and `OperatingSystem` if the daemon was unable to obtain this information.
* `GET /info` now returns information about the product license, if a license * `GET /info` now returns information about the product license, if a license
has been applied to the daemon. has been applied to the daemon.
* `POST /swarm/init` now accepts a `DefaultAddrPool` property to set global scope default address pool
* `POST /swarm/init` now accepts a `SubnetSize` property to set global scope networks by giving the
length of the subnet masks for every such network
## V1.38 API changes ## V1.38 API changes

View file

@ -313,3 +313,50 @@ func noServices(client client.ServiceAPIClient) func(log poll.LogT) poll.Result
} }
} }
} }
func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
defer setupTest(t)()
var ops = []func(*daemon.Daemon){}
ipAddr := []string{"20.20.0.0/16"}
ops = append(ops, daemon.WithSwarmDefaultAddrPool(ipAddr))
ops = append(ops, daemon.WithSwarmDefaultAddrPoolSubnetSize(24))
d := swarm.NewSwarm(t, testEnv, ops...)
cli := d.NewClientT(t)
defer cli.Close()
// Create a overlay network
name := "saanvisthira" + t.Name()
network.CreateNoError(t, context.Background(), cli, name,
network.WithDriver("overlay"))
var instances uint64 = 1
serviceName := "TestService" + t.Name()
serviceID := swarm.CreateService(t, d,
swarm.ServiceWithReplicas(instances),
swarm.ServiceWithName(serviceName),
swarm.ServiceWithNetwork(name),
)
poll.WaitOn(t, serviceRunningCount(cli, serviceID, instances), swarm.ServicePoll)
_, _, err := cli.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
assert.NilError(t, err)
out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
assert.NilError(t, err)
assert.Equal(t, out.IPAM.Config[0].Subnet, "20.20.0.0/24")
err = cli.ServiceRemove(context.Background(), serviceID)
assert.NilError(t, err)
d.SwarmLeave(true)
d.Stop(t)
// Clean up , set it back to original one to make sure other tests don't fail
ipAddr = []string{"10.10.0.0/8"}
ops = append(ops, daemon.WithSwarmDefaultAddrPool(ipAddr))
ops = append(ops, daemon.WithSwarmDefaultAddrPoolSubnetSize(24))
d = swarm.NewSwarm(t, testEnv, ops...)
d.SwarmLeave(true)
defer d.Stop(t)
}

View file

@ -73,7 +73,8 @@ type Daemon struct {
// swarm related field // swarm related field
swarmListenAddr string swarmListenAddr string
SwarmPort int // FIXME(vdemeester) should probably not be exported SwarmPort int // FIXME(vdemeester) should probably not be exported
DefaultAddrPool []string
SubnetSize uint32
// cached information // cached information
CachedInfo types.Info CachedInfo types.Info
} }

View file

@ -34,6 +34,20 @@ func WithSwarmListenAddr(listenAddr string) func(*Daemon) {
} }
} }
// WithSwarmDefaultAddrPool sets the swarm default address pool to use for swarm mode
func WithSwarmDefaultAddrPool(defaultAddrPool []string) func(*Daemon) {
return func(d *Daemon) {
d.DefaultAddrPool = defaultAddrPool
}
}
// WithSwarmDefaultAddrPoolSubnetSize sets the subnet length mask of swarm default address pool to use for swarm mode
func WithSwarmDefaultAddrPoolSubnetSize(subnetSize uint32) func(*Daemon) {
return func(d *Daemon) {
d.SubnetSize = subnetSize
}
}
// WithEnvironment sets options from internal/test/environment.Execution struct // WithEnvironment sets options from internal/test/environment.Execution struct
func WithEnvironment(e environment.Execution) func(*Daemon) { func WithEnvironment(e environment.Execution) func(*Daemon) {
return func(d *Daemon) { return func(d *Daemon) {

View file

@ -69,6 +69,10 @@ func (d *Daemon) SwarmInit(t assert.TestingT, req swarm.InitRequest) {
if req.ListenAddr == "" { if req.ListenAddr == "" {
req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort)
} }
if req.DefaultAddrPool == nil {
req.DefaultAddrPool = d.DefaultAddrPool
req.SubnetSize = d.SubnetSize
}
cli := d.NewClientT(t) cli := d.NewClientT(t)
defer cli.Close() defer cli.Close()
_, err := cli.SwarmInit(context.Background(), req) _, err := cli.SwarmInit(context.Background(), req)

View file

@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b
#get libnetwork packages #get libnetwork packages
# When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
github.com/docker/libnetwork f30a35b091cc2a431ef9856c75c343f75bb5f2e2 github.com/docker/libnetwork a79d3687931697244b8e03485bf7b2042f8ec6b6
github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@ -125,7 +125,7 @@ github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d
github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef
# cluster # cluster
github.com/docker/swarmkit 8852e8840e30d69db0b39a4a3d6447362e17c64f github.com/docker/swarmkit cfa742c8abe6f8e922f6e4e920153c408e7d9c3b
github.com/gogo/protobuf v1.0.0 github.com/gogo/protobuf v1.0.0
github.com/cloudflare/cfssl 1.3.2 github.com/cloudflare/cfssl 1.3.2
github.com/fernet/fernet-go 1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2 github.com/fernet/fernet-go 1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2

View file

@ -45,9 +45,10 @@ func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) {
a := &Allocator{} a := &Allocator{}
// Load predefined subnet pools // Load predefined subnet pools
a.predefined = map[string][]*net.IPNet{ a.predefined = map[string][]*net.IPNet{
localAddressSpace: ipamutils.PredefinedBroadNetworks, localAddressSpace: ipamutils.GetLocalScopeDefaultNetworks(),
globalAddressSpace: ipamutils.PredefinedGranularNetworks, globalAddressSpace: ipamutils.GetGlobalScopeDefaultNetworks(),
} }
// Initialize asIndices map // Initialize asIndices map

View file

@ -35,7 +35,7 @@ func Init(ic ipamapi.Callback, l, g interface{}) error {
} }
} }
ipamutils.InitNetworks(GetDefaultIPAddressPool()) ipamutils.ConfigLocalScopeDefaultNetworks(GetDefaultIPAddressPool())
a, err := ipam.NewAllocator(localDs, globalDs) a, err := ipam.NewAllocator(localDs, globalDs)
if err != nil { if err != nil {

View file

@ -37,7 +37,7 @@ func InitDockerDefault(ic ipamapi.Callback, l, g interface{}) error {
} }
} }
ipamutils.InitNetworks(nil) ipamutils.ConfigLocalScopeDefaultNetworks(nil)
a, err := ipam.NewAllocator(localDs, globalDs) a, err := ipam.NewAllocator(localDs, globalDs)
if err != nil { if err != nil {

View file

@ -5,23 +5,20 @@ import (
"fmt" "fmt"
"net" "net"
"sync" "sync"
"github.com/sirupsen/logrus"
) )
var ( var (
// PredefinedBroadNetworks contains a list of 31 IPv4 private networks with host size 16 and 12 // PredefinedLocalScopeDefaultNetworks contains a list of 31 IPv4 private networks with host size 16 and 12
// (172.17-31.x.x/16, 192.168.x.x/20) which do not overlap with the networks in `PredefinedGranularNetworks` // (172.17-31.x.x/16, 192.168.x.x/20) which do not overlap with the networks in `PredefinedGlobalScopeDefaultNetworks`
PredefinedBroadNetworks []*net.IPNet PredefinedLocalScopeDefaultNetworks []*net.IPNet
// PredefinedGranularNetworks contains a list of 64K IPv4 private networks with host size 8 // PredefinedGlobalScopeDefaultNetworks contains a list of 64K IPv4 private networks with host size 8
// (10.x.x.x/24) which do not overlap with the networks in `PredefinedBroadNetworks` // (10.x.x.x/24) which do not overlap with the networks in `PredefinedLocalScopeDefaultNetworks`
PredefinedGranularNetworks []*net.IPNet PredefinedGlobalScopeDefaultNetworks []*net.IPNet
initNetworksOnce sync.Once mutex sync.Mutex
localScopeDefaultNetworks = []*NetworkToSplit{{"172.17.0.0/16", 16}, {"172.18.0.0/16", 16}, {"172.19.0.0/16", 16},
defaultBroadNetwork = []*NetworkToSplit{{"172.17.0.0/16", 16}, {"172.18.0.0/16", 16}, {"172.19.0.0/16", 16},
{"172.20.0.0/14", 16}, {"172.24.0.0/14", 16}, {"172.28.0.0/14", 16}, {"172.20.0.0/14", 16}, {"172.24.0.0/14", 16}, {"172.28.0.0/14", 16},
{"192.168.0.0/16", 20}} {"192.168.0.0/16", 20}}
defaultGranularNetwork = []*NetworkToSplit{{"10.0.0.0/8", 24}} globalScopeDefaultNetworks = []*NetworkToSplit{{"10.0.0.0/8", 24}}
) )
// NetworkToSplit represent a network that has to be split in chunks with mask length Size. // NetworkToSplit represent a network that has to be split in chunks with mask length Size.
@ -34,19 +31,61 @@ type NetworkToSplit struct {
Size int `json:"size"` Size int `json:"size"`
} }
// InitNetworks initializes the broad network pool and the granular network pool func init() {
func InitNetworks(defaultAddressPool []*NetworkToSplit) { var err error
initNetworksOnce.Do(func() { if PredefinedGlobalScopeDefaultNetworks, err = splitNetworks(globalScopeDefaultNetworks); err != nil {
// error ingnored should never fail //we are going to panic in case of error as we should never get into this state
PredefinedGranularNetworks, _ = splitNetworks(defaultGranularNetwork) panic("InitAddressPools failed to initialize the global scope default address pool")
if defaultAddressPool == nil { }
defaultAddressPool = defaultBroadNetwork
} if PredefinedLocalScopeDefaultNetworks, err = splitNetworks(localScopeDefaultNetworks); err != nil {
var err error //we are going to panic in case of error as we should never get into this state
if PredefinedBroadNetworks, err = splitNetworks(defaultAddressPool); err != nil { panic("InitAddressPools failed to initialize the local scope default address pool")
logrus.WithError(err).Error("InitAddressPools failed to initialize the default address pool") }
} }
})
// configDefaultNetworks configures local as well global default pool based on input
func configDefaultNetworks(defaultAddressPool []*NetworkToSplit, result *[]*net.IPNet) error {
mutex.Lock()
defer mutex.Unlock()
defaultNetworks, err := splitNetworks(defaultAddressPool)
if err != nil {
return err
}
*result = defaultNetworks
return nil
}
// GetGlobalScopeDefaultNetworks returns PredefinedGlobalScopeDefaultNetworks
func GetGlobalScopeDefaultNetworks() []*net.IPNet {
mutex.Lock()
defer mutex.Unlock()
return PredefinedGlobalScopeDefaultNetworks
}
// GetLocalScopeDefaultNetworks returns PredefinedLocalScopeDefaultNetworks
func GetLocalScopeDefaultNetworks() []*net.IPNet {
mutex.Lock()
defer mutex.Unlock()
return PredefinedLocalScopeDefaultNetworks
}
// ConfigGlobalScopeDefaultNetworks configures global default pool.
// Ideally this will be called from SwarmKit as part of swarm init
func ConfigGlobalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error {
if defaultAddressPool == nil {
defaultAddressPool = globalScopeDefaultNetworks
}
return configDefaultNetworks(defaultAddressPool, &PredefinedGlobalScopeDefaultNetworks)
}
// ConfigLocalScopeDefaultNetworks configures local default pool.
// Ideally this will be called during libnetwork init
func ConfigLocalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error {
if defaultAddressPool == nil {
return nil
}
return configDefaultNetworks(defaultAddressPool, &PredefinedLocalScopeDefaultNetworks)
} }
// splitNetworks takes a slice of networks, split them accordingly and returns them // splitNetworks takes a slice of networks, split them accordingly and returns them

View file

@ -94,8 +94,8 @@ func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
} }
if link == nil || len(v4Nets) == 0 { if link == nil || len(v4Nets) == 0 {
// Choose from predefined broad networks // Choose from predefined local scope networks
v4Net, err := FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }

View file

@ -84,9 +84,13 @@ func (*ListNodesRequest) ProtoMessage() {}
func (*ListNodesRequest) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{2} } func (*ListNodesRequest) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{2} }
type ListNodesRequest_Filters struct { type ListNodesRequest_Filters struct {
Names []string `protobuf:"bytes,1,rep,name=names" json:"names,omitempty"` Names []string `protobuf:"bytes,1,rep,name=names" json:"names,omitempty"`
IDPrefixes []string `protobuf:"bytes,2,rep,name=id_prefixes,json=idPrefixes" json:"id_prefixes,omitempty"` IDPrefixes []string `protobuf:"bytes,2,rep,name=id_prefixes,json=idPrefixes" json:"id_prefixes,omitempty"`
Labels map[string]string `protobuf:"bytes,3,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Labels refers to engine labels, which are labels set by the user on the
// node and reported back to the managers
Labels map[string]string `protobuf:"bytes,3,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// NodeLabels are labels set on the node object on the managers.
NodeLabels map[string]string `protobuf:"bytes,7,rep,name=node_labels,json=nodeLabels" json:"node_labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Memberships []NodeSpec_Membership `protobuf:"varint,4,rep,name=memberships,enum=docker.swarmkit.v1.NodeSpec_Membership" json:"memberships,omitempty"` Memberships []NodeSpec_Membership `protobuf:"varint,4,rep,name=memberships,enum=docker.swarmkit.v1.NodeSpec_Membership" json:"memberships,omitempty"`
Roles []NodeRole `protobuf:"varint,5,rep,name=roles,enum=docker.swarmkit.v1.NodeRole" json:"roles,omitempty"` Roles []NodeRole `protobuf:"varint,5,rep,name=roles,enum=docker.swarmkit.v1.NodeRole" json:"roles,omitempty"`
// NamePrefixes matches all objects with the given prefixes // NamePrefixes matches all objects with the given prefixes
@ -1105,6 +1109,13 @@ func (m *ListNodesRequest_Filters) CopyFrom(src interface{}) {
} }
} }
if o.NodeLabels != nil {
m.NodeLabels = make(map[string]string, len(o.NodeLabels))
for k, v := range o.NodeLabels {
m.NodeLabels[k] = v
}
}
if o.Memberships != nil { if o.Memberships != nil {
m.Memberships = make([]NodeSpec_Membership, len(o.Memberships)) m.Memberships = make([]NodeSpec_Membership, len(o.Memberships))
copy(m.Memberships, o.Memberships) copy(m.Memberships, o.Memberships)
@ -3640,6 +3651,23 @@ func (m *ListNodesRequest_Filters) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], s) i += copy(dAtA[i:], s)
} }
} }
if len(m.NodeLabels) > 0 {
for k, _ := range m.NodeLabels {
dAtA[i] = 0x3a
i++
v := m.NodeLabels[k]
mapSize := 1 + len(k) + sovControl(uint64(len(k))) + 1 + len(v) + sovControl(uint64(len(v)))
i = encodeVarintControl(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintControl(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
dAtA[i] = 0x12
i++
i = encodeVarintControl(dAtA, i, uint64(len(v)))
i += copy(dAtA[i:], v)
}
}
return i, nil return i, nil
} }
@ -6985,6 +7013,14 @@ func (m *ListNodesRequest_Filters) Size() (n int) {
n += 1 + l + sovControl(uint64(l)) n += 1 + l + sovControl(uint64(l))
} }
} }
if len(m.NodeLabels) > 0 {
for k, v := range m.NodeLabels {
_ = k
_ = v
mapEntrySize := 1 + len(k) + sovControl(uint64(len(k))) + 1 + len(v) + sovControl(uint64(len(v)))
n += mapEntrySize + 1 + sovControl(uint64(mapEntrySize))
}
}
return n return n
} }
@ -7883,6 +7919,16 @@ func (this *ListNodesRequest_Filters) String() string {
mapStringForLabels += fmt.Sprintf("%v: %v,", k, this.Labels[k]) mapStringForLabels += fmt.Sprintf("%v: %v,", k, this.Labels[k])
} }
mapStringForLabels += "}" mapStringForLabels += "}"
keysForNodeLabels := make([]string, 0, len(this.NodeLabels))
for k, _ := range this.NodeLabels {
keysForNodeLabels = append(keysForNodeLabels, k)
}
sortkeys.Strings(keysForNodeLabels)
mapStringForNodeLabels := "map[string]string{"
for _, k := range keysForNodeLabels {
mapStringForNodeLabels += fmt.Sprintf("%v: %v,", k, this.NodeLabels[k])
}
mapStringForNodeLabels += "}"
s := strings.Join([]string{`&ListNodesRequest_Filters{`, s := strings.Join([]string{`&ListNodesRequest_Filters{`,
`Names:` + fmt.Sprintf("%v", this.Names) + `,`, `Names:` + fmt.Sprintf("%v", this.Names) + `,`,
`IDPrefixes:` + fmt.Sprintf("%v", this.IDPrefixes) + `,`, `IDPrefixes:` + fmt.Sprintf("%v", this.IDPrefixes) + `,`,
@ -7890,6 +7936,7 @@ func (this *ListNodesRequest_Filters) String() string {
`Memberships:` + fmt.Sprintf("%v", this.Memberships) + `,`, `Memberships:` + fmt.Sprintf("%v", this.Memberships) + `,`,
`Roles:` + fmt.Sprintf("%v", this.Roles) + `,`, `Roles:` + fmt.Sprintf("%v", this.Roles) + `,`,
`NamePrefixes:` + fmt.Sprintf("%v", this.NamePrefixes) + `,`, `NamePrefixes:` + fmt.Sprintf("%v", this.NamePrefixes) + `,`,
`NodeLabels:` + mapStringForNodeLabels + `,`,
`}`, `}`,
}, "") }, "")
return s return s
@ -9221,6 +9268,124 @@ func (m *ListNodesRequest_Filters) Unmarshal(dAtA []byte) error {
} }
m.NamePrefixes = append(m.NamePrefixes, string(dAtA[iNdEx:postIndex])) m.NamePrefixes = append(m.NamePrefixes, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field NodeLabels", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowControl
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthControl
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.NodeLabels == nil {
m.NodeLabels = make(map[string]string)
}
var mapkey string
var mapvalue string
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowControl
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowControl
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthControl
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
var stringLenmapvalue uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowControl
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapvalue := int(stringLenmapvalue)
if intStringLenmapvalue < 0 {
return ErrInvalidLengthControl
}
postStringIndexmapvalue := iNdEx + intStringLenmapvalue
if postStringIndexmapvalue > l {
return io.ErrUnexpectedEOF
}
mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue])
iNdEx = postStringIndexmapvalue
} else {
iNdEx = entryPreIndex
skippy, err := skipControl(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthControl
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.NodeLabels[mapkey] = mapvalue
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipControl(dAtA[iNdEx:]) skippy, err := skipControl(dAtA[iNdEx:])
@ -15950,139 +16115,141 @@ var (
func init() { proto.RegisterFile("github.com/docker/swarmkit/api/control.proto", fileDescriptorControl) } func init() { proto.RegisterFile("github.com/docker/swarmkit/api/control.proto", fileDescriptorControl) }
var fileDescriptorControl = []byte{ var fileDescriptorControl = []byte{
// 2137 bytes of a gzipped FileDescriptorProto // 2167 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5a, 0x4f, 0x73, 0x1b, 0x49, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5a, 0x4d, 0x73, 0x1b, 0x49,
0x15, 0xb7, 0xfe, 0xd8, 0x92, 0x9f, 0x6c, 0xd9, 0xee, 0x38, 0xa0, 0x52, 0x82, 0x9d, 0x9a, 0x90, 0x19, 0xb6, 0x3e, 0x6c, 0xc9, 0xaf, 0x6c, 0xd9, 0xee, 0x78, 0x41, 0xa5, 0x04, 0x3b, 0x35, 0x21,
0x44, 0xd9, 0x32, 0x12, 0xab, 0xb0, 0x6c, 0x58, 0x8a, 0x3f, 0x6b, 0x3b, 0x9b, 0xd5, 0x7a, 0xe3, 0x89, 0xb2, 0x65, 0x24, 0x56, 0x61, 0xd9, 0xb0, 0xb0, 0xc0, 0xda, 0xce, 0x66, 0xb5, 0xde, 0x38,
0xa4, 0xc6, 0xc9, 0x16, 0x37, 0x95, 0x2c, 0xb5, 0xbd, 0x13, 0xc9, 0x1a, 0x31, 0x33, 0xf2, 0xae, 0xa9, 0x71, 0xb2, 0xc5, 0x85, 0x52, 0xc9, 0x52, 0xdb, 0x3b, 0x91, 0xac, 0x11, 0x33, 0x23, 0xef,
0x8b, 0x0b, 0x50, 0xcb, 0x81, 0x0f, 0x40, 0x15, 0x57, 0xae, 0x1c, 0x38, 0x70, 0xe2, 0xc0, 0x07, 0xba, 0xb8, 0x00, 0x15, 0x7e, 0x02, 0x55, 0x5c, 0x39, 0x51, 0xc5, 0x81, 0x03, 0x27, 0x0e, 0xfc,
0x48, 0x71, 0xe2, 0xc8, 0xc9, 0xb0, 0xaa, 0x82, 0xe2, 0xc4, 0x67, 0xa0, 0xba, 0xfb, 0xf5, 0xfc, 0x80, 0x14, 0x27, 0x8e, 0x9c, 0x0c, 0xab, 0x2a, 0xaa, 0x38, 0xf1, 0x1b, 0xa8, 0xee, 0x7e, 0x7b,
0x53, 0xcf, 0x8c, 0x24, 0xab, 0xca, 0x39, 0x59, 0xd3, 0xf3, 0x7b, 0xfd, 0x5e, 0xf7, 0xfb, 0xf5, 0xbe, 0xd4, 0x33, 0xa3, 0x0f, 0x57, 0x79, 0x4f, 0x96, 0x7a, 0x9e, 0xf7, 0xa3, 0xfb, 0x7d, 0xfa,
0x6f, 0xba, 0x5f, 0x1b, 0x76, 0x4e, 0x0d, 0xe7, 0xf3, 0xe1, 0x71, 0xb5, 0x6d, 0x9e, 0xd5, 0x3a, 0x51, 0xf7, 0x3b, 0x86, 0x9d, 0x53, 0xc3, 0xf9, 0x7c, 0x78, 0x5c, 0x6d, 0x9b, 0x67, 0xb5, 0x8e,
0x66, 0xbb, 0x4b, 0xad, 0x9a, 0xfd, 0x45, 0xcb, 0x3a, 0xeb, 0x1a, 0x4e, 0xad, 0x35, 0x30, 0x6a, 0xd9, 0xee, 0x52, 0xab, 0x66, 0x7f, 0xd1, 0xb2, 0xce, 0xba, 0x86, 0x53, 0x6b, 0x0d, 0x8c, 0x5a,
0x6d, 0xb3, 0xef, 0x58, 0x66, 0xaf, 0x3a, 0xb0, 0x4c, 0xc7, 0x24, 0x44, 0x40, 0xaa, 0x12, 0x52, 0xdb, 0xec, 0x3b, 0x96, 0xd9, 0xab, 0x0e, 0x2c, 0xd3, 0x31, 0x09, 0x11, 0x90, 0xaa, 0x84, 0x54,
0x3d, 0x7f, 0xb7, 0xfc, 0x4e, 0x42, 0x0f, 0xf6, 0x80, 0xb6, 0x6d, 0x61, 0x5f, 0x4e, 0xf2, 0x66, 0xcf, 0xdf, 0x29, 0xbf, 0x9d, 0xe0, 0xc1, 0x1e, 0xd0, 0xb6, 0x2d, 0xec, 0xcb, 0x49, 0xd1, 0xcc,
0x1e, 0xbf, 0xa6, 0x6d, 0x47, 0xa2, 0x93, 0x7a, 0x76, 0x2e, 0x06, 0x54, 0x62, 0x37, 0x4f, 0xcd, 0xe3, 0x57, 0xb4, 0xed, 0x48, 0x74, 0x92, 0x67, 0xe7, 0x62, 0x40, 0x25, 0x76, 0xf3, 0xd4, 0x3c,
0x53, 0x93, 0xff, 0xac, 0xb1, 0x5f, 0xd8, 0xfa, 0x7e, 0x4c, 0x0f, 0x1c, 0x71, 0x3c, 0x3c, 0xa9, 0x35, 0xf9, 0xc7, 0x1a, 0xfb, 0x84, 0xa3, 0xef, 0xc5, 0x78, 0xe0, 0x88, 0xe3, 0xe1, 0x49, 0x6d,
0x0d, 0x7a, 0xc3, 0x53, 0xa3, 0x8f, 0x7f, 0x84, 0xa1, 0xf6, 0x1e, 0x14, 0x9f, 0x52, 0xe7, 0xd0, 0xd0, 0x1b, 0x9e, 0x1a, 0x7d, 0xfc, 0x23, 0x0c, 0xb5, 0x77, 0xa1, 0xf8, 0x84, 0x3a, 0x87, 0x66,
0xec, 0x50, 0x9d, 0xfe, 0x7c, 0x48, 0x6d, 0x87, 0xdc, 0x85, 0x5c, 0xdf, 0xec, 0xd0, 0xa6, 0xd1, 0x87, 0xea, 0xf4, 0x17, 0x43, 0x6a, 0x3b, 0xe4, 0x0e, 0xe4, 0xfa, 0x66, 0x87, 0x36, 0x8d, 0x4e,
0x29, 0xa5, 0xee, 0xa4, 0x2a, 0xcb, 0xbb, 0x30, 0xba, 0xdc, 0x5e, 0x62, 0x88, 0xc6, 0xbe, 0xbe, 0x29, 0x75, 0x3b, 0x55, 0x59, 0xde, 0x85, 0xd1, 0xe5, 0xf6, 0x12, 0x43, 0x34, 0xf6, 0xf5, 0x25,
0xc4, 0x5e, 0x35, 0x3a, 0xda, 0x4f, 0x60, 0xcd, 0x35, 0xb3, 0x07, 0x66, 0xdf, 0xa6, 0x64, 0x07, 0xf6, 0xa8, 0xd1, 0xd1, 0x7e, 0x02, 0x6b, 0xae, 0x99, 0x3d, 0x30, 0xfb, 0x36, 0x25, 0x3b, 0x90,
0xb2, 0xec, 0x25, 0x37, 0x2a, 0xd4, 0x4b, 0xd5, 0xf1, 0x19, 0xac, 0x72, 0x3c, 0x47, 0x69, 0xff, 0x65, 0x0f, 0xb9, 0x51, 0xa1, 0x5e, 0xaa, 0x8e, 0xaf, 0x60, 0x95, 0xe3, 0x39, 0x4a, 0x7b, 0xbd,
0xc9, 0xc0, 0xfa, 0xa7, 0x86, 0xcd, 0xbb, 0xb0, 0xa5, 0xeb, 0x8f, 0x20, 0x77, 0x62, 0xf4, 0x1c, 0x08, 0xeb, 0x9f, 0x1a, 0x36, 0x77, 0x61, 0xcb, 0xd0, 0x1f, 0x41, 0xee, 0xc4, 0xe8, 0x39, 0xd4,
0x6a, 0xd9, 0xd8, 0xcb, 0x8e, 0xaa, 0x97, 0xb0, 0x59, 0xf5, 0x23, 0x61, 0xa3, 0x4b, 0xe3, 0xf2, 0xb2, 0xd1, 0xcb, 0x8e, 0xca, 0x4b, 0xd8, 0xac, 0xfa, 0x91, 0xb0, 0xd1, 0xa5, 0x71, 0xf9, 0x8f,
0x6f, 0x33, 0x90, 0xc3, 0x46, 0xb2, 0x09, 0x8b, 0xfd, 0xd6, 0x19, 0x65, 0x3d, 0x66, 0x2a, 0xcb, 0x59, 0xc8, 0xe1, 0x20, 0xd9, 0x84, 0xc5, 0x7e, 0xeb, 0x8c, 0x32, 0x8f, 0x99, 0xca, 0xb2, 0x2e,
0xba, 0x78, 0x20, 0x35, 0x28, 0x18, 0x9d, 0xe6, 0xc0, 0xa2, 0x27, 0xc6, 0x97, 0xd4, 0x2e, 0xa5, 0xbe, 0x90, 0x1a, 0x14, 0x8c, 0x4e, 0x73, 0x60, 0xd1, 0x13, 0xe3, 0x4b, 0x6a, 0x97, 0xd2, 0xec,
0xd9, 0xbb, 0xdd, 0xe2, 0xe8, 0x72, 0x1b, 0x1a, 0xfb, 0x2f, 0xb0, 0x55, 0x07, 0xa3, 0x23, 0x7f, 0xd9, 0x6e, 0x71, 0x74, 0xb9, 0x0d, 0x8d, 0xfd, 0xe7, 0x38, 0xaa, 0x83, 0xd1, 0x91, 0x9f, 0xc9,
0x93, 0x17, 0xb0, 0xd4, 0x6b, 0x1d, 0xd3, 0x9e, 0x5d, 0xca, 0xdc, 0xc9, 0x54, 0x0a, 0xf5, 0xc7, 0x73, 0x58, 0xea, 0xb5, 0x8e, 0x69, 0xcf, 0x2e, 0x65, 0x6e, 0x67, 0x2a, 0x85, 0xfa, 0xa3, 0x69,
0xd3, 0x44, 0x56, 0xfd, 0x94, 0x9b, 0x3e, 0xe9, 0x3b, 0xd6, 0x85, 0x8e, 0xfd, 0x90, 0x67, 0x50, 0x32, 0xab, 0x7e, 0xca, 0x4d, 0x1f, 0xf7, 0x1d, 0xeb, 0x42, 0x47, 0x3f, 0xe4, 0x29, 0x14, 0xce,
0x38, 0xa3, 0x67, 0xc7, 0xd4, 0xb2, 0x3f, 0x37, 0x06, 0x76, 0x29, 0x7b, 0x27, 0x53, 0x29, 0xd6, 0xe8, 0xd9, 0x31, 0xb5, 0xec, 0xcf, 0x8d, 0x81, 0x5d, 0xca, 0xde, 0xce, 0x54, 0x8a, 0xf5, 0xfb,
0x1f, 0x44, 0x4d, 0xdb, 0xd1, 0x80, 0xb6, 0xab, 0xcf, 0x5c, 0xfc, 0x6e, 0x7a, 0x7d, 0x41, 0xf7, 0x51, 0xcb, 0x76, 0x34, 0xa0, 0xed, 0xea, 0x53, 0x17, 0xbf, 0x9b, 0x5e, 0x5f, 0xd0, 0xfd, 0xf6,
0xdb, 0x93, 0xef, 0xc3, 0xa2, 0x65, 0xf6, 0xa8, 0x5d, 0x5a, 0xe4, 0x1d, 0xdd, 0x8e, 0x9c, 0x7f, 0xe4, 0xfb, 0xb0, 0x68, 0x99, 0x3d, 0x6a, 0x97, 0x16, 0xb9, 0xa3, 0x5b, 0x91, 0xeb, 0x6f, 0xf6,
0xb3, 0x47, 0xb9, 0xb5, 0x80, 0x93, 0xbb, 0xb0, 0xca, 0xa6, 0xc4, 0x9b, 0x8b, 0x25, 0x3e, 0x4f, 0x28, 0xb7, 0x16, 0x70, 0x72, 0x07, 0x56, 0xd9, 0x92, 0x78, 0x6b, 0xb1, 0xc4, 0xd7, 0x69, 0x85,
0x2b, 0xac, 0x51, 0x8e, 0xbe, 0xfc, 0x03, 0x28, 0xf8, 0x86, 0x40, 0xd6, 0x21, 0xd3, 0xa5, 0x17, 0x0d, 0xba, 0xb3, 0xff, 0x39, 0x14, 0x38, 0x27, 0x70, 0x09, 0x72, 0x7c, 0x09, 0x7e, 0x34, 0xd5,
0x82, 0x1e, 0x3a, 0xfb, 0xc9, 0x66, 0xf9, 0xbc, 0xd5, 0x1b, 0xd2, 0x52, 0x9a, 0xb7, 0x89, 0x87, 0x12, 0xb0, 0x41, 0xff, 0x32, 0x40, 0xdf, 0x1d, 0x28, 0xff, 0x00, 0x0a, 0xbe, 0x47, 0x64, 0x1d,
0x0f, 0xd2, 0x8f, 0x53, 0xda, 0x1e, 0x6c, 0xf8, 0xa6, 0x05, 0xb9, 0x52, 0x85, 0x45, 0xc6, 0x02, 0x32, 0x5d, 0x7a, 0x21, 0xd8, 0xa7, 0xb3, 0x8f, 0xac, 0x88, 0xe7, 0xad, 0xde, 0x90, 0x96, 0xd2,
0x91, 0x94, 0x38, 0xb2, 0x08, 0x98, 0xf6, 0xc7, 0x14, 0x6c, 0xbc, 0x1a, 0x74, 0x5a, 0x0e, 0x9d, 0x7c, 0x4c, 0x7c, 0x79, 0x3f, 0xfd, 0x28, 0x55, 0xfe, 0x00, 0xd6, 0x42, 0x9e, 0xa7, 0x31, 0xd7,
0x96, 0xa9, 0xe4, 0xc7, 0xb0, 0xc2, 0x41, 0xe7, 0xd4, 0xb2, 0x0d, 0xb3, 0xcf, 0x03, 0x2c, 0xd4, 0xf6, 0x60, 0xc3, 0x97, 0x31, 0x32, 0xb9, 0x0a, 0x8b, 0x2c, 0x39, 0x41, 0x99, 0x38, 0x2a, 0x0b,
0x6f, 0xa9, 0x3c, 0x7e, 0x26, 0x20, 0x7a, 0x81, 0x19, 0xe0, 0x03, 0xf9, 0x2e, 0x64, 0xd9, 0xc2, 0x98, 0xf6, 0xa7, 0x14, 0x6c, 0xbc, 0x1c, 0x74, 0x5a, 0x0e, 0x9d, 0x76, 0x1f, 0x91, 0x1f, 0xc3,
0x2e, 0x65, 0xb8, 0xdd, 0xed, 0xb8, 0xfc, 0xe8, 0x1c, 0xa9, 0xed, 0x02, 0xf1, 0xc7, 0x3a, 0xd3, 0x0a, 0x07, 0x9d, 0x53, 0xcb, 0x36, 0xcc, 0x3e, 0x4f, 0xb0, 0x50, 0xbf, 0xa9, 0x8a, 0xf8, 0x99,
0xf2, 0x38, 0x84, 0x0d, 0x9d, 0x9e, 0x99, 0xe7, 0xd3, 0x8f, 0x77, 0x13, 0x16, 0x4f, 0x4c, 0xab, 0x80, 0xe8, 0xbc, 0x12, 0xf8, 0x85, 0x7c, 0x17, 0xb2, 0x4c, 0x76, 0x4a, 0x19, 0x6e, 0x77, 0x2b,
0x2d, 0x32, 0x91, 0xd7, 0xc5, 0x83, 0xb6, 0x09, 0xc4, 0xdf, 0x9f, 0x88, 0x09, 0x17, 0xff, 0xcb, 0x8e, 0x3d, 0x3a, 0x47, 0x6a, 0xbb, 0x40, 0xfc, 0xb9, 0xce, 0xb4, 0x79, 0x0f, 0x61, 0x43, 0xa7,
0x96, 0xdd, 0xf5, 0xb9, 0x70, 0x5a, 0x76, 0x37, 0xe4, 0x82, 0x21, 0x98, 0x0b, 0xf6, 0xca, 0x5d, 0x67, 0xe6, 0xf9, 0xf4, 0xf3, 0xdd, 0x84, 0xc5, 0x13, 0xd3, 0x6a, 0x8b, 0x4a, 0xe4, 0x75, 0xf1,
0xfc, 0xc2, 0xcc, 0x1b, 0x1d, 0x7b, 0x19, 0x37, 0x3a, 0x8e, 0xe7, 0x28, 0xed, 0xb1, 0x1c, 0xdd, 0x45, 0xdb, 0x04, 0xe2, 0xf7, 0x27, 0x72, 0x42, 0x69, 0x7a, 0xd1, 0xb2, 0xbb, 0xbe, 0x10, 0x4e,
0xd4, 0xae, 0xdd, 0x71, 0xf8, 0xbd, 0x6b, 0x7f, 0xcd, 0x0a, 0x31, 0x61, 0x8d, 0x33, 0x88, 0x89, 0xcb, 0xee, 0x86, 0x42, 0x30, 0x04, 0x0b, 0xc1, 0x1e, 0xb9, 0xd2, 0x24, 0xcc, 0xbc, 0xd9, 0xb1,
0xdf, 0x6c, 0x5c, 0x4c, 0xfe, 0x79, 0x8d, 0x62, 0xa2, 0x8a, 0x4c, 0x29, 0x26, 0x35, 0x28, 0xd8, 0x87, 0x71, 0xb3, 0xe3, 0x78, 0x8e, 0xd2, 0x1e, 0xc9, 0xd9, 0x4d, 0x1d, 0xda, 0x9d, 0x87, 0x3f,
0xd4, 0x3a, 0x37, 0xda, 0x8c, 0x1d, 0x42, 0x4c, 0x30, 0x84, 0x23, 0xd1, 0xdc, 0xd8, 0xb7, 0x75, 0xba, 0xf6, 0xb7, 0xac, 0x90, 0x3a, 0x36, 0x38, 0x83, 0xd4, 0xf9, 0xcd, 0xc6, 0xa5, 0xee, 0x5f,
0x40, 0x48, 0xa3, 0x63, 0x93, 0xfb, 0x90, 0x47, 0x2e, 0x09, 0xc5, 0x58, 0xde, 0x2d, 0x8c, 0x2e, 0x99, 0xeb, 0x93, 0x3a, 0x55, 0x66, 0x4a, 0xa9, 0xab, 0x41, 0xc1, 0xa6, 0xd6, 0xb9, 0xd1, 0x66,
0xb7, 0x73, 0x82, 0x4c, 0xb6, 0x9e, 0x13, 0x6c, 0xb2, 0xc9, 0xc7, 0x50, 0xec, 0x50, 0xdb, 0xb0, 0xec, 0x10, 0x52, 0x87, 0x29, 0x1c, 0x89, 0xe1, 0xc6, 0xbe, 0xad, 0x03, 0x42, 0x1a, 0x1d, 0x9b,
0x68, 0xa7, 0x69, 0x3b, 0x2d, 0x07, 0xf5, 0xa1, 0x58, 0xff, 0x56, 0x54, 0x8a, 0x8f, 0x18, 0x8a, 0xdc, 0x83, 0x3c, 0x72, 0x49, 0xe8, 0xd9, 0xf2, 0x6e, 0x61, 0x74, 0xb9, 0x9d, 0x13, 0x64, 0xb2,
0x0b, 0xcc, 0x2a, 0x1a, 0xf2, 0x16, 0x85, 0xd0, 0xe4, 0xc6, 0x85, 0x86, 0xdc, 0x06, 0x18, 0x0e, 0xf5, 0x9c, 0x60, 0x93, 0x4d, 0x3e, 0x86, 0x62, 0x87, 0xda, 0x86, 0x45, 0x3b, 0x4d, 0xdb, 0x69,
0x9a, 0x8e, 0xd9, 0x64, 0xeb, 0xa7, 0x94, 0xe7, 0x14, 0xce, 0x0f, 0x07, 0x2f, 0xcd, 0xfd, 0x96, 0x39, 0xa8, 0x5e, 0xc5, 0xfa, 0xb7, 0xa2, 0x4a, 0x7c, 0xc4, 0x50, 0x5c, 0xfe, 0x56, 0xd1, 0x90,
0x43, 0x49, 0x19, 0xf2, 0xd6, 0xb0, 0xef, 0x18, 0x2c, 0x03, 0xcb, 0xdc, 0xda, 0x7d, 0x9e, 0x83, 0x8f, 0x28, 0x64, 0x30, 0xa7, 0x90, 0xc1, 0x5b, 0x00, 0xc3, 0x41, 0xd3, 0x31, 0x9b, 0x6c, 0xff,
0x44, 0xe1, 0x64, 0x7b, 0x12, 0xc5, 0x38, 0x17, 0x2b, 0x51, 0x9c, 0x84, 0x02, 0xa6, 0x1d, 0xc0, 0x94, 0xf2, 0x9c, 0xc2, 0xf9, 0xe1, 0xe0, 0x85, 0xb9, 0xdf, 0x72, 0x28, 0x29, 0x43, 0xde, 0x1a,
0xe6, 0x9e, 0x45, 0x5b, 0x0e, 0xc5, 0x09, 0x97, 0x34, 0x7c, 0x84, 0xfa, 0x21, 0x38, 0xb8, 0xad, 0xf6, 0x1d, 0x83, 0x55, 0x60, 0x99, 0x5b, 0xbb, 0xdf, 0xe7, 0x50, 0x38, 0x29, 0x51, 0xb8, 0xd8,
0xea, 0x06, 0x2d, 0x7c, 0x12, 0x72, 0x08, 0x37, 0x43, 0x9d, 0x61, 0x54, 0xef, 0x41, 0x0e, 0x93, 0x9e, 0x44, 0x31, 0xce, 0xc5, 0x4a, 0x14, 0x27, 0xa1, 0x80, 0x69, 0x07, 0xb0, 0xb9, 0x67, 0xd1,
0x88, 0x1d, 0xde, 0x8a, 0xe9, 0x50, 0x97, 0x58, 0xed, 0x35, 0x6c, 0x3c, 0xa5, 0x4e, 0x28, 0xb2, 0x96, 0x43, 0x71, 0xc1, 0x25, 0x0d, 0x1f, 0xa2, 0x7e, 0x08, 0x0e, 0x6e, 0xab, 0xdc, 0xa0, 0x85,
0x1d, 0x00, 0x8f, 0x33, 0xb8, 0xe6, 0x56, 0x47, 0x97, 0xdb, 0xcb, 0x2e, 0x65, 0xf4, 0x65, 0x97, 0x4f, 0x42, 0x0e, 0xe1, 0xad, 0x90, 0x33, 0xcc, 0xea, 0x5d, 0xc8, 0x61, 0x11, 0xd1, 0xe1, 0xcd,
0x31, 0xe4, 0x01, 0xac, 0x19, 0x7d, 0x9b, 0x5a, 0x4e, 0xb3, 0x43, 0x4f, 0x5a, 0xc3, 0x9e, 0x63, 0x18, 0x87, 0xba, 0xc4, 0x6a, 0xaf, 0x60, 0xe3, 0x09, 0x75, 0x42, 0x99, 0xed, 0x00, 0x78, 0x9c,
0xa3, 0xc2, 0x14, 0x45, 0xf3, 0x3e, 0xb6, 0x6a, 0x07, 0x40, 0xfc, 0xbe, 0xae, 0x16, 0xf8, 0x9f, 0xc1, 0x3d, 0xb7, 0x3a, 0xba, 0xdc, 0x5e, 0x76, 0x29, 0xa3, 0x2f, 0xbb, 0x8c, 0x21, 0xf7, 0x61,
0xd3, 0xb0, 0x29, 0xc4, 0xf4, 0x4a, 0xc1, 0xef, 0xc3, 0x9a, 0x44, 0x4f, 0xf1, 0x1d, 0x28, 0xa2, 0xcd, 0xe8, 0xdb, 0xd4, 0x72, 0x9a, 0x1d, 0x7a, 0xd2, 0x1a, 0xf6, 0x1c, 0x1b, 0x15, 0xa6, 0x28,
0x8d, 0xfc, 0x14, 0x3c, 0x0a, 0x7c, 0x0a, 0x26, 0x4b, 0x25, 0x79, 0x06, 0x79, 0xcb, 0xec, 0xf5, 0x86, 0xf7, 0x71, 0x54, 0x3b, 0x00, 0xe2, 0x8f, 0x35, 0x5f, 0xe2, 0x7f, 0x49, 0xc3, 0xa6, 0x10,
0x8e, 0x5b, 0xed, 0x6e, 0x29, 0x7b, 0x27, 0x55, 0x29, 0xd6, 0xdf, 0x55, 0x19, 0xaa, 0x06, 0x59, 0xd3, 0xb9, 0x92, 0xdf, 0x87, 0x35, 0x89, 0x9e, 0xe2, 0x77, 0xa0, 0x88, 0x36, 0xf2, 0xa7, 0xe0,
0xd5, 0xd1, 0x50, 0x77, 0xbb, 0xd0, 0x34, 0xc8, 0xcb, 0x56, 0x92, 0x87, 0xec, 0xe1, 0xf3, 0xc3, 0x61, 0xe0, 0xa7, 0x60, 0xb2, 0x52, 0x92, 0xa7, 0x90, 0xb7, 0xcc, 0x5e, 0xef, 0xb8, 0xd5, 0xee,
0x27, 0xeb, 0x0b, 0x64, 0x05, 0xf2, 0x2f, 0xf4, 0x27, 0x9f, 0x35, 0x9e, 0xbf, 0x3a, 0x5a, 0x4f, 0x96, 0xb2, 0xb7, 0x53, 0x95, 0x62, 0xfd, 0x1d, 0x95, 0xa1, 0x6a, 0x92, 0x55, 0x1d, 0x0d, 0x75,
0x31, 0xf6, 0x84, 0xba, 0xbb, 0x5a, 0x12, 0xf6, 0x61, 0x53, 0x88, 0xee, 0x55, 0x72, 0xa0, 0x7d, 0xd7, 0x85, 0xa6, 0x41, 0x5e, 0x8e, 0x92, 0x3c, 0x64, 0x0f, 0x9f, 0x1d, 0x3e, 0x5e, 0x5f, 0x20,
0x13, 0x6e, 0x86, 0x7a, 0x41, 0xf5, 0xfe, 0x2a, 0x03, 0x37, 0xd8, 0xfa, 0xc3, 0x76, 0x57, 0xc0, 0x2b, 0x90, 0x7f, 0xae, 0x3f, 0xfe, 0xac, 0xf1, 0xec, 0xe5, 0xd1, 0x7a, 0x8a, 0xb1, 0x27, 0xe4,
0x1b, 0x61, 0x01, 0xaf, 0x45, 0xc9, 0x64, 0xc8, 0x72, 0x5c, 0xc3, 0xff, 0x90, 0x9e, 0xbb, 0x86, 0x6e, 0xbe, 0x22, 0xec, 0xc3, 0xa6, 0x10, 0xdd, 0x79, 0x6a, 0xa0, 0x7d, 0x13, 0xde, 0x0a, 0x79,
0x1f, 0x85, 0x34, 0xfc, 0x87, 0x53, 0x06, 0xa7, 0x94, 0xf1, 0x31, 0x8d, 0xcc, 0x2a, 0x34, 0xd2, 0x41, 0xf5, 0x7e, 0x9d, 0x81, 0x1b, 0x6c, 0xff, 0xe1, 0xb8, 0x2b, 0xe0, 0x8d, 0xb0, 0x80, 0xd7,
0xaf, 0x82, 0x8b, 0xf3, 0x53, 0xc1, 0xe7, 0xb0, 0x19, 0x0c, 0x17, 0x49, 0xf3, 0x3e, 0xe4, 0x31, 0xa2, 0x64, 0x32, 0x64, 0x39, 0xae, 0xe1, 0x7f, 0x48, 0x5f, 0xb9, 0x86, 0x1f, 0x85, 0x34, 0xfc,
0x89, 0x52, 0x0b, 0x63, 0x59, 0xe3, 0x82, 0x3d, 0x45, 0x3c, 0xa4, 0xce, 0x17, 0xa6, 0xd5, 0x9d, 0x87, 0x53, 0x26, 0xa7, 0x94, 0xf1, 0x31, 0x8d, 0xcc, 0x2a, 0x34, 0xd2, 0xaf, 0x82, 0x8b, 0x57,
0x42, 0x11, 0xd1, 0x42, 0xa5, 0x88, 0x6e, 0x67, 0x1e, 0xa7, 0xfb, 0xa2, 0x29, 0x8e, 0xd3, 0xd2, 0xa7, 0x82, 0xcf, 0x60, 0x33, 0x98, 0x2e, 0x92, 0xe6, 0x3d, 0xc8, 0x63, 0x11, 0xa5, 0x16, 0xc6,
0x4a, 0x62, 0xb5, 0x57, 0x5c, 0x11, 0x43, 0x91, 0x11, 0xc8, 0xb2, 0x99, 0xc6, 0xf9, 0xe2, 0xbf, 0xb2, 0xc6, 0x05, 0x7b, 0x8a, 0x78, 0x48, 0x9d, 0x2f, 0x4c, 0xab, 0x3b, 0x85, 0x22, 0xa2, 0x85,
0x19, 0xc9, 0xd1, 0x86, 0x91, 0x3c, 0xed, 0x91, 0x1c, 0x6d, 0x19, 0xc9, 0x11, 0xd0, 0xe8, 0xa0, 0x4a, 0x11, 0x5d, 0x67, 0x1e, 0xa7, 0xfb, 0x62, 0x28, 0x8e, 0xd3, 0xd2, 0x4a, 0x62, 0xb5, 0x97,
0xf8, 0xcd, 0x29, 0xc6, 0x9f, 0xc9, 0x75, 0x37, 0xf7, 0x30, 0xdd, 0xb5, 0x18, 0x8a, 0x54, 0xfb, 0x5c, 0x11, 0x43, 0x99, 0x11, 0xc8, 0xb2, 0x95, 0xc6, 0xf5, 0xe2, 0x9f, 0x19, 0xc9, 0xd1, 0x86,
0x6f, 0x5a, 0xac, 0x45, 0x6c, 0x9f, 0x61, 0x2d, 0x86, 0x2c, 0xc7, 0xd7, 0xe2, 0x6f, 0xae, 0x71, 0x91, 0x3c, 0xed, 0x91, 0x1c, 0x6d, 0x19, 0xc9, 0x11, 0xd0, 0xe8, 0xa0, 0xf8, 0x5d, 0x51, 0x8e,
0x2d, 0x46, 0x04, 0x37, 0xf3, 0x5a, 0x9c, 0xc3, 0x7a, 0xf3, 0x42, 0xf2, 0xd6, 0x1b, 0x26, 0x2a, 0x3f, 0x93, 0xfb, 0xee, 0xca, 0xd3, 0x74, 0xf7, 0x62, 0x28, 0x53, 0xed, 0xbf, 0x69, 0xb1, 0x17,
0x76, 0xbd, 0xc9, 0xcc, 0xb9, 0x60, 0xed, 0x43, 0x4e, 0xe9, 0xbd, 0xde, 0xd0, 0x76, 0xa8, 0xe5, 0x71, 0x7c, 0x86, 0xbd, 0x18, 0xb2, 0x1c, 0xdf, 0x8b, 0xbf, 0xbd, 0xc6, 0xbd, 0x18, 0x91, 0xdc,
0xd3, 0xe8, 0xb6, 0x68, 0x09, 0x69, 0x34, 0xe2, 0x18, 0x2f, 0x10, 0xe0, 0xd2, 0xd7, 0xed, 0xc2, 0xcc, 0x7b, 0xf1, 0x0a, 0xf6, 0x9b, 0x97, 0x92, 0xb7, 0xdf, 0xb0, 0x50, 0xb1, 0xfb, 0x4d, 0x56,
0xa3, 0x2f, 0x42, 0xe2, 0xe8, 0x2b, 0xad, 0x24, 0xd6, 0xe5, 0x12, 0xbe, 0x98, 0x81, 0x4b, 0x21, 0xce, 0x05, 0x6b, 0x1f, 0x72, 0x4a, 0xef, 0xf5, 0x86, 0xb6, 0x43, 0x2d, 0x9f, 0x46, 0xb7, 0xc5,
0xcb, 0xb7, 0x8b, 0x4b, 0x11, 0xc1, 0x5d, 0x27, 0x97, 0xbc, 0x90, 0x3c, 0x2e, 0x61, 0x36, 0x62, 0x48, 0x48, 0xa3, 0x11, 0xc7, 0x78, 0x81, 0x00, 0x97, 0xbe, 0xae, 0x0b, 0x8f, 0xbe, 0x08, 0x89,
0xb9, 0x24, 0x53, 0xe7, 0x82, 0xb5, 0xdf, 0xa5, 0xa0, 0x70, 0x40, 0x2f, 0x74, 0xd3, 0x69, 0x39, 0xa3, 0xaf, 0xb4, 0x92, 0x58, 0x97, 0x4b, 0xf8, 0x60, 0x06, 0x2e, 0x85, 0x2c, 0xbf, 0x5e, 0x5c,
0x6c, 0xeb, 0xf3, 0x0e, 0x6c, 0x30, 0x92, 0x51, 0xab, 0xf9, 0xda, 0x34, 0xfa, 0x4d, 0xc7, 0xec, 0x8a, 0x48, 0xee, 0x3a, 0xb9, 0xe4, 0xa5, 0xe4, 0x71, 0x09, 0xab, 0x11, 0xcb, 0x25, 0x59, 0x3a,
0xd2, 0x3e, 0x0f, 0x2d, 0xaf, 0xaf, 0x89, 0x17, 0x9f, 0x98, 0x46, 0xff, 0x25, 0x6b, 0x26, 0x3b, 0x17, 0xac, 0xfd, 0x2e, 0x05, 0x85, 0x03, 0x7a, 0xa1, 0x9b, 0x4e, 0xcb, 0x61, 0x47, 0x9f, 0xb7,
0x40, 0xce, 0x5a, 0xfd, 0xd6, 0x69, 0x10, 0x2c, 0x36, 0x8b, 0xeb, 0xf8, 0x46, 0x89, 0x1e, 0xf6, 0x61, 0x83, 0x91, 0x8c, 0x5a, 0xcd, 0x57, 0xa6, 0xd1, 0x6f, 0x3a, 0x66, 0x97, 0xf6, 0x79, 0x6a,
0x7b, 0x66, 0xbb, 0xdb, 0x64, 0xa3, 0xce, 0x04, 0xd0, 0xaf, 0xf8, 0x8b, 0x03, 0x7a, 0xa1, 0xfd, 0x79, 0x7d, 0x4d, 0x3c, 0xf8, 0xc4, 0x34, 0xfa, 0x2f, 0xd8, 0x30, 0xd9, 0x01, 0x72, 0xd6, 0xea,
0xda, 0xdd, 0x0f, 0x5e, 0x85, 0xe7, 0x6c, 0x3f, 0x28, 0xd1, 0xd3, 0xec, 0x07, 0xd1, 0x66, 0x8a, 0xb7, 0x4e, 0x83, 0x60, 0x71, 0x58, 0x5c, 0xc7, 0x27, 0x4a, 0xf4, 0xb0, 0xdf, 0x33, 0xdb, 0xdd,
0xfd, 0x20, 0x7a, 0xf7, 0xed, 0x07, 0x3f, 0x64, 0xfb, 0x41, 0x31, 0xab, 0x7c, 0x3f, 0x18, 0x61, 0x26, 0x9b, 0x75, 0x26, 0x80, 0x7e, 0xc9, 0x1f, 0x1c, 0xd0, 0x0b, 0xed, 0x37, 0xee, 0x79, 0x70,
0xe8, 0x9b, 0xfc, 0xdd, 0xec, 0x9b, 0xcb, 0xed, 0x05, 0xdd, 0x35, 0xf3, 0xf6, 0x77, 0x73, 0x5a, 0x1e, 0x9e, 0xb3, 0xf3, 0xa0, 0x44, 0x4f, 0x73, 0x1e, 0x44, 0x9b, 0x29, 0xce, 0x83, 0x18, 0xdd,
0xa8, 0x3f, 0x82, 0x75, 0xbe, 0x63, 0x6f, 0x5b, 0xd4, 0x91, 0xf3, 0xf9, 0x10, 0x96, 0x6d, 0xde, 0x77, 0x1e, 0xfc, 0x90, 0x9d, 0x07, 0xc5, 0xaa, 0xf2, 0xf3, 0x60, 0x84, 0xa1, 0x6f, 0xf1, 0x77,
0xe0, 0x4d, 0xe7, 0xca, 0xe8, 0x72, 0x3b, 0x2f, 0x50, 0x8d, 0x7d, 0xf6, 0x9d, 0xe7, 0xbf, 0x3a, 0xb3, 0x6f, 0x2e, 0xb7, 0x17, 0x74, 0xd7, 0xcc, 0x3b, 0xdf, 0x5d, 0xd1, 0x46, 0xfd, 0x00, 0xd6,
0xda, 0x53, 0x3c, 0x5c, 0x08, 0x73, 0x0c, 0xa5, 0x0e, 0x4b, 0x02, 0x80, 0x91, 0x94, 0xd5, 0x7b, 0xf9, 0x89, 0xbd, 0x6d, 0x51, 0x47, 0xae, 0xe7, 0x03, 0x58, 0xb6, 0xf9, 0x80, 0xb7, 0x9c, 0x2b,
0x06, 0x6e, 0x83, 0x48, 0xed, 0x2f, 0x29, 0xb8, 0x21, 0x37, 0xae, 0xb3, 0xc5, 0x42, 0x76, 0xa1, 0xa3, 0xcb, 0xed, 0xbc, 0x40, 0x35, 0xf6, 0xd9, 0xef, 0x3c, 0xff, 0xd4, 0xd1, 0x9e, 0xe0, 0xe5,
0x88, 0xd0, 0x29, 0xf2, 0xba, 0x2a, 0x4c, 0x64, 0x5a, 0xeb, 0x81, 0xb4, 0x6e, 0x45, 0x07, 0xee, 0x42, 0x98, 0x63, 0x2a, 0x75, 0x58, 0x12, 0x00, 0xcc, 0xa4, 0xac, 0x3e, 0x33, 0x70, 0x1b, 0x44,
0xdb, 0x9e, 0x7c, 0xe2, 0x1d, 0x53, 0xae, 0x3c, 0x0d, 0xff, 0x4e, 0x03, 0x11, 0x3b, 0x31, 0xf6, 0x6a, 0x7f, 0x4d, 0xc1, 0x0d, 0x79, 0x70, 0x9d, 0x2d, 0x17, 0xb2, 0x0b, 0x45, 0x84, 0x4e, 0x51,
0xe8, 0xca, 0xe6, 0xc7, 0x61, 0xd9, 0xac, 0x46, 0xef, 0x38, 0xfd, 0x86, 0xe3, 0xaa, 0xf9, 0xd5, 0xd7, 0x55, 0x61, 0x22, 0xcb, 0x5a, 0x0f, 0x94, 0x75, 0x2b, 0x3a, 0x71, 0xdf, 0xf1, 0xe4, 0x13,
0xfc, 0x55, 0x53, 0x0f, 0xa9, 0xe6, 0x07, 0xd3, 0xc5, 0x76, 0x2d, 0xa2, 0x79, 0x20, 0x8f, 0x1d, 0xef, 0x9a, 0x32, 0xf7, 0x32, 0xfc, 0x27, 0x0d, 0x44, 0x9c, 0xc4, 0xd8, 0x57, 0x57, 0x36, 0x3f,
0x18, 0x11, 0xa6, 0xec, 0x7b, 0xec, 0x90, 0xc4, 0x9b, 0x50, 0x32, 0xe3, 0x72, 0x26, 0xa1, 0x5a, 0x0e, 0xcb, 0x66, 0x35, 0xfa, 0xc4, 0xe9, 0x37, 0x1c, 0x57, 0xcd, 0xd7, 0x57, 0xaf, 0x9a, 0x7a,
0x03, 0x6e, 0xc8, 0x13, 0xbb, 0x9f, 0xba, 0xf5, 0xc0, 0x5e, 0x77, 0x62, 0x2e, 0x05, 0xbb, 0xba, 0x48, 0x35, 0xdf, 0x9f, 0x2e, 0xb7, 0x6b, 0x11, 0xcd, 0x03, 0x79, 0xed, 0xc0, 0x8c, 0xb0, 0x64,
0x02, 0x97, 0x7e, 0x0a, 0x37, 0xe4, 0xa1, 0x6b, 0xc6, 0xd5, 0xfd, 0x0d, 0xef, 0xf0, 0xe7, 0x8f, 0xdf, 0x63, 0x97, 0x24, 0x3e, 0x84, 0x92, 0x19, 0x57, 0x33, 0x09, 0xd5, 0x1a, 0x70, 0x43, 0xde,
0x06, 0x45, 0x63, 0xcf, 0xec, 0x9f, 0x18, 0xa7, 0xbe, 0x6e, 0xdb, 0xbc, 0x21, 0xd4, 0xad, 0x40, 0xd8, 0xfd, 0xd4, 0xad, 0x07, 0xce, 0xba, 0x13, 0x73, 0x29, 0xe8, 0x6a, 0x0e, 0x2e, 0xfd, 0x14,
0xb1, 0x6e, 0xc5, 0x6b, 0x57, 0x34, 0xa4, 0xb9, 0x37, 0x42, 0x01, 0x88, 0x1b, 0x21, 0xda, 0x20, 0x6e, 0xc8, 0x4b, 0xd7, 0x8c, 0xbb, 0xfb, 0x1b, 0xde, 0xe5, 0xcf, 0x9f, 0x0d, 0x8a, 0xc6, 0x9e,
0xd2, 0x27, 0x1a, 0xb3, 0xc6, 0xc2, 0x44, 0x03, 0xa1, 0xd3, 0x88, 0x86, 0x30, 0x99, 0x42, 0x34, 0xd9, 0x3f, 0x31, 0x4e, 0x7d, 0x6e, 0xdb, 0x7c, 0x20, 0xe4, 0x56, 0xa0, 0x98, 0x5b, 0xf1, 0xd8,
0x84, 0x67, 0x95, 0x68, 0xcc, 0x61, 0x1a, 0xa4, 0x68, 0x88, 0xe6, 0x19, 0x44, 0x23, 0x68, 0xf8, 0x15, 0x0d, 0x69, 0xee, 0xcd, 0x50, 0x00, 0xe2, 0x66, 0x88, 0x36, 0x88, 0xf4, 0x89, 0xc6, 0xac,
0x76, 0x89, 0x86, 0x3a, 0xb6, 0xeb, 0x14, 0x0d, 0x37, 0x22, 0x4f, 0x34, 0x44, 0x22, 0x62, 0x45, 0xb9, 0x30, 0xd1, 0x40, 0xe8, 0x34, 0xa2, 0x21, 0x4c, 0xa6, 0x10, 0x0d, 0x11, 0x59, 0x25, 0x1a,
0x03, 0x73, 0x26, 0xa1, 0x9e, 0x68, 0x04, 0xa9, 0x3b, 0x81, 0x68, 0xa8, 0xb8, 0x14, 0xec, 0xea, 0x57, 0xb0, 0x0c, 0x52, 0x34, 0xc4, 0xf0, 0x0c, 0xa2, 0x11, 0x34, 0xfc, 0x7a, 0x89, 0x86, 0x3a,
0x0a, 0x5c, 0x72, 0x45, 0x63, 0xe6, 0xd5, 0xed, 0x8a, 0x46, 0x30, 0x9a, 0xfa, 0xaf, 0x6e, 0x41, 0xb7, 0xeb, 0x14, 0x0d, 0x37, 0x23, 0x4f, 0x34, 0x44, 0x21, 0x62, 0x45, 0x03, 0x6b, 0x26, 0xa1,
0x6e, 0x4f, 0x5c, 0xb4, 0x12, 0x03, 0x72, 0x78, 0x85, 0x48, 0x34, 0x55, 0x50, 0xc1, 0x6b, 0xc9, 0x9e, 0x68, 0x04, 0xa9, 0x3b, 0x81, 0x68, 0xa8, 0xb8, 0x14, 0x74, 0x35, 0x07, 0x97, 0x5c, 0xd1,
0xf2, 0xdd, 0x58, 0x0c, 0x8a, 0xd2, 0xcd, 0xbf, 0xfd, 0xe9, 0x7f, 0xbf, 0x4f, 0xaf, 0xc1, 0x2a, 0x98, 0x79, 0x77, 0xbb, 0xa2, 0x11, 0xcc, 0xa6, 0xfe, 0xeb, 0x9b, 0x90, 0xdb, 0x13, 0xaf, 0x81,
0x07, 0x7d, 0x07, 0xb7, 0x8f, 0xc4, 0x84, 0x65, 0xf7, 0x0e, 0x8a, 0x7c, 0x7b, 0x92, 0x9b, 0xbb, 0x89, 0x01, 0x39, 0x7c, 0xc1, 0x49, 0x34, 0x55, 0x52, 0xc1, 0x97, 0xa6, 0xe5, 0x3b, 0xb1, 0x18,
0xf2, 0xbd, 0x04, 0x54, 0xbc, 0x43, 0x0b, 0xc0, 0xbb, 0x02, 0x22, 0xf7, 0xa2, 0x0b, 0x7e, 0xfe, 0x14, 0xa5, 0xb7, 0xfe, 0xfe, 0xe7, 0xff, 0xfd, 0x3e, 0xbd, 0x06, 0xab, 0x1c, 0xf4, 0x1d, 0x3c,
0x11, 0xde, 0x4f, 0x82, 0x25, 0xfa, 0xf4, 0xae, 0x78, 0xd4, 0x3e, 0xc7, 0xae, 0x94, 0xd4, 0x3e, 0x3e, 0x12, 0x13, 0x96, 0xdd, 0x77, 0x50, 0xe4, 0xdb, 0x93, 0xbc, 0x54, 0x2b, 0xdf, 0x4d, 0x40,
0x15, 0x37, 0x45, 0x11, 0x3e, 0x45, 0x0e, 0x5f, 0xb6, 0xec, 0x6e, 0x64, 0x0e, 0x7d, 0x57, 0x3c, 0xc5, 0x07, 0xb4, 0x00, 0xbc, 0x57, 0x40, 0xe4, 0x6e, 0x74, 0xc3, 0xcf, 0x3f, 0xc3, 0x7b, 0x49,
0x91, 0x39, 0x0c, 0x5c, 0xe6, 0xc4, 0xe7, 0x90, 0x17, 0xe9, 0xa3, 0x73, 0xe8, 0xbf, 0x30, 0x89, 0xb0, 0xc4, 0x98, 0xde, 0x2b, 0x1e, 0x75, 0xcc, 0xb1, 0x57, 0x4a, 0xea, 0x98, 0x8a, 0x37, 0x45,
0xce, 0x61, 0xa0, 0xd2, 0x9f, 0x38, 0x9f, 0x7c, 0x78, 0x31, 0xf3, 0xe9, 0x1f, 0xe1, 0xfd, 0x24, 0x11, 0x31, 0x45, 0x0d, 0x5f, 0xb4, 0xec, 0x6e, 0x64, 0x0d, 0x7d, 0xaf, 0x78, 0x22, 0x6b, 0x18,
0x58, 0xa2, 0x4f, 0xaf, 0x76, 0xae, 0xf6, 0x39, 0x56, 0xc7, 0x57, 0xfb, 0x1c, 0x2f, 0xc1, 0x47, 0x78, 0x99, 0x13, 0x5f, 0x43, 0xde, 0xa4, 0x8f, 0xae, 0xa1, 0xff, 0x85, 0x49, 0x74, 0x0d, 0x03,
0xf9, 0xfc, 0x12, 0x56, 0xfc, 0x75, 0x3f, 0xf2, 0x60, 0xc2, 0x42, 0x66, 0xb9, 0x92, 0x0c, 0x8c, 0x9d, 0xfe, 0xc4, 0xf5, 0xe4, 0xd3, 0x8b, 0x59, 0x4f, 0xff, 0x0c, 0xef, 0x25, 0xc1, 0x12, 0x63,
0xf7, 0xfc, 0x0b, 0x58, 0x0d, 0xdc, 0x72, 0x10, 0x65, 0x8f, 0xaa, 0x5b, 0x95, 0xf2, 0xc3, 0x09, 0x7a, 0xbd, 0x73, 0x75, 0xcc, 0xb1, 0x3e, 0xbe, 0x3a, 0xe6, 0x78, 0x0b, 0x3e, 0x2a, 0xe6, 0x97,
0x90, 0x89, 0xce, 0x03, 0x45, 0x72, 0xb5, 0x73, 0x55, 0x59, 0x5e, 0xed, 0x5c, 0x59, 0x71, 0x8f, 0xb0, 0xe2, 0xef, 0xfb, 0x91, 0xfb, 0x13, 0x36, 0x32, 0xcb, 0x95, 0x64, 0x60, 0x7c, 0xe4, 0x5f,
0x71, 0x1e, 0xa8, 0x85, 0xab, 0x9d, 0xab, 0x8a, 0xee, 0x6a, 0xe7, 0xea, 0xc2, 0x7a, 0x2c, 0xc9, 0xc2, 0x6a, 0xe0, 0x2d, 0x07, 0x51, 0x7a, 0x54, 0xbd, 0x55, 0x29, 0x3f, 0x98, 0x00, 0x99, 0x18,
0xb0, 0x7e, 0x14, 0x49, 0xb2, 0x60, 0xcd, 0x31, 0x92, 0x64, 0xe1, 0x02, 0x62, 0x3c, 0xc9, 0x64, 0x3c, 0xd0, 0x24, 0x57, 0x07, 0x57, 0xb5, 0xe5, 0xd5, 0xc1, 0x95, 0x1d, 0xf7, 0x98, 0xe0, 0x81,
0xb1, 0x2b, 0x9a, 0x64, 0xa1, 0x0a, 0x5d, 0x34, 0xc9, 0xc2, 0x75, 0xb3, 0x44, 0x92, 0xc9, 0x01, 0x5e, 0xb8, 0x3a, 0xb8, 0xaa, 0xe9, 0xae, 0x0e, 0xae, 0x6e, 0xac, 0xc7, 0x92, 0x0c, 0xfb, 0x47,
0xc7, 0x90, 0x2c, 0x34, 0xe6, 0x87, 0x13, 0x20, 0x27, 0xcc, 0x73, 0xac, 0x73, 0x55, 0x91, 0x37, 0x91, 0x24, 0x0b, 0xf6, 0x1c, 0x23, 0x49, 0x16, 0x6e, 0x20, 0xc6, 0x93, 0x4c, 0x36, 0xbb, 0xa2,
0x2e, 0xcf, 0x13, 0x3a, 0x17, 0x79, 0xc6, 0xd3, 0x7e, 0x64, 0x9e, 0x83, 0x75, 0x94, 0xc8, 0x3c, 0x49, 0x16, 0xea, 0xd0, 0x45, 0x93, 0x2c, 0xdc, 0x37, 0x4b, 0x24, 0x99, 0x9c, 0x70, 0x0c, 0xc9,
0x87, 0x4a, 0x0d, 0x09, 0x79, 0x96, 0x85, 0xa8, 0xe8, 0x3c, 0x87, 0xaa, 0x67, 0xd1, 0x79, 0x0e, 0x42, 0x73, 0x7e, 0x30, 0x01, 0x72, 0xc2, 0x3a, 0xc7, 0x06, 0x57, 0x35, 0x79, 0xe3, 0xea, 0x3c,
0xd7, 0xb4, 0x12, 0xd7, 0xb3, 0x1c, 0x70, 0xcc, 0x7a, 0x0e, 0x8d, 0xf9, 0xe1, 0x04, 0xc8, 0xc4, 0x61, 0x70, 0x51, 0x67, 0xbc, 0xed, 0x47, 0xd6, 0x39, 0xd8, 0x47, 0x89, 0xac, 0x73, 0xa8, 0xd5,
0x8f, 0x93, 0x5b, 0x02, 0x51, 0x7f, 0x9c, 0xc2, 0x05, 0x96, 0xf2, 0xbd, 0x04, 0x54, 0xe2, 0x3c, 0x90, 0x50, 0x67, 0xd9, 0x88, 0x8a, 0xae, 0x73, 0xa8, 0x7b, 0x16, 0x5d, 0xe7, 0x70, 0x4f, 0x2b,
0xfb, 0xeb, 0x0d, 0xea, 0x79, 0x56, 0xd4, 0x52, 0xca, 0x95, 0x64, 0x60, 0xbc, 0xe7, 0x21, 0x14, 0x71, 0x3f, 0xcb, 0x09, 0xc7, 0xec, 0xe7, 0xd0, 0x9c, 0x1f, 0x4c, 0x80, 0x4c, 0xfc, 0x71, 0x72,
0x7c, 0xa7, 0x66, 0x72, 0x7f, 0xb2, 0x83, 0x7e, 0xf9, 0x41, 0x22, 0x2e, 0x71, 0xc0, 0xfe, 0x43, 0x5b, 0x20, 0xea, 0x1f, 0xa7, 0x70, 0x83, 0xa5, 0x7c, 0x37, 0x01, 0x95, 0xb8, 0xce, 0xfe, 0x7e,
0xb1, 0x7a, 0xc0, 0x8a, 0x13, 0x78, 0xb9, 0x92, 0x0c, 0x4c, 0xf4, 0xec, 0x3f, 0x00, 0xab, 0x3d, 0x83, 0x7a, 0x9d, 0x15, 0xbd, 0x94, 0x72, 0x25, 0x19, 0x18, 0x1f, 0x79, 0x08, 0x05, 0xdf, 0xad,
0x2b, 0x0e, 0xd9, 0xe5, 0x4a, 0x32, 0x70, 0x12, 0x56, 0x89, 0x2d, 0x74, 0x24, 0xab, 0x02, 0x7b, 0x99, 0xdc, 0x9b, 0xec, 0xa2, 0x5f, 0xbe, 0x9f, 0x88, 0x4b, 0x9c, 0xb0, 0xff, 0x52, 0xac, 0x9e,
0xf4, 0x48, 0x56, 0x05, 0xf7, 0xe1, 0x89, 0xac, 0x42, 0x9f, 0x31, 0xac, 0x0a, 0xba, 0xad, 0x24, 0xb0, 0xe2, 0x06, 0x5e, 0xae, 0x24, 0x03, 0x13, 0x23, 0xfb, 0x2f, 0xc0, 0xea, 0xc8, 0x8a, 0x4b,
0x03, 0x27, 0x62, 0x15, 0x1e, 0xab, 0xa2, 0x59, 0x15, 0x3c, 0x09, 0x46, 0xb3, 0x2a, 0x74, 0x3e, 0x76, 0xb9, 0x92, 0x0c, 0x9c, 0x84, 0x55, 0xe2, 0x08, 0x1d, 0xc9, 0xaa, 0xc0, 0x19, 0x3d, 0x92,
0x4b, 0x64, 0x55, 0xdc, 0x80, 0x15, 0x47, 0xb4, 0x38, 0x56, 0x4d, 0x3c, 0xd5, 0xfe, 0x13, 0x52, 0x55, 0xc1, 0x73, 0x78, 0x22, 0xab, 0x30, 0x66, 0x0c, 0xab, 0x82, 0x61, 0x2b, 0xc9, 0xc0, 0x89,
0x1c, 0xab, 0x26, 0xf0, 0xac, 0x3a, 0x6c, 0x45, 0x78, 0xde, 0x2d, 0xbd, 0xf9, 0x7a, 0x6b, 0xe1, 0x58, 0x85, 0xd7, 0xaa, 0x68, 0x56, 0x05, 0x6f, 0x82, 0xd1, 0xac, 0x0a, 0xdd, 0xcf, 0x12, 0x59,
0x1f, 0x5f, 0x6f, 0x2d, 0xfc, 0x72, 0xb4, 0x95, 0x7a, 0x33, 0xda, 0x4a, 0xfd, 0x7d, 0xb4, 0x95, 0x15, 0x37, 0x61, 0xc5, 0x15, 0x2d, 0x8e, 0x55, 0x13, 0x2f, 0xb5, 0xff, 0x86, 0x14, 0xc7, 0xaa,
0xfa, 0xd7, 0x68, 0x2b, 0x75, 0xbc, 0xc4, 0xff, 0x25, 0xf4, 0xd1, 0xff, 0x03, 0x00, 0x00, 0xff, 0x09, 0x22, 0xab, 0x2e, 0x5b, 0x11, 0x91, 0x77, 0x4b, 0x6f, 0xbe, 0xda, 0x5a, 0xf8, 0xe7, 0x57,
0xff, 0x47, 0x18, 0x50, 0x6c, 0x2b, 0x2b, 0x00, 0x00, 0x5b, 0x0b, 0xbf, 0x1a, 0x6d, 0xa5, 0xde, 0x8c, 0xb6, 0x52, 0xff, 0x18, 0x6d, 0xa5, 0xfe, 0x3d,
0xda, 0x4a, 0x1d, 0x2f, 0xf1, 0x7f, 0x58, 0x7d, 0xf8, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x96,
0x0e, 0xd9, 0x9f, 0xc9, 0x2b, 0x00, 0x00,
} }

View file

@ -175,7 +175,11 @@ message ListNodesRequest {
message Filters { message Filters {
repeated string names = 1; repeated string names = 1;
repeated string id_prefixes = 2; repeated string id_prefixes = 2;
// Labels refers to engine labels, which are labels set by the user on the
// node and reported back to the managers
map<string, string> labels = 3; map<string, string> labels = 3;
// NodeLabels are labels set on the node object on the managers.
map<string, string> node_labels = 7;
repeated NodeSpec.Membership memberships = 4 [packed=false]; repeated NodeSpec.Membership memberships = 4 [packed=false];
repeated NodeRole roles = 5 [packed=false]; repeated NodeRole roles = 5 [packed=false];
// NamePrefixes matches all objects with the given prefixes // NamePrefixes matches all objects with the given prefixes

View file

@ -274,6 +274,13 @@ type Cluster struct {
// reject joining the cluster. Nodes that report themselves to be non-FIPS // reject joining the cluster. Nodes that report themselves to be non-FIPS
// should be rejected from the cluster. // should be rejected from the cluster.
FIPS bool `protobuf:"varint,10,opt,name=fips,proto3" json:"fips,omitempty"` FIPS bool `protobuf:"varint,10,opt,name=fips,proto3" json:"fips,omitempty"`
// This field specifies default subnet pools for global scope networks. If
// unspecified, Docker will use the predefined subnets as it works on older releases.
// Format Example : {"20.20.0.0/16",""20.20.0.0/16"}
DefaultAddressPool []string `protobuf:"bytes,11,rep,name=defaultAddressPool" json:"defaultAddressPool,omitempty"`
// This flag specifies the default subnet size of global scope networks by giving
// the length of the subnet masks for every such network
SubnetSize uint32 `protobuf:"varint,12,opt,name=subnetSize,proto3" json:"subnetSize,omitempty"`
} }
func (m *Cluster) Reset() { *m = Cluster{} } func (m *Cluster) Reset() { *m = Cluster{} }
@ -658,6 +665,11 @@ func (m *Cluster) CopyFrom(src interface{}) {
} }
} }
if o.DefaultAddressPool != nil {
m.DefaultAddressPool = make([]string, len(o.DefaultAddressPool))
copy(m.DefaultAddressPool, o.DefaultAddressPool)
}
} }
func (m *Secret) Copy() *Secret { func (m *Secret) Copy() *Secret {
@ -1440,6 +1452,26 @@ func (m *Cluster) MarshalTo(dAtA []byte) (int, error) {
} }
i++ i++
} }
if len(m.DefaultAddressPool) > 0 {
for _, s := range m.DefaultAddressPool {
dAtA[i] = 0x5a
i++
l = len(s)
for l >= 1<<7 {
dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
l >>= 7
i++
}
dAtA[i] = uint8(l)
i++
i += copy(dAtA[i:], s)
}
}
if m.SubnetSize != 0 {
dAtA[i] = 0x60
i++
i = encodeVarintObjects(dAtA, i, uint64(m.SubnetSize))
}
return i, nil return i, nil
} }
@ -1924,6 +1956,15 @@ func (m *Cluster) Size() (n int) {
if m.FIPS { if m.FIPS {
n += 2 n += 2
} }
if len(m.DefaultAddressPool) > 0 {
for _, s := range m.DefaultAddressPool {
l = len(s)
n += 1 + l + sovObjects(uint64(l))
}
}
if m.SubnetSize != 0 {
n += 1 + sovObjects(uint64(m.SubnetSize))
}
return n return n
} }
@ -4804,6 +4845,8 @@ func (this *Cluster) String() string {
`BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`, `BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`,
`UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
`FIPS:` + fmt.Sprintf("%v", this.FIPS) + `,`, `FIPS:` + fmt.Sprintf("%v", this.FIPS) + `,`,
`DefaultAddressPool:` + fmt.Sprintf("%v", this.DefaultAddressPool) + `,`,
`SubnetSize:` + fmt.Sprintf("%v", this.SubnetSize) + `,`,
`}`, `}`,
}, "") }, "")
return s return s
@ -7241,6 +7284,54 @@ func (m *Cluster) Unmarshal(dAtA []byte) error {
} }
} }
m.FIPS = bool(v != 0) m.FIPS = bool(v != 0)
case 11:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DefaultAddressPool", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowObjects
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthObjects
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DefaultAddressPool = append(m.DefaultAddressPool, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 12:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SubnetSize", wireType)
}
m.SubnetSize = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowObjects
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.SubnetSize |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipObjects(dAtA[iNdEx:]) skippy, err := skipObjects(dAtA[iNdEx:])
@ -8037,102 +8128,104 @@ var (
func init() { proto.RegisterFile("github.com/docker/swarmkit/api/objects.proto", fileDescriptorObjects) } func init() { proto.RegisterFile("github.com/docker/swarmkit/api/objects.proto", fileDescriptorObjects) }
var fileDescriptorObjects = []byte{ var fileDescriptorObjects = []byte{
// 1544 bytes of a gzipped FileDescriptorProto // 1581 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4d, 0x73, 0xdb, 0x4c, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4b, 0x73, 0x1b, 0x4b,
0x1d, 0xaf, 0x6c, 0xc5, 0x2f, 0x7f, 0x27, 0x26, 0xec, 0x13, 0x82, 0x6a, 0x82, 0x1d, 0xfc, 0x0c, 0x15, 0xce, 0x48, 0x63, 0x3d, 0x8e, 0x6c, 0x61, 0xfa, 0x1a, 0x33, 0x11, 0x46, 0x32, 0xba, 0x05,
0xcc, 0x33, 0xcf, 0x74, 0x9c, 0x12, 0x0a, 0xa4, 0x81, 0xd2, 0xda, 0x49, 0x68, 0x3d, 0xa5, 0x34, 0x75, 0xeb, 0x56, 0x4a, 0xbe, 0x98, 0x0b, 0x38, 0x86, 0xcb, 0x8d, 0x64, 0x9b, 0x44, 0x15, 0x42,
0xb3, 0x29, 0x2d, 0x37, 0xb1, 0x91, 0x36, 0xae, 0xb0, 0xac, 0xd5, 0x68, 0xd7, 0x2e, 0xbe, 0xf5, 0x5c, 0xed, 0x90, 0xb0, 0x1b, 0x5a, 0x33, 0x6d, 0x65, 0xd0, 0x68, 0x7a, 0x6a, 0xba, 0xa5, 0x20,
0x1c, 0x3e, 0x40, 0x6e, 0x1c, 0xfa, 0x2d, 0xb8, 0x70, 0xe0, 0xd4, 0x23, 0xc3, 0x81, 0xe1, 0x94, 0x56, 0x59, 0x9b, 0x1f, 0xe0, 0x1d, 0x8b, 0xfc, 0x0b, 0x36, 0x2c, 0x58, 0x65, 0xc9, 0x8a, 0x62,
0xa1, 0xfe, 0x16, 0xcc, 0x70, 0x60, 0x76, 0xb5, 0xb2, 0x95, 0x58, 0x79, 0x63, 0x3a, 0x19, 0x4e, 0xe5, 0x22, 0xfa, 0x17, 0x54, 0xb1, 0xa0, 0xba, 0xa7, 0x47, 0x1a, 0x5b, 0xe3, 0x17, 0x95, 0x72,
0xd1, 0x6a, 0x7f, 0xbf, 0xff, 0x9b, 0xfe, 0x6f, 0x31, 0xdc, 0xeb, 0x79, 0xe2, 0xed, 0xf0, 0xb0, 0xb1, 0x72, 0x3f, 0xbe, 0xef, 0xf4, 0x39, 0x67, 0xce, 0xcb, 0x82, 0x07, 0x7d, 0x4f, 0xbc, 0x1e,
0xe5, 0xb0, 0xc1, 0x86, 0xcb, 0x9c, 0x3e, 0x8d, 0x36, 0xf8, 0x3b, 0x12, 0x0d, 0xfa, 0x9e, 0xd8, 0xf5, 0x5a, 0x0e, 0x1b, 0x6e, 0xb9, 0xcc, 0x19, 0xd0, 0x68, 0x8b, 0xbf, 0x21, 0xd1, 0x70, 0xe0,
0x20, 0xa1, 0xb7, 0xc1, 0x0e, 0x7f, 0x4f, 0x1d, 0xc1, 0x5b, 0x61, 0xc4, 0x04, 0x43, 0x28, 0x86, 0x89, 0x2d, 0x12, 0x7a, 0x5b, 0xac, 0xf7, 0x7b, 0xea, 0x08, 0xde, 0x0a, 0x23, 0x26, 0x18, 0x42,
0xb4, 0x12, 0x48, 0x6b, 0xf4, 0xc3, 0xda, 0xd7, 0x57, 0x48, 0x10, 0xe3, 0x90, 0x6a, 0xfe, 0x95, 0x31, 0xa4, 0x95, 0x40, 0x5a, 0xe3, 0x1f, 0xd6, 0x3e, 0xbf, 0x46, 0x82, 0x98, 0x84, 0x54, 0xf3,
0x58, 0x1e, 0x52, 0x27, 0xc1, 0x36, 0x7a, 0x8c, 0xf5, 0x7c, 0xba, 0xa1, 0x4e, 0x87, 0xc3, 0xa3, 0xaf, 0xc5, 0xf2, 0x90, 0x3a, 0x09, 0xb6, 0xd1, 0x67, 0xac, 0xef, 0xd3, 0x2d, 0xb5, 0xeb, 0x8d,
0x0d, 0xe1, 0x0d, 0x28, 0x17, 0x64, 0x10, 0x6a, 0xc0, 0x4a, 0x8f, 0xf5, 0x98, 0x7a, 0xdc, 0x90, 0x8e, 0xb7, 0x84, 0x37, 0xa4, 0x5c, 0x90, 0x61, 0xa8, 0x01, 0x6b, 0x7d, 0xd6, 0x67, 0x6a, 0xb9,
0x4f, 0xfa, 0xed, 0xdd, 0xf3, 0x34, 0x12, 0x8c, 0xf5, 0xd5, 0x4f, 0x2f, 0xd1, 0x3e, 0x85, 0x87, 0x25, 0x57, 0xfa, 0xf4, 0xfe, 0x45, 0x1a, 0x09, 0x26, 0xfa, 0xea, 0xa7, 0x57, 0xbc, 0x3e, 0x83,
0xfe, 0xb0, 0xe7, 0x05, 0xfa, 0x4f, 0x4c, 0x6c, 0xfe, 0xd9, 0x00, 0xf3, 0x05, 0x15, 0x04, 0xfd, 0x87, 0xfe, 0xa8, 0xef, 0x05, 0xfa, 0x4f, 0x4c, 0x6c, 0xfe, 0xc5, 0x00, 0xf3, 0x19, 0x15, 0x04,
0x0c, 0x8a, 0x23, 0x1a, 0x71, 0x8f, 0x05, 0x96, 0xb1, 0x6e, 0x7c, 0x55, 0xd9, 0xfc, 0x4e, 0x6b, 0xfd, 0x0c, 0x8a, 0x63, 0x1a, 0x71, 0x8f, 0x05, 0x96, 0xb1, 0x69, 0x7c, 0x56, 0xd9, 0xfe, 0x4e,
0x3e, 0x22, 0xad, 0xd7, 0x31, 0xa4, 0x63, 0x7e, 0x3c, 0x6d, 0xdc, 0xc1, 0x09, 0x03, 0x3d, 0x04, 0x6b, 0xd1, 0x23, 0xad, 0x97, 0x31, 0xa4, 0x63, 0xbe, 0x3f, 0x6b, 0xdc, 0xc3, 0x09, 0x03, 0x3d,
0x70, 0x22, 0x4a, 0x04, 0x75, 0x6d, 0x22, 0xac, 0x9c, 0xe2, 0xd7, 0x5a, 0xb1, 0xb9, 0xad, 0x44, 0x04, 0x70, 0x22, 0x4a, 0x04, 0x75, 0x6d, 0x22, 0xac, 0x9c, 0xe2, 0xd7, 0x5a, 0xb1, 0xba, 0xad,
0x7f, 0xeb, 0x55, 0xe2, 0x25, 0x2e, 0x6b, 0x74, 0x5b, 0x48, 0xea, 0x30, 0x74, 0x13, 0x6a, 0xfe, 0xe4, 0xfd, 0xd6, 0x8b, 0xc4, 0x4a, 0x5c, 0xd6, 0xe8, 0xb6, 0x90, 0xd4, 0x51, 0xe8, 0x26, 0xd4,
0x6a, 0xaa, 0x46, 0xb7, 0x45, 0xf3, 0xfd, 0x02, 0x98, 0xbf, 0x66, 0x2e, 0x45, 0xab, 0x90, 0xf3, 0xfc, 0xf5, 0x54, 0x8d, 0x6e, 0x8b, 0xe6, 0xdb, 0x25, 0x30, 0x7f, 0xcd, 0x5c, 0x8a, 0xd6, 0x21,
0x5c, 0x65, 0x76, 0xb9, 0x53, 0x98, 0x9c, 0x36, 0x72, 0xdd, 0x5d, 0x9c, 0xf3, 0x5c, 0xb4, 0x09, 0xe7, 0xb9, 0x4a, 0xed, 0x72, 0xa7, 0x30, 0x3d, 0x6b, 0xe4, 0xba, 0xfb, 0x38, 0xe7, 0xb9, 0x68,
0xe6, 0x80, 0x0a, 0xa2, 0x0d, 0xb2, 0xb2, 0x1c, 0x92, 0xbe, 0x6b, 0x6f, 0x14, 0x16, 0xfd, 0x04, 0x1b, 0xcc, 0x21, 0x15, 0x44, 0x2b, 0x64, 0x65, 0x19, 0x24, 0x6d, 0xd7, 0xd6, 0x28, 0x2c, 0xfa,
0x4c, 0xf9, 0xa9, 0xb4, 0x25, 0x6b, 0x59, 0x1c, 0xa9, 0xf3, 0x20, 0xa4, 0x4e, 0xc2, 0x93, 0x78, 0x09, 0x98, 0xf2, 0x53, 0x69, 0x4d, 0x36, 0xb2, 0x38, 0xf2, 0xcd, 0xa3, 0x90, 0x3a, 0x09, 0x4f,
0xb4, 0x07, 0x15, 0x97, 0x72, 0x27, 0xf2, 0x42, 0x21, 0x63, 0x68, 0x2a, 0xfa, 0x97, 0x17, 0xd1, 0xe2, 0xd1, 0x01, 0x54, 0x5c, 0xca, 0x9d, 0xc8, 0x0b, 0x85, 0xf4, 0xa1, 0xa9, 0xe8, 0x9f, 0x5e,
0x77, 0x67, 0x50, 0x9c, 0xe6, 0xa1, 0x9f, 0x43, 0x81, 0x0b, 0x22, 0x86, 0xdc, 0x5a, 0x50, 0x12, 0x46, 0xdf, 0x9f, 0x43, 0x71, 0x9a, 0x87, 0x7e, 0x0e, 0x05, 0x2e, 0x88, 0x18, 0x71, 0x6b, 0x49,
0xea, 0x17, 0x1a, 0xa0, 0x50, 0xda, 0x04, 0xcd, 0x41, 0xcf, 0xa0, 0x3a, 0x20, 0x01, 0xe9, 0xd1, 0x49, 0xa8, 0x5f, 0xaa, 0x80, 0x42, 0x69, 0x15, 0x34, 0x07, 0x3d, 0x81, 0xea, 0x90, 0x04, 0xa4,
0xc8, 0xd6, 0x52, 0x0a, 0x4a, 0xca, 0xf7, 0x32, 0x5d, 0x8f, 0x91, 0xb1, 0x20, 0xbc, 0x34, 0x48, 0x4f, 0x23, 0x5b, 0x4b, 0x29, 0x28, 0x29, 0xdf, 0xcb, 0x34, 0x3d, 0x46, 0xc6, 0x82, 0xf0, 0xca,
0x1f, 0x51, 0x17, 0x80, 0x08, 0x41, 0x9c, 0xb7, 0x03, 0x1a, 0x08, 0xab, 0xa8, 0xa4, 0x7c, 0x3f, 0x30, 0xbd, 0x45, 0x5d, 0x00, 0x22, 0x04, 0x71, 0x5e, 0x0f, 0x69, 0x20, 0xac, 0xa2, 0x92, 0xf2,
0xd3, 0x16, 0x2a, 0xde, 0xb1, 0xa8, 0xdf, 0x9e, 0x82, 0x3b, 0x39, 0xcb, 0xc0, 0x29, 0x32, 0x7a, 0xfd, 0x4c, 0x5d, 0xa8, 0x78, 0xc3, 0xa2, 0x41, 0x7b, 0x06, 0xee, 0xe4, 0x2c, 0x03, 0xa7, 0xc8,
0x0a, 0x15, 0x87, 0x46, 0xc2, 0x3b, 0xf2, 0x1c, 0x22, 0xa8, 0x55, 0x52, 0xb2, 0x1a, 0x59, 0xb2, 0xe8, 0x31, 0x54, 0x1c, 0x1a, 0x09, 0xef, 0xd8, 0x73, 0x88, 0xa0, 0x56, 0x49, 0xc9, 0x6a, 0x64,
0x76, 0x66, 0x30, 0xed, 0x58, 0x9a, 0x89, 0xee, 0x83, 0x19, 0x31, 0x9f, 0x5a, 0xe5, 0x75, 0xe3, 0xc9, 0xda, 0x9b, 0xc3, 0xb4, 0x61, 0x69, 0x26, 0xfa, 0x02, 0xcc, 0x88, 0xf9, 0xd4, 0x2a, 0x6f,
0xab, 0xea, 0xc5, 0x9f, 0x06, 0x33, 0x9f, 0x62, 0x85, 0x94, 0xaa, 0x67, 0x86, 0x70, 0x0b, 0xd6, 0x1a, 0x9f, 0x55, 0x2f, 0xff, 0x34, 0x98, 0xf9, 0x14, 0x2b, 0xa4, 0x7c, 0x7a, 0xae, 0x08, 0xb7,
0xf3, 0xd7, 0x76, 0x03, 0xa7, 0x99, 0xdb, 0xab, 0xc7, 0x27, 0x4d, 0x04, 0xcb, 0x25, 0x63, 0xd9, 0x60, 0x33, 0x7f, 0x63, 0x33, 0x70, 0x9a, 0xb9, 0xbb, 0x7e, 0x72, 0xda, 0x44, 0xb0, 0x5a, 0x32,
0x50, 0x79, 0x66, 0xdc, 0x37, 0x7e, 0x6b, 0xfc, 0xce, 0x68, 0xfe, 0x27, 0x0f, 0xc5, 0x03, 0x1a, 0x56, 0x0d, 0x15, 0x67, 0xc6, 0x17, 0xc6, 0x6f, 0x8d, 0xdf, 0x19, 0xcd, 0xff, 0xe4, 0xa1, 0x78,
0x8d, 0x3c, 0xe7, 0xf3, 0x66, 0xe1, 0xc3, 0x33, 0x59, 0x98, 0x19, 0x2c, 0xad, 0x76, 0x2e, 0x11, 0x44, 0xa3, 0xb1, 0xe7, 0x7c, 0xdc, 0x28, 0x7c, 0x78, 0x2e, 0x0a, 0x33, 0x9d, 0xa5, 0x9f, 0x5d,
0xb7, 0xa0, 0x44, 0x03, 0x37, 0x64, 0x5e, 0x20, 0x74, 0x16, 0x66, 0x46, 0x6a, 0x4f, 0x63, 0xf0, 0x08, 0xc4, 0x1d, 0x28, 0xd1, 0xc0, 0x0d, 0x99, 0x17, 0x08, 0x1d, 0x85, 0x99, 0x9e, 0x3a, 0xd0,
0x14, 0x8d, 0xf6, 0x60, 0x29, 0x2e, 0x2e, 0xfb, 0x4c, 0x0a, 0xae, 0x67, 0xd1, 0x7f, 0xa3, 0x80, 0x18, 0x3c, 0x43, 0xa3, 0x03, 0x58, 0x89, 0x93, 0xcb, 0x3e, 0x17, 0x82, 0x9b, 0x59, 0xf4, 0xdf,
0x3a, 0x77, 0x16, 0x87, 0xa9, 0x13, 0xda, 0x85, 0xa5, 0x30, 0xa2, 0x23, 0x8f, 0x0d, 0xb9, 0xad, 0x28, 0xa0, 0x8e, 0x9d, 0xe5, 0x51, 0x6a, 0x87, 0xf6, 0x61, 0x25, 0x8c, 0xe8, 0xd8, 0x63, 0x23,
0x9c, 0x28, 0x5c, 0xcb, 0x09, 0xbc, 0x98, 0xb0, 0xe4, 0x09, 0xfd, 0x02, 0x16, 0x25, 0xd9, 0x4e, 0x6e, 0x2b, 0x23, 0x0a, 0x37, 0x32, 0x02, 0x2f, 0x27, 0x2c, 0xb9, 0x43, 0xbf, 0x80, 0x65, 0x49,
0x9a, 0x12, 0x5c, 0xd9, 0x94, 0x70, 0x45, 0x12, 0xf4, 0x01, 0xbd, 0x84, 0x6f, 0x9d, 0xb1, 0x62, 0xb6, 0x93, 0xa2, 0x04, 0xd7, 0x16, 0x25, 0x5c, 0x91, 0x04, 0xbd, 0x41, 0xcf, 0xe1, 0x5b, 0xe7,
0x2a, 0xa8, 0x72, 0xb5, 0xa0, 0x2f, 0xd2, 0x96, 0xe8, 0x97, 0xdb, 0xe8, 0xf8, 0xa4, 0x59, 0x85, 0xb4, 0x98, 0x09, 0xaa, 0x5c, 0x2f, 0xe8, 0x93, 0xb4, 0x26, 0xfa, 0x70, 0x17, 0x9d, 0x9c, 0x36,
0xc5, 0x74, 0x0a, 0x34, 0xff, 0x94, 0x83, 0x52, 0x12, 0x48, 0xf4, 0x40, 0x7f, 0x33, 0xe3, 0xe2, 0xab, 0xb0, 0x9c, 0x0e, 0x81, 0xe6, 0x9f, 0x73, 0x50, 0x4a, 0x1c, 0x89, 0xbe, 0xd4, 0xdf, 0xcc,
0xa8, 0x25, 0x58, 0xe5, 0x6f, 0xfc, 0xb9, 0x1e, 0xc0, 0x42, 0xc8, 0x22, 0xc1, 0xad, 0x9c, 0x4a, 0xb8, 0xdc, 0x6b, 0x09, 0x56, 0xd9, 0x1b, 0x7f, 0xae, 0x2f, 0x61, 0x29, 0x64, 0x91, 0xe0, 0x56,
0xce, 0xcc, 0x7a, 0xdf, 0x67, 0x91, 0xd8, 0x61, 0xc1, 0x91, 0xd7, 0xc3, 0x31, 0x18, 0xbd, 0x81, 0x4e, 0x05, 0x67, 0x66, 0xbe, 0x1f, 0xb2, 0x48, 0xec, 0xb1, 0xe0, 0xd8, 0xeb, 0xe3, 0x18, 0x8c,
0xca, 0xc8, 0x8b, 0xc4, 0x90, 0xf8, 0xb6, 0x17, 0x72, 0x2b, 0xaf, 0xb8, 0x3f, 0xb8, 0x4c, 0x65, 0x5e, 0x41, 0x65, 0xec, 0x45, 0x62, 0x44, 0x7c, 0xdb, 0x0b, 0xb9, 0x95, 0x57, 0xdc, 0x1f, 0x5c,
0xeb, 0x75, 0x8c, 0xef, 0xee, 0x77, 0xaa, 0x93, 0xd3, 0x06, 0x4c, 0x8f, 0x1c, 0x83, 0x16, 0xd5, 0xf5, 0x64, 0xeb, 0x65, 0x8c, 0xef, 0x1e, 0x76, 0xaa, 0xd3, 0xb3, 0x06, 0xcc, 0xb6, 0x1c, 0x83,
0x0d, 0x79, 0xed, 0x05, 0x94, 0xa7, 0x37, 0xe8, 0x1e, 0x40, 0x10, 0xd7, 0x85, 0x3d, 0xcd, 0xec, 0x16, 0xd5, 0x0d, 0x79, 0xed, 0x19, 0x94, 0x67, 0x37, 0xe8, 0x01, 0x40, 0x10, 0xe7, 0x85, 0x3d,
0xa5, 0xc9, 0x69, 0xa3, 0xac, 0xab, 0xa5, 0xbb, 0x8b, 0xcb, 0x1a, 0xd0, 0x75, 0x11, 0x02, 0x93, 0x8b, 0xec, 0x95, 0xe9, 0x59, 0xa3, 0xac, 0xb3, 0xa5, 0xbb, 0x8f, 0xcb, 0x1a, 0xd0, 0x75, 0x11,
0xb8, 0x6e, 0xa4, 0xf2, 0xbc, 0x8c, 0xd5, 0x73, 0xf3, 0x8f, 0x45, 0x30, 0x5f, 0x11, 0xde, 0xbf, 0x02, 0x93, 0xb8, 0x6e, 0xa4, 0xe2, 0xbc, 0x8c, 0xd5, 0xba, 0xf9, 0xa7, 0x22, 0x98, 0x2f, 0x08,
0xed, 0x16, 0x2d, 0x75, 0xce, 0x55, 0xc6, 0x3d, 0x00, 0x1e, 0xe7, 0x9b, 0x74, 0xc7, 0x9c, 0xb9, 0x1f, 0xdc, 0x75, 0x89, 0x96, 0x6f, 0x2e, 0x64, 0xc6, 0x03, 0x00, 0x1e, 0xc7, 0x9b, 0x34, 0xc7,
0xa3, 0xb3, 0x50, 0xba, 0xa3, 0x01, 0xb1, 0x3b, 0xdc, 0x67, 0x42, 0x15, 0x81, 0x89, 0xd5, 0x33, 0x9c, 0x9b, 0xa3, 0xa3, 0x50, 0x9a, 0xa3, 0x01, 0xb1, 0x39, 0xdc, 0x67, 0x42, 0x25, 0x81, 0x89,
0xfa, 0x12, 0x8a, 0x01, 0x73, 0x15, 0xbd, 0xa0, 0xe8, 0x30, 0x39, 0x6d, 0x14, 0x64, 0xd3, 0xe9, 0xd5, 0x1a, 0x7d, 0x0a, 0xc5, 0x80, 0xb9, 0x8a, 0x5e, 0x50, 0x74, 0x98, 0x9e, 0x35, 0x0a, 0xb2,
0xee, 0xe2, 0x82, 0xbc, 0xea, 0xba, 0xaa, 0xe9, 0x04, 0x01, 0x13, 0x44, 0x36, 0x74, 0xae, 0x7b, 0xe8, 0x74, 0xf7, 0x71, 0x41, 0x5e, 0x75, 0x5d, 0x55, 0x74, 0x82, 0x80, 0x09, 0x22, 0x0b, 0x3a,
0x67, 0x66, 0xf6, 0xb7, 0x67, 0xb0, 0xa4, 0xdf, 0xa5, 0x98, 0xe8, 0x35, 0x7c, 0x91, 0xd8, 0x9b, 0xd7, 0xb5, 0x33, 0x33, 0xfa, 0xdb, 0x73, 0x58, 0x52, 0xef, 0x52, 0x4c, 0xf4, 0x12, 0x3e, 0x49,
0x16, 0x58, 0xba, 0x89, 0x40, 0xa4, 0x25, 0xa4, 0x6e, 0x52, 0x33, 0xa6, 0x7c, 0xf1, 0x8c, 0x51, 0xf4, 0x4d, 0x0b, 0x2c, 0xdd, 0x46, 0x20, 0xd2, 0x12, 0x52, 0x37, 0xa9, 0x1e, 0x53, 0xbe, 0xbc,
0x11, 0xcc, 0x9a, 0x31, 0x1d, 0x58, 0x72, 0x29, 0xf7, 0x22, 0xea, 0xaa, 0x36, 0x41, 0x55, 0x65, 0xc7, 0x28, 0x0f, 0x66, 0xf5, 0x98, 0x0e, 0xac, 0xb8, 0x94, 0x7b, 0x11, 0x75, 0x55, 0x99, 0xa0,
0x56, 0x37, 0xbf, 0x7b, 0x99, 0x10, 0x8a, 0x17, 0x35, 0x47, 0x9d, 0x50, 0x1b, 0x4a, 0x3a, 0x6f, 0x2a, 0x33, 0xab, 0xdb, 0xdf, 0xbd, 0x4a, 0x08, 0xc5, 0xcb, 0x9a, 0xa3, 0x76, 0xa8, 0x0d, 0x25,
0xb8, 0x55, 0xb9, 0x49, 0x53, 0x9e, 0xd2, 0xce, 0xb4, 0xb9, 0xc5, 0x1b, 0xb5, 0xb9, 0x87, 0x00, 0x1d, 0x37, 0xdc, 0xaa, 0xdc, 0xa6, 0x28, 0xcf, 0x68, 0xe7, 0xca, 0xdc, 0xf2, 0xad, 0xca, 0xdc,
0x3e, 0xeb, 0xd9, 0x6e, 0xe4, 0x8d, 0x68, 0x64, 0x2d, 0xe9, 0x8d, 0x23, 0x83, 0xbb, 0xab, 0x10, 0x43, 0x00, 0x9f, 0xf5, 0x6d, 0x37, 0xf2, 0xc6, 0x34, 0xb2, 0x56, 0xf4, 0xc4, 0x91, 0xc1, 0xdd,
0xb8, 0xec, 0xb3, 0x5e, 0xfc, 0x38, 0xd7, 0x94, 0xaa, 0x37, 0x6c, 0x4a, 0x04, 0x6a, 0x84, 0x73, 0x57, 0x08, 0x5c, 0xf6, 0x59, 0x3f, 0x5e, 0x2e, 0x14, 0xa5, 0xea, 0x2d, 0x8b, 0x12, 0x81, 0x1a,
0xaf, 0x17, 0x50, 0xd7, 0xee, 0xd1, 0x80, 0x46, 0x9e, 0x63, 0x47, 0x94, 0xb3, 0x61, 0xe4, 0x50, 0xe1, 0xdc, 0xeb, 0x07, 0xd4, 0xb5, 0xfb, 0x34, 0xa0, 0x91, 0xe7, 0xd8, 0x11, 0xe5, 0x6c, 0x14,
0x6e, 0x7d, 0x43, 0x45, 0x22, 0x73, 0x67, 0x78, 0x1a, 0x83, 0xb1, 0xc6, 0x62, 0x2b, 0x11, 0x73, 0x39, 0x94, 0x5b, 0xdf, 0x50, 0x9e, 0xc8, 0x9c, 0x19, 0x1e, 0xc7, 0x60, 0xac, 0xb1, 0xd8, 0x4a,
0xee, 0x82, 0x6f, 0xd7, 0x8e, 0x4f, 0x9a, 0xab, 0xb0, 0x92, 0x6e, 0x53, 0x5b, 0xc6, 0x13, 0xe3, 0xc4, 0x5c, 0xb8, 0xe0, 0xbb, 0xb5, 0x93, 0xd3, 0xe6, 0x3a, 0xac, 0xa5, 0xcb, 0xd4, 0x8e, 0xf1,
0x99, 0xb1, 0x6f, 0x34, 0xff, 0x9a, 0x83, 0x6f, 0xce, 0xc5, 0x14, 0xfd, 0x18, 0x8a, 0x3a, 0xaa, 0xc8, 0x78, 0x62, 0x1c, 0x1a, 0xcd, 0xbf, 0xe5, 0xe0, 0x9b, 0x0b, 0x3e, 0x45, 0x3f, 0x86, 0xa2,
0x97, 0x6d, 0x7e, 0x9a, 0x87, 0x13, 0x2c, 0x5a, 0x83, 0xb2, 0x2c, 0x71, 0xca, 0x39, 0x8d, 0x9b, 0xf6, 0xea, 0x55, 0x93, 0x9f, 0xe6, 0xe1, 0x04, 0x8b, 0x36, 0xa0, 0x2c, 0x53, 0x9c, 0x72, 0x4e,
0x57, 0x19, 0xcf, 0x5e, 0x20, 0x0b, 0x8a, 0xc4, 0xf7, 0x88, 0xbc, 0xcb, 0xab, 0xbb, 0xe4, 0x88, 0xe3, 0xe2, 0x55, 0xc6, 0xf3, 0x03, 0x64, 0x41, 0x91, 0xf8, 0x1e, 0x91, 0x77, 0x79, 0x75, 0x97,
0x86, 0xb0, 0x1a, 0x87, 0xde, 0x9e, 0x0d, 0x58, 0x9b, 0x85, 0x82, 0x5b, 0xa6, 0xf2, 0xff, 0xf1, 0x6c, 0xd1, 0x08, 0xd6, 0x63, 0xd7, 0xdb, 0xf3, 0x06, 0x6b, 0xb3, 0x50, 0x70, 0xcb, 0x54, 0xf6,
0xb5, 0x32, 0x41, 0x7f, 0x9c, 0xd9, 0x8b, 0x97, 0xa1, 0xe0, 0x7b, 0x81, 0x88, 0xc6, 0x78, 0xc5, 0x7f, 0x7d, 0xa3, 0x48, 0xd0, 0x1f, 0x67, 0x7e, 0xf0, 0x3c, 0x14, 0xfc, 0x20, 0x10, 0xd1, 0x04,
0xcd, 0xb8, 0xaa, 0x3d, 0x85, 0xbb, 0x17, 0x52, 0xd0, 0x32, 0xe4, 0xfb, 0x74, 0x1c, 0xb7, 0x27, 0xaf, 0xb9, 0x19, 0x57, 0xb5, 0xc7, 0x70, 0xff, 0x52, 0x0a, 0x5a, 0x85, 0xfc, 0x80, 0x4e, 0xe2,
0x2c, 0x1f, 0xd1, 0x0a, 0x2c, 0x8c, 0x88, 0x3f, 0xa4, 0xba, 0x9b, 0xc5, 0x87, 0xed, 0xdc, 0x96, 0xf2, 0x84, 0xe5, 0x12, 0xad, 0xc1, 0xd2, 0x98, 0xf8, 0x23, 0xaa, 0xab, 0x59, 0xbc, 0xd9, 0xcd,
0xd1, 0xfc, 0x90, 0x83, 0xa2, 0x36, 0xe7, 0xb6, 0x47, 0xbe, 0x56, 0x3b, 0xd7, 0xd8, 0x1e, 0xc1, 0xed, 0x18, 0xcd, 0x77, 0x39, 0x28, 0x6a, 0x75, 0xee, 0xba, 0xe5, 0xeb, 0x67, 0x17, 0x0a, 0xdb,
0xa2, 0x0e, 0x69, 0x5c, 0x91, 0xe6, 0x95, 0x39, 0x5d, 0x89, 0xf1, 0x71, 0x35, 0x3e, 0x02, 0xd3, 0x57, 0xb0, 0xac, 0x5d, 0x1a, 0x67, 0xa4, 0x79, 0x6d, 0x4c, 0x57, 0x62, 0x7c, 0x9c, 0x8d, 0x5f,
0x0b, 0xc9, 0x40, 0x8f, 0xfb, 0x4c, 0xcd, 0xdd, 0xfd, 0xf6, 0x8b, 0x97, 0x61, 0xdc, 0x58, 0x4a, 0x81, 0xe9, 0x85, 0x64, 0xa8, 0xdb, 0x7d, 0xe6, 0xcb, 0xdd, 0xc3, 0xf6, 0xb3, 0xe7, 0x61, 0x5c,
0x93, 0xd3, 0x86, 0x29, 0x5f, 0x60, 0x45, 0xcb, 0x1c, 0x8c, 0x7f, 0x5f, 0x80, 0xe2, 0x8e, 0x3f, 0x58, 0x4a, 0xd3, 0xb3, 0x86, 0x29, 0x0f, 0xb0, 0xa2, 0x65, 0x36, 0xc6, 0x77, 0x05, 0x28, 0xee,
0xe4, 0x82, 0x46, 0xb7, 0x1d, 0x24, 0xad, 0x76, 0x2e, 0x48, 0x3b, 0x50, 0x8c, 0x18, 0x13, 0xb6, 0xf9, 0x23, 0x2e, 0x68, 0x74, 0xd7, 0x4e, 0xd2, 0xcf, 0x2e, 0x38, 0x69, 0x0f, 0x8a, 0x11, 0x63,
0x43, 0x2e, 0x8b, 0x0f, 0x66, 0x4c, 0xec, 0xb4, 0x3b, 0x55, 0x49, 0x94, 0xbd, 0x3d, 0x3e, 0xe3, 0xc2, 0x76, 0xc8, 0x55, 0xfe, 0xc1, 0x8c, 0x89, 0xbd, 0x76, 0xa7, 0x2a, 0x89, 0xb2, 0xb6, 0xc7,
0x82, 0xa4, 0xee, 0x10, 0xf4, 0x06, 0x56, 0x93, 0x89, 0x78, 0xc8, 0x98, 0xe0, 0x22, 0x22, 0xa1, 0x7b, 0x5c, 0x90, 0xd4, 0x3d, 0x82, 0x5e, 0xc1, 0x7a, 0xd2, 0x11, 0x7b, 0x8c, 0x09, 0x2e, 0x22,
0xdd, 0xa7, 0x63, 0xb9, 0x2b, 0xe5, 0x2f, 0x5a, 0xb4, 0xf7, 0x02, 0x27, 0x1a, 0xab, 0xe0, 0x3d, 0x12, 0xda, 0x03, 0x3a, 0x91, 0xb3, 0x52, 0xfe, 0xb2, 0x41, 0xfb, 0x20, 0x70, 0xa2, 0x89, 0x72,
0xa7, 0x63, 0xbc, 0xa2, 0x05, 0x74, 0x12, 0xfe, 0x73, 0x3a, 0xe6, 0xe8, 0x31, 0xac, 0xd1, 0x29, 0xde, 0x53, 0x3a, 0xc1, 0x6b, 0x5a, 0x40, 0x27, 0xe1, 0x3f, 0xa5, 0x13, 0x8e, 0xbe, 0x86, 0x0d,
0x4c, 0x4a, 0xb4, 0x7d, 0x32, 0x90, 0xb3, 0xde, 0x76, 0x7c, 0xe6, 0xf4, 0xd5, 0xb8, 0x31, 0xf1, 0x3a, 0x83, 0x49, 0x89, 0xb6, 0x4f, 0x86, 0xb2, 0xd7, 0xdb, 0x8e, 0xcf, 0x9c, 0x81, 0x6a, 0x37,
0x5d, 0x9a, 0x16, 0xf5, 0xab, 0x18, 0xb1, 0x23, 0x01, 0x88, 0x83, 0x75, 0xe8, 0x13, 0xa7, 0xef, 0x26, 0xbe, 0x4f, 0xd3, 0xa2, 0x7e, 0x15, 0x23, 0xf6, 0x24, 0x00, 0x71, 0xb0, 0x7a, 0x3e, 0x71,
0x7b, 0x5c, 0xfe, 0x2f, 0x95, 0xda, 0x9b, 0xe5, 0xc4, 0x90, 0xb6, 0x6d, 0x5d, 0x12, 0xad, 0x56, 0x06, 0xbe, 0xc7, 0xe5, 0xff, 0x52, 0xa9, 0xb9, 0x59, 0x76, 0x0c, 0xa9, 0xdb, 0xce, 0x15, 0xde,
0x67, 0xc6, 0x4d, 0x6d, 0xe1, 0xba, 0xa2, 0xbe, 0x7d, 0x98, 0x7d, 0x8b, 0x3a, 0x50, 0x19, 0x06, 0x6a, 0x75, 0xe6, 0xdc, 0xd4, 0x14, 0xae, 0x33, 0xea, 0xdb, 0xbd, 0xec, 0x5b, 0xd4, 0x81, 0xca,
0x52, 0x7d, 0x1c, 0x83, 0xf2, 0x75, 0x63, 0x00, 0x31, 0x4b, 0x79, 0xbe, 0x06, 0xe6, 0x91, 0xdc, 0x28, 0x90, 0xcf, 0xc7, 0x3e, 0x28, 0xdf, 0xd4, 0x07, 0x10, 0xb3, 0x94, 0xe5, 0x1b, 0x60, 0x1e,
0x61, 0xe4, 0x18, 0x29, 0xc5, 0xc9, 0xf5, 0xcb, 0xee, 0xfe, 0x01, 0x56, 0x6f, 0x6b, 0x23, 0x58, 0xcb, 0x19, 0x46, 0xb6, 0x91, 0x52, 0x1c, 0x5c, 0xbf, 0xec, 0x1e, 0x1e, 0x61, 0x75, 0x8a, 0x5a,
0xbb, 0xcc, 0xb4, 0x8c, 0xca, 0x7d, 0x92, 0xae, 0xdc, 0xca, 0xe6, 0xd7, 0x59, 0xd6, 0x64, 0x8b, 0x80, 0x5c, 0x7a, 0x4c, 0x46, 0xbe, 0x68, 0xc7, 0xb5, 0xe5, 0x90, 0x31, 0x5f, 0xf5, 0x8c, 0x32,
0x4c, 0x55, 0x79, 0x66, 0x52, 0xff, 0xc5, 0x80, 0xc2, 0x01, 0x75, 0x22, 0x2a, 0x3e, 0x6b, 0x4e, 0xce, 0xb8, 0x41, 0x75, 0x00, 0x3e, 0xea, 0x05, 0x54, 0x1c, 0x79, 0x7f, 0xa4, 0xaa, 0x31, 0xac,
0x6f, 0x9d, 0xc9, 0xe9, 0x7a, 0xf6, 0x9a, 0x2c, 0xb5, 0xce, 0xa5, 0x74, 0x0d, 0x4a, 0x5e, 0x20, 0xe0, 0xd4, 0x49, 0x6d, 0x0c, 0x1b, 0x57, 0x99, 0x9a, 0x51, 0x09, 0x1e, 0xa5, 0x2b, 0x41, 0x65,
0x68, 0x14, 0x10, 0x5f, 0xe5, 0x74, 0x09, 0x4f, 0xcf, 0x99, 0x0e, 0x7c, 0x30, 0xa0, 0x10, 0xef, 0xfb, 0xf3, 0x2c, 0xeb, 0xb2, 0x45, 0xa6, 0xaa, 0x46, 0x66, 0x92, 0xfc, 0xd5, 0x80, 0xc2, 0x11,
0x91, 0xb7, 0xed, 0x40, 0xac, 0xf5, 0xbc, 0x03, 0x99, 0x46, 0xfe, 0xdb, 0x80, 0x52, 0x32, 0xce, 0x75, 0x22, 0x2a, 0x3e, 0x6a, 0x8e, 0xec, 0x9c, 0xcb, 0x91, 0x7a, 0xf6, 0xd8, 0x2d, 0x5f, 0x5d,
0x3e, 0xab, 0x99, 0xe7, 0xf6, 0xb2, 0xfc, 0xff, 0xbc, 0x97, 0x21, 0x30, 0xfb, 0x5e, 0xa0, 0x37, 0x48, 0x91, 0x1a, 0x94, 0xbc, 0x40, 0xd0, 0x28, 0x20, 0xbe, 0xca, 0x91, 0x12, 0x9e, 0xed, 0xb3,
0x48, 0xac, 0x9e, 0x51, 0x0b, 0x8a, 0x21, 0x19, 0xfb, 0x8c, 0xb8, 0xba, 0x8d, 0xae, 0xcc, 0xfd, 0xb3, 0xdc, 0x80, 0x42, 0x3c, 0x97, 0xde, 0xb5, 0x01, 0xf1, 0xab, 0x17, 0x0d, 0xc8, 0x54, 0xf2,
0x86, 0xd1, 0x0e, 0xc6, 0x38, 0x01, 0x6d, 0xaf, 0x1c, 0x9f, 0x34, 0x97, 0xa1, 0x9a, 0xf6, 0xfc, 0xdf, 0x06, 0x94, 0x92, 0xf6, 0xf8, 0x51, 0xd5, 0xbc, 0x30, 0xe7, 0xe5, 0xff, 0xe7, 0x39, 0x0f,
0xad, 0xd1, 0xfc, 0x87, 0x01, 0xe5, 0xbd, 0x3f, 0x08, 0x1a, 0xa8, 0x6d, 0xe1, 0xff, 0xd2, 0xf9, 0x81, 0x39, 0xf0, 0x02, 0x3d, 0x91, 0x62, 0xb5, 0x46, 0x2d, 0x28, 0x86, 0x64, 0xe2, 0x33, 0xe2,
0xf5, 0xf9, 0xdf, 0x39, 0xca, 0x67, 0x7e, 0xc2, 0xc8, 0xfa, 0xa8, 0x1d, 0xeb, 0xe3, 0xa7, 0xfa, 0xea, 0xb2, 0xbc, 0xb6, 0xf0, 0x9b, 0x48, 0x3b, 0x98, 0xe0, 0x04, 0xb4, 0xbb, 0x76, 0x72, 0xda,
0x9d, 0x7f, 0x7e, 0xaa, 0xdf, 0x79, 0x3f, 0xa9, 0x1b, 0x1f, 0x27, 0x75, 0xe3, 0x6f, 0x93, 0xba, 0x5c, 0x85, 0x6a, 0xda, 0xf2, 0xd7, 0x46, 0xf3, 0x1f, 0x06, 0x94, 0x0f, 0xfe, 0x20, 0x68, 0xa0,
0xf1, 0xaf, 0x49, 0xdd, 0x38, 0x2c, 0xa8, 0xf8, 0xfc, 0xe8, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x8f, 0xff, 0x4b, 0xe3, 0x37, 0x17, 0x7f, 0x37, 0x29, 0x9f, 0xfb, 0x49, 0x24, 0xeb, 0xa3,
0x07, 0xc5, 0x5a, 0x5b, 0xae, 0x13, 0x00, 0x00, 0x76, 0xac, 0xf7, 0x1f, 0xea, 0xf7, 0xfe, 0xf9, 0xa1, 0x7e, 0xef, 0xed, 0xb4, 0x6e, 0xbc, 0x9f,
0xd6, 0x8d, 0xbf, 0x4f, 0xeb, 0xc6, 0xbf, 0xa6, 0x75, 0xa3, 0x57, 0x50, 0xfe, 0xf9, 0xd1, 0x7f,
0x03, 0x00, 0x00, 0xff, 0xff, 0xbc, 0x6b, 0x9b, 0xd8, 0xfe, 0x13, 0x00, 0x00,
} }

View file

@ -342,6 +342,15 @@ message Cluster {
// reject joining the cluster. Nodes that report themselves to be non-FIPS // reject joining the cluster. Nodes that report themselves to be non-FIPS
// should be rejected from the cluster. // should be rejected from the cluster.
bool fips = 10 [(gogoproto.customname) = "FIPS"]; bool fips = 10 [(gogoproto.customname) = "FIPS"];
// This field specifies default subnet pools for global scope networks. If
// unspecified, Docker will use the predefined subnets as it works on older releases.
// Format Example : {"20.20.0.0/16",""20.20.0.0/16"}
repeated string defaultAddressPool = 11;
// This flag specifies the default subnet size of global scope networks by giving
// the length of the subnet masks for every such network
uint32 subnetSize = 12;
} }
// Secret represents a secret that should be passed to a container or a node, // Secret represents a secret that should be passed to a container or a node,

View file

@ -1,6 +1,7 @@
package allocator package allocator
import ( import (
"net"
"sync" "sync"
"github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/plugingetter"
@ -32,6 +33,13 @@ type Allocator struct {
// pluginGetter provides access to docker's plugin inventory. // pluginGetter provides access to docker's plugin inventory.
pluginGetter plugingetter.PluginGetter pluginGetter plugingetter.PluginGetter
// DefaultAddrPool specifies default subnet pool for global scope networks
defaultAddrPool []*net.IPNet
// SubnetSize specifies the subnet size of the networks created from
// the default subnet pool
subnetSize int
} }
// taskBallot controls how the voting for task allocation is // taskBallot controls how the voting for task allocation is
@ -65,15 +73,17 @@ type allocActor struct {
// New returns a new instance of Allocator for use during allocation // New returns a new instance of Allocator for use during allocation
// stage of the manager. // stage of the manager.
func New(store *store.MemoryStore, pg plugingetter.PluginGetter) (*Allocator, error) { func New(store *store.MemoryStore, pg plugingetter.PluginGetter, defaultAddrPool []*net.IPNet, subnetSize int) (*Allocator, error) {
a := &Allocator{ a := &Allocator{
store: store, store: store,
taskBallot: &taskBallot{ taskBallot: &taskBallot{
votes: make(map[string][]string), votes: make(map[string][]string),
}, },
stopChan: make(chan struct{}), stopChan: make(chan struct{}),
doneChan: make(chan struct{}), doneChan: make(chan struct{}),
pluginGetter: pg, pluginGetter: pg,
defaultAddrPool: defaultAddrPool,
subnetSize: subnetSize,
} }
return a, nil return a, nil

View file

@ -1,14 +1,45 @@
package cnmallocator package cnmallocator
import ( import (
"net"
"strconv"
"strings"
"github.com/docker/libnetwork/drvregistry" "github.com/docker/libnetwork/drvregistry"
"github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipamapi"
builtinIpam "github.com/docker/libnetwork/ipams/builtin" builtinIpam "github.com/docker/libnetwork/ipams/builtin"
nullIpam "github.com/docker/libnetwork/ipams/null" nullIpam "github.com/docker/libnetwork/ipams/null"
remoteIpam "github.com/docker/libnetwork/ipams/remote" remoteIpam "github.com/docker/libnetwork/ipams/remote"
"github.com/docker/libnetwork/ipamutils"
"github.com/sirupsen/logrus"
) )
func initIPAMDrivers(r *drvregistry.DrvRegistry) error { func initIPAMDrivers(r *drvregistry.DrvRegistry, defaultAddrPool []*net.IPNet, subnetSize int) error {
var addressPool []*ipamutils.NetworkToSplit
var str strings.Builder
str.WriteString("Subnetlist - ")
// Extract defaultAddrPool param info and construct ipamutils.NetworkToSplit
// from the info. We will be using it to call Libnetwork API
// We also need to log new address pool info whenever swarm init
// happens with default address pool option
if defaultAddrPool != nil {
for _, p := range defaultAddrPool {
addressPool = append(addressPool, &ipamutils.NetworkToSplit{
Base: p.String(),
Size: subnetSize,
})
str.WriteString(p.String() + ",")
}
str.WriteString(": Size ")
str.WriteString(strconv.Itoa(subnetSize))
}
if err := ipamutils.ConfigGlobalScopeDefaultNetworks(addressPool); err != nil {
return err
}
if addressPool != nil {
logrus.Infof("Swarm initialized global default address pool to: " + str.String())
}
for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){ for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
builtinIpam.Init, builtinIpam.Init,
remoteIpam.Init, remoteIpam.Init,

View file

@ -87,7 +87,7 @@ type initializer struct {
} }
// New returns a new NetworkAllocator handle // New returns a new NetworkAllocator handle
func New(pg plugingetter.PluginGetter) (networkallocator.NetworkAllocator, error) { func New(pg plugingetter.PluginGetter, defaultAddrPool []*net.IPNet, subnetSize int) (networkallocator.NetworkAllocator, error) {
na := &cnmNetworkAllocator{ na := &cnmNetworkAllocator{
networks: make(map[string]*network), networks: make(map[string]*network),
services: make(map[string]struct{}), services: make(map[string]struct{}),
@ -106,7 +106,7 @@ func New(pg plugingetter.PluginGetter) (networkallocator.NetworkAllocator, error
return nil, err return nil, err
} }
if err = initIPAMDrivers(reg); err != nil { if err = initIPAMDrivers(reg, defaultAddrPool, subnetSize); err != nil {
return nil, err return nil, err
} }

View file

@ -68,7 +68,7 @@ type networkContext struct {
} }
func (a *Allocator) doNetworkInit(ctx context.Context) (err error) { func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
na, err := cnmallocator.New(a.pluginGetter) na, err := cnmallocator.New(a.pluginGetter, a.defaultAddrPool, a.subnetSize)
if err != nil { if err != nil {
return err return err
} }
@ -181,10 +181,6 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
if IsIngressNetwork(n) { if IsIngressNetwork(n) {
nc.ingressNetwork = n nc.ingressNetwork = n
} }
err := a.allocateNodes(ctx, false)
if err != nil {
log.G(ctx).WithError(err).Error(err)
}
case api.EventDeleteNetwork: case api.EventDeleteNetwork:
n := v.Network.Copy() n := v.Network.Copy()
@ -337,19 +333,12 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
nc.somethingWasDeallocated = true nc.somethingWasDeallocated = true
} }
} else { } else {
allocatedNetworks, err := a.getAllocatedNetworks() // if this isn't a delete, we should try reallocating the node. if this
if err != nil { // is a creation, then the node will be allocated only for ingress.
log.G(ctx).WithError(err).Errorf("Error listing allocated networks in network %s", node.ID) if err := a.reallocateNode(ctx, node.ID); err != nil {
} log.G(ctx).WithError(err).Errorf(
"error reallocating network resources for node %v", node.ID,
isAllocated := a.allocateNode(ctx, node, false, allocatedNetworks) )
if isAllocated {
if err := a.store.Batch(func(batch *store.Batch) error {
return a.commitAllocatedNode(ctx, batch, node)
}); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
}
} }
} }
} }
@ -394,6 +383,69 @@ func (a *Allocator) getAllocatedNetworks() ([]*api.Network, error) {
return allocatedNetworks, nil return allocatedNetworks, nil
} }
// getNodeNetworks returns all networks that should be allocated for a node
func (a *Allocator) getNodeNetworks(nodeID string) ([]*api.Network, error) {
var (
// no need to initialize networks. we only append to it, and appending
// to a nil slice is valid. this has the added bonus of making this nil
// if we return an error
networks []*api.Network
err error
)
a.store.View(func(tx store.ReadTx) {
// get all tasks currently assigned to this node. it's no big deal if
// the tasks change in the meantime, there's no race to clean up
// unneeded network attachments on a node.
var tasks []*api.Task
tasks, err = store.FindTasks(tx, store.ByNodeID(nodeID))
if err != nil {
return
}
// we need to keep track of network IDs that we've already added to the
// list of networks we're going to return. we could do
// map[string]*api.Network and then convert to []*api.Network and
// return that, but it seems cleaner to have a separate set and list.
networkIDs := map[string]struct{}{}
for _, task := range tasks {
// we don't need to check if a task is before the Assigned state.
// the only way we have a task with a NodeID that isn't yet in
// Assigned is if it's a global service task. this check is not
// necessary:
// if task.Status.State < api.TaskStateAssigned {
// continue
// }
if task.Status.State > api.TaskStateRunning {
// we don't need to have network attachments for a task that's
// already in a terminal state
continue
}
// now go through the task's network attachments and find all of
// the networks
for _, attachment := range task.Networks {
// if the network is an overlay network, and the network ID is
// not yet in the set of network IDs, then add it to the set
// and add the network to the list of networks we'll be
// returning
if _, ok := networkIDs[attachment.Network.ID]; isOverlayNetwork(attachment.Network) && !ok {
networkIDs[attachment.Network.ID] = struct{}{}
// we don't need to worry about retrieving the network from
// the store, because the network in the attachment is an
// identical copy of the network in the store.
networks = append(networks, attachment.Network)
}
}
}
})
// finally, we need the ingress network if one exists.
if a.netCtx != nil && a.netCtx.ingressNetwork != nil {
networks = append(networks, a.netCtx.ingressNetwork)
}
return networks, err
}
func (a *Allocator) allocateNodes(ctx context.Context, existingAddressesOnly bool) error { func (a *Allocator) allocateNodes(ctx context.Context, existingAddressesOnly bool) error {
// Allocate nodes in the store so far before we process watched events. // Allocate nodes in the store so far before we process watched events.
var ( var (
@ -409,13 +461,12 @@ func (a *Allocator) allocateNodes(ctx context.Context, existingAddressesOnly boo
return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources") return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources")
} }
allocatedNetworks, err := a.getAllocatedNetworks()
if err != nil {
return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources")
}
for _, node := range nodes { for _, node := range nodes {
isAllocated := a.allocateNode(ctx, node, existingAddressesOnly, allocatedNetworks) networks, err := a.getNodeNetworks(node.ID)
if err != nil {
return errors.Wrap(err, "error getting all networks needed by node")
}
isAllocated := a.allocateNode(ctx, node, existingAddressesOnly, networks)
if isAllocated { if isAllocated {
allocatedNodes = append(allocatedNodes, node) allocatedNodes = append(allocatedNodes, node)
} }
@ -842,6 +893,14 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, ev events.Event) {
} }
} }
// if we're deallocating the task, we also might need to deallocate the
// node's network attachment, if this is the last task on the node that
// needs it. we can do that by doing the same dance to reallocate a
// node
if err := a.reallocateNode(ctx, t.NodeID); err != nil {
logger.WithError(err).Errorf("error reallocating node %v", t.NodeID)
}
// Cleanup any task references that might exist // Cleanup any task references that might exist
delete(nc.pendingTasks, t.ID) delete(nc.pendingTasks, t.ID)
delete(nc.unallocatedTasks, t.ID) delete(nc.unallocatedTasks, t.ID)
@ -849,6 +908,16 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, ev events.Event) {
return return
} }
// if the task has a node ID, we should allocate an attachment for the node
// this happens if the task is in any non-terminal state.
if t.NodeID != "" && t.Status.State <= api.TaskStateRunning {
if err := a.reallocateNode(ctx, t.NodeID); err != nil {
// TODO(dperny): not entire sure what the error handling flow here
// should be... for now, just log and keep going
logger.WithError(err).Errorf("error reallocating node %v", t.NodeID)
}
}
// If we are already in allocated state, there is // If we are already in allocated state, there is
// absolutely nothing else to do. // absolutely nothing else to do.
if t.Status.State >= api.TaskStatePending { if t.Status.State >= api.TaskStatePending {
@ -887,13 +956,25 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, ev events.Event) {
log.G(ctx).Debugf("task %v was marked pending allocation", t.ID) log.G(ctx).Debugf("task %v was marked pending allocation", t.ID)
} }
// allocateNode takes a context, a node, whether or not new allocations should
// be made, and the networks to allocate. it then makes sure an attachment is
// allocated for every network in the provided networks, allocating new
// attachments if existingAddressesOnly is false. it return true if something
// new was allocated or something was removed, or false otherwise.
//
// additionally, allocateNode will remove and free any attachments for networks
// not in the set of networks passed in.
func (a *Allocator) allocateNode(ctx context.Context, node *api.Node, existingAddressesOnly bool, networks []*api.Network) bool { func (a *Allocator) allocateNode(ctx context.Context, node *api.Node, existingAddressesOnly bool, networks []*api.Network) bool {
var allocated bool var allocated bool
nc := a.netCtx nc := a.netCtx
// go through all of the networks we've passed in
for _, network := range networks { for _, network := range networks {
// for each one, create space for an attachment. then, search through
// all of the attachments already on the node. if the attachment
// exists, then copy it to the node. if not, we'll allocate it below.
var lbAttachment *api.NetworkAttachment var lbAttachment *api.NetworkAttachment
for _, na := range node.Attachments { for _, na := range node.Attachments {
if na.Network != nil && na.Network.ID == network.ID { if na.Network != nil && na.Network.ID == network.ID {
@ -927,8 +1008,83 @@ func (a *Allocator) allocateNode(ctx context.Context, node *api.Node, existingAd
allocated = true allocated = true
} }
return allocated
// if we're only initializing existing addresses, we should stop here and
// not deallocate anything
if existingAddressesOnly {
return allocated
}
// now that we've allocated everything new, we have to remove things that
// do not belong. we have to do this last because we can easily roll back
// attachments we've allocated if something goes wrong by freeing them, but
// we can't roll back deallocating attachments by reacquiring them.
// we're using a trick to filter without allocating see the official go
// wiki on github:
// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
attachments := node.Attachments[:0]
for _, attach := range node.Attachments {
// for every attachment, go through every network. if the attachment
// belongs to one of the networks, then go to the next attachment. if
// no network matches, then the the attachment should be removed.
attachmentBelongs := false
for _, network := range networks {
if network.ID == attach.Network.ID {
attachmentBelongs = true
break
}
}
if attachmentBelongs {
attachments = append(attachments, attach)
} else {
// free the attachment and remove it from the node's attachments by
// re-slicing
if err := a.netCtx.nwkAllocator.DeallocateAttachment(node, attach); err != nil {
// if deallocation fails, there's nothing we can do besides log
// an error and keep going
log.G(ctx).WithError(err).Errorf(
"error deallocating attachment for network %v on node %v",
attach.Network.ID, node.ID,
)
}
// strictly speaking, nothing was allocated, but something was
// deallocated and that counts.
allocated = true
// also, set the somethingWasDeallocated flag so the allocator
// knows that it can now try again.
a.netCtx.somethingWasDeallocated = true
}
}
node.Attachments = attachments
return allocated
}
func (a *Allocator) reallocateNode(ctx context.Context, nodeID string) error {
var (
node *api.Node
)
a.store.View(func(tx store.ReadTx) {
node = store.GetNode(tx, nodeID)
})
if node == nil {
return errors.Errorf("node %v cannot be found", nodeID)
}
networks, err := a.getNodeNetworks(node.ID)
if err != nil {
return errors.Wrapf(err, "error getting networks for node %v", nodeID)
}
if a.allocateNode(ctx, node, false, networks) {
// if something was allocated, commit the node
if err := a.store.Batch(func(batch *store.Batch) error {
return a.commitAllocatedNode(ctx, batch, node)
}); err != nil {
return errors.Wrapf(err, "error committing allocation for node %v", nodeID)
}
}
return nil
} }
func (a *Allocator) commitAllocatedNode(ctx context.Context, batch *store.Batch, node *api.Node) error { func (a *Allocator) commitAllocatedNode(ctx context.Context, batch *store.Batch, node *api.Node) error {

View file

@ -265,6 +265,8 @@ func redactClusters(clusters []*api.Cluster) []*api.Cluster {
Spec: *redactedSpec, Spec: *redactedSpec,
RootCA: *redactedRootCA, RootCA: *redactedRootCA,
BlacklistedCertificates: cluster.BlacklistedCertificates, BlacklistedCertificates: cluster.BlacklistedCertificates,
DefaultAddressPool: cluster.DefaultAddressPool,
SubnetSize: cluster.SubnetSize,
} }
redactedClusters = append(redactedClusters, newCluster) redactedClusters = append(redactedClusters, newCluster)
} }

View file

@ -141,6 +141,12 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) (
} }
return filterMatchLabels(e.Description.Engine.Labels, request.Filters.Labels) return filterMatchLabels(e.Description.Engine.Labels, request.Filters.Labels)
}, },
func(e *api.Node) bool {
if len(request.Filters.NodeLabels) == 0 {
return true
}
return filterMatchLabels(e.Spec.Annotations.Labels, request.Filters.NodeLabels)
},
func(e *api.Node) bool { func(e *api.Node) bool {
if len(request.Filters.Roles) == 0 { if len(request.Filters.Roles) == 0 {
return true return true

View file

@ -126,6 +126,13 @@ type Config struct {
// first node in the cluster, this setting is used to set the cluster-wide mandatory // first node in the cluster, this setting is used to set the cluster-wide mandatory
// FIPS setting. // FIPS setting.
FIPS bool FIPS bool
// DefaultAddrPool specifies default subnet pool for global scope networks
DefaultAddrPool []*net.IPNet
// SubnetSize specifies the subnet size of the networks created from
// the default subnet pool
SubnetSize int
} }
// Manager is the cluster manager for Swarm. // Manager is the cluster manager for Swarm.
@ -924,20 +931,33 @@ func (m *Manager) becomeLeader(ctx context.Context) {
Key: m.config.UnlockKey, Key: m.config.UnlockKey,
}} }}
} }
s.Update(func(tx store.Tx) error { s.Update(func(tx store.Tx) error {
// Add a default cluster object to the // Add a default cluster object to the
// store. Don't check the error because // store. Don't check the error because
// we expect this to fail unless this // we expect this to fail unless this
// is a brand new cluster. // is a brand new cluster.
err := store.CreateCluster(tx, defaultClusterObject( clusterObj := defaultClusterObject(
clusterID, clusterID,
initialCAConfig, initialCAConfig,
raftCfg, raftCfg,
api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers},
unlockKeys, unlockKeys,
rootCA, rootCA,
m.config.FIPS)) m.config.FIPS,
nil,
0)
var defaultAddrPool []string
for _, p := range m.config.DefaultAddrPool {
defaultAddrPool = append(defaultAddrPool, p.String())
}
// If defaultAddrPool is valid we update cluster object with new value
if defaultAddrPool != nil {
clusterObj.DefaultAddressPool = defaultAddrPool
clusterObj.SubnetSize = uint32(m.config.SubnetSize)
}
err := store.CreateCluster(tx, clusterObj)
if err != nil && err != store.ErrExist { if err != nil && err != store.ErrExist {
log.G(ctx).WithError(err).Errorf("error creating cluster object") log.G(ctx).WithError(err).Errorf("error creating cluster object")
@ -981,7 +1001,26 @@ func (m *Manager) becomeLeader(ctx context.Context) {
// shutdown underlying manager processes when leadership is // shutdown underlying manager processes when leadership is
// lost. // lost.
m.allocator, err = allocator.New(s, m.config.PluginGetter) // If DefaultAddrPool is null, Read from store and check if
// DefaultAddrPool info is stored in cluster object
if m.config.DefaultAddrPool == nil {
var cluster *api.Cluster
s.View(func(tx store.ReadTx) {
cluster = store.GetCluster(tx, clusterID)
})
if cluster.DefaultAddressPool != nil {
for _, address := range cluster.DefaultAddressPool {
_, b, err := net.ParseCIDR(address)
if err != nil {
log.G(ctx).WithError(err).Error("Default Address Pool reading failed for cluster object %s", address)
}
m.config.DefaultAddrPool = append(m.config.DefaultAddrPool, b)
}
}
m.config.SubnetSize = int(cluster.SubnetSize)
}
m.allocator, err = allocator.New(s, m.config.PluginGetter, m.config.DefaultAddrPool, m.config.SubnetSize)
if err != nil { if err != nil {
log.G(ctx).WithError(err).Error("failed to create allocator") log.G(ctx).WithError(err).Error("failed to create allocator")
// TODO(stevvooe): It doesn't seem correct here to fail // TODO(stevvooe): It doesn't seem correct here to fail
@ -1103,7 +1142,9 @@ func defaultClusterObject(
encryptionConfig api.EncryptionConfig, encryptionConfig api.EncryptionConfig,
initialUnlockKeys []*api.EncryptionKey, initialUnlockKeys []*api.EncryptionKey,
rootCA *ca.RootCA, rootCA *ca.RootCA,
fips bool) *api.Cluster { fips bool,
defaultAddressPool []string,
subnetSize int) *api.Cluster {
var caKey []byte var caKey []byte
if rcaSigner, err := rootCA.Signer(); err == nil { if rcaSigner, err := rootCA.Signer(); err == nil {
caKey = rcaSigner.Key caKey = rcaSigner.Key
@ -1134,8 +1175,10 @@ func defaultClusterObject(
Manager: ca.GenerateJoinToken(rootCA, fips), Manager: ca.GenerateJoinToken(rootCA, fips),
}, },
}, },
UnlockKeys: initialUnlockKeys, UnlockKeys: initialUnlockKeys,
FIPS: fips, FIPS: fips,
DefaultAddressPool: defaultAddressPool,
SubnetSize: uint32(subnetSize),
} }
} }

View file

@ -105,6 +105,13 @@ type Config struct {
// for connections to the remote API (including the raft service). // for connections to the remote API (including the raft service).
AdvertiseRemoteAPI string AdvertiseRemoteAPI string
// DefaultAddrPool specifies default subnet pool for global scope networks
DefaultAddrPool []*net.IPNet
// SubnetSize specifies the subnet size of the networks created from
// the default subnet pool
SubnetSize int
// Executor specifies the executor to use for the agent. // Executor specifies the executor to use for the agent.
Executor exec.Executor Executor exec.Executor
@ -995,6 +1002,8 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
PluginGetter: n.config.PluginGetter, PluginGetter: n.config.PluginGetter,
RootCAPaths: rootPaths, RootCAPaths: rootPaths,
FIPS: n.config.FIPS, FIPS: n.config.FIPS,
DefaultAddrPool: n.config.DefaultAddrPool,
SubnetSize: n.config.SubnetSize,
}) })
if err != nil { if err != nil {
return false, err return false, err

View file

@ -33,7 +33,7 @@ github.com/docker/go-connections 7beb39f0b969b075d1325fecb092faf27fd357b6
github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1
github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef
github.com/docker/libnetwork d00ceed44cc447c77f25cdf5d59e83163bdcb4c9 github.com/docker/libnetwork a79d3687931697244b8e03485bf7b2042f8ec6b6
github.com/opencontainers/runc ad0f5255060d36872be04de22f8731f38ef2d7b1 github.com/opencontainers/runc ad0f5255060d36872be04de22f8731f38ef2d7b1
github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/go-digest v1.0.0-rc1
github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/image-spec v1.0.1