mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
Merge pull request #25159 from diogomonica/adding-force-to-node-remove
Adding force to node rm
This commit is contained in:
Sebastiaan van Stijn
GitHub
commit
10ae908bfa
27 changed files with 403 additions and 205 deletions
|
|
@ -7,26 +7,36 @@ import (
|
|||
|
||||
"github.com/docker/docker/api/client"
|
||||
"github.com/docker/docker/cli"
|
||||
"github.com/docker/engine-api/types"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type removeOptions struct {
|
||||
force bool
|
||||
}
|
||||
|
||||
func newRemoveCommand(dockerCli *client.DockerCli) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "rm NODE [NODE...]",
|
||||
opts := removeOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "rm [OPTIONS] NODE [NODE...]",
|
||||
Aliases: []string{"remove"},
|
||||
Short: "Remove one or more nodes from the swarm",
|
||||
Args: cli.RequiresMinArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runRemove(dockerCli, args)
|
||||
return runRemove(dockerCli, args, opts)
|
||||
},
|
||||
}
|
||||
flags := cmd.Flags()
|
||||
flags.BoolVar(&opts.force, "force", false, "Force remove an active node")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runRemove(dockerCli *client.DockerCli, args []string) error {
|
||||
func runRemove(dockerCli *client.DockerCli, args []string, opts removeOptions) error {
|
||||
client := dockerCli.Client()
|
||||
ctx := context.Background()
|
||||
for _, nodeID := range args {
|
||||
err := client.NodeRemove(ctx, nodeID)
|
||||
err := client.NodeRemove(ctx, nodeID, types.NodeRemoveOptions{Force: opts.force})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ type Backend interface {
|
|||
GetNodes(basictypes.NodeListOptions) ([]types.Node, error)
|
||||
GetNode(string) (types.Node, error)
|
||||
UpdateNode(string, uint64, types.NodeSpec) error
|
||||
RemoveNode(string) error
|
||||
RemoveNode(string, bool) error
|
||||
GetTasks(basictypes.TaskListOptions) ([]types.Task, error)
|
||||
GetTask(string) (types.Task, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -219,7 +219,13 @@ func (sr *swarmRouter) updateNode(ctx context.Context, w http.ResponseWriter, r
|
|||
}
|
||||
|
||||
func (sr *swarmRouter) removeNode(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
|
||||
if err := sr.backend.RemoveNode(vars["id"]); err != nil {
|
||||
if err := httputils.ParseForm(r); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
force := httputils.BoolValue(r, "force")
|
||||
|
||||
if err := sr.backend.RemoveNode(vars["id"], force); err != nil {
|
||||
logrus.Errorf("Error removing node %s: %v", vars["id"], err)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1023,7 +1023,7 @@ func (c *Cluster) UpdateNode(nodeID string, version uint64, spec types.NodeSpec)
|
|||
}
|
||||
|
||||
// RemoveNode removes a node from a cluster
|
||||
func (c *Cluster) RemoveNode(input string) error {
|
||||
func (c *Cluster) RemoveNode(input string, force bool) error {
|
||||
c.RLock()
|
||||
defer c.RUnlock()
|
||||
|
||||
|
|
@ -1039,7 +1039,7 @@ func (c *Cluster) RemoveNode(input string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID}); err != nil {
|
||||
if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID, Force: force}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ parent = "smn_cli"
|
|||
# node rm
|
||||
|
||||
```markdown
|
||||
Usage: docker node rm NODE [NODE...]
|
||||
Usage: docker node rm [OPTIONS] NODE [NODE...]
|
||||
|
||||
Remove one or more nodes from the swarm
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ Aliases:
|
|||
rm, remove
|
||||
|
||||
Options:
|
||||
--force Force remove an active node
|
||||
--help Print usage
|
||||
```
|
||||
|
||||
|
|
@ -30,6 +31,24 @@ Example output:
|
|||
$ docker node rm swarm-node-02
|
||||
Node swarm-node-02 removed from swarm
|
||||
|
||||
Removes nodes from the swarm that are in the down state. Attempting to remove
|
||||
an active node will result in an error:
|
||||
|
||||
```bash
|
||||
$ docker node rm swarm-node-03
|
||||
Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not down and can't be removed
|
||||
```
|
||||
|
||||
If a worker node becomes compromised, exhibits unexpected or unwanted behavior, or if you lose access to it so
|
||||
that a clean shutdown is impossible, you can use the force option.
|
||||
|
||||
```bash
|
||||
$ docker node rm --force swarm-node-03
|
||||
Node swarm-node-03 removed from swarm
|
||||
```
|
||||
|
||||
Note that manager nodes have to be demoted to worker nodes before they can be removed
|
||||
from the cluster.
|
||||
|
||||
## Related information
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,8 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith
|
|||
clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git
|
||||
clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
|
||||
clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d
|
||||
clone git github.com/docker/engine-api 3d1601b9d2436a70b0dfc045a23f6503d19195df
|
||||
|
||||
clone git github.com/docker/engine-api 228c7390a733320d48697cb41ae8cde4942cd3e5
|
||||
clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
|
||||
clone git github.com/imdario/mergo 0.2.1
|
||||
|
||||
|
|
@ -139,7 +140,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
|
|||
clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267
|
||||
|
||||
# cluster
|
||||
clone git github.com/docker/swarmkit 9d4c2f73124e70f8fa85f9076635b827d17b109f
|
||||
clone git github.com/docker/swarmkit e1c0d64515d839b76e2ef33d396c74933753ffaf
|
||||
clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
|
||||
clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
|
||||
clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b
|
||||
|
|
|
|||
|
|
@ -228,6 +228,17 @@ func (d *SwarmDaemon) getNode(c *check.C, id string) *swarm.Node {
|
|||
return &node
|
||||
}
|
||||
|
||||
func (d *SwarmDaemon) removeNode(c *check.C, id string, force bool) {
|
||||
url := "/nodes/" + id
|
||||
if force {
|
||||
url += "?force=1"
|
||||
}
|
||||
|
||||
status, out, err := d.SockRequest("DELETE", url, nil)
|
||||
c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out)))
|
||||
c.Assert(err, checker.IsNil)
|
||||
}
|
||||
|
||||
func (d *SwarmDaemon) updateNode(c *check.C, id string, f ...nodeConstructor) {
|
||||
for i := 0; ; i++ {
|
||||
node := d.getNode(c, id)
|
||||
|
|
|
|||
|
|
@ -691,6 +691,37 @@ func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) {
|
|||
c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause)
|
||||
}
|
||||
|
||||
func (s *DockerSwarmSuite) TestApiSwarmNodeRemove(c *check.C) {
|
||||
testRequires(c, Network)
|
||||
d1 := s.AddDaemon(c, true, true)
|
||||
d2 := s.AddDaemon(c, true, false)
|
||||
_ = s.AddDaemon(c, true, false)
|
||||
|
||||
nodes := d1.listNodes(c)
|
||||
c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes))
|
||||
|
||||
// Getting the info so we can take the NodeID
|
||||
d2Info, err := d2.info()
|
||||
c.Assert(err, checker.IsNil)
|
||||
|
||||
// forceful removal of d2 should work
|
||||
d1.removeNode(c, d2Info.NodeID, true)
|
||||
|
||||
nodes = d1.listNodes(c)
|
||||
c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
|
||||
|
||||
// Restart the node that was removed
|
||||
err = d2.Restart()
|
||||
c.Assert(err, checker.IsNil)
|
||||
|
||||
// Give some time for the node to rejoin
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// Make sure the node didn't rejoin
|
||||
nodes = d1.listNodes(c)
|
||||
c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
|
||||
}
|
||||
|
||||
func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) {
|
||||
d1 := s.AddDaemon(c, true, true)
|
||||
d2 := s.AddDaemon(c, true, false)
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
package client
|
||||
|
||||
// DefaultDockerHost defines os specific default if DOCKER_HOST is unset
|
||||
const DefaultDockerHost = "tcp://127.0.0.1:2375"
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
// +build linux freebsd solaris openbsd
|
||||
// +build linux freebsd solaris openbsd darwin
|
||||
|
||||
package client
|
||||
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ type NetworkAPIClient interface {
|
|||
type NodeAPIClient interface {
|
||||
NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error)
|
||||
NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error)
|
||||
NodeRemove(ctx context.Context, nodeID string) error
|
||||
NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error
|
||||
NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,21 @@
|
|||
package client
|
||||
|
||||
import "golang.org/x/net/context"
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"github.com/docker/engine-api/types"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// NodeRemove removes a Node.
|
||||
func (cli *Client) NodeRemove(ctx context.Context, nodeID string) error {
|
||||
resp, err := cli.delete(ctx, "/nodes/"+nodeID, nil, nil)
|
||||
func (cli *Client) NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error {
|
||||
query := url.Values{}
|
||||
if options.Force {
|
||||
query.Set("force", "1")
|
||||
}
|
||||
|
||||
resp, err := cli.delete(ctx, "/nodes/"+nodeID, query, nil)
|
||||
ensureReaderClosed(resp)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,11 +241,16 @@ func (v VersionResponse) ServerOK() bool {
|
|||
return v.Server != nil
|
||||
}
|
||||
|
||||
// NodeListOptions holds parameters to list nodes with.
|
||||
// NodeListOptions holds parameters to list nodes with.
|
||||
type NodeListOptions struct {
|
||||
Filter filters.Args
|
||||
}
|
||||
|
||||
// NodeRemoveOptions holds parameters to remove nodes with.
|
||||
type NodeRemoveOptions struct {
|
||||
Force bool
|
||||
}
|
||||
|
||||
// ServiceCreateOptions contains the options to use when creating a service.
|
||||
type ServiceCreateOptions struct {
|
||||
// EncodedRegistryAuth is the encoded registry authorization credentials to
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
initialSessionFailureBackoff = time.Second
|
||||
initialSessionFailureBackoff = 100 * time.Millisecond
|
||||
maxSessionFailureBackoff = 8 * time.Second
|
||||
)
|
||||
|
||||
// Agent implements the primary node functionality for a member of a swarm
|
||||
// cluster. The primary functionality id to run and report on the status of
|
||||
// cluster. The primary functionality is to run and report on the status of
|
||||
// tasks assigned to the node.
|
||||
type Agent struct {
|
||||
config *Config
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ func (n *Node) run(ctx context.Context) (err error) {
|
|||
if n.config.JoinAddr != "" || n.config.ForceNewCluster {
|
||||
n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
|
||||
if n.config.JoinAddr != "" {
|
||||
n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, 1)
|
||||
n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, picker.DefaultObservationWeight)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -647,7 +647,7 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
|
|||
go func(ready chan struct{}) {
|
||||
select {
|
||||
case <-ready:
|
||||
n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5)
|
||||
n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, picker.DefaultObservationWeight)
|
||||
case <-connCtx.Done():
|
||||
}
|
||||
}(ready)
|
||||
|
|
|
|||
|
|
@ -200,7 +200,7 @@ func (tm *taskManager) run(ctx context.Context) {
|
|||
cancel() // cancel outstanding if necessary.
|
||||
} else {
|
||||
// If this channel op fails, it means there is already a
|
||||
// message un the run queue.
|
||||
// message on the run queue.
|
||||
select {
|
||||
case run <- struct{}{}:
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ func (*UpdateNodeResponse) Descriptor() ([]byte, []int) { return fileDescriptorC
|
|||
// RemoveNodeRequest requests to delete the specified node from store.
|
||||
type RemoveNodeRequest struct {
|
||||
NodeID string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"`
|
||||
}
|
||||
|
||||
func (m *RemoveNodeRequest) Reset() { *m = RemoveNodeRequest{} }
|
||||
|
|
@ -786,6 +787,7 @@ func (m *RemoveNodeRequest) Copy() *RemoveNodeRequest {
|
|||
|
||||
o := &RemoveNodeRequest{
|
||||
NodeID: m.NodeID,
|
||||
Force: m.Force,
|
||||
}
|
||||
|
||||
return o
|
||||
|
|
@ -1473,9 +1475,10 @@ func (this *RemoveNodeRequest) GoString() string {
|
|||
if this == nil {
|
||||
return "nil"
|
||||
}
|
||||
s := make([]string, 0, 5)
|
||||
s := make([]string, 0, 6)
|
||||
s = append(s, "&api.RemoveNodeRequest{")
|
||||
s = append(s, "NodeID: "+fmt.Sprintf("%#v", this.NodeID)+",\n")
|
||||
s = append(s, "Force: "+fmt.Sprintf("%#v", this.Force)+",\n")
|
||||
s = append(s, "}")
|
||||
return strings.Join(s, "")
|
||||
}
|
||||
|
|
@ -2938,6 +2941,16 @@ func (m *RemoveNodeRequest) MarshalTo(data []byte) (int, error) {
|
|||
i = encodeVarintControl(data, i, uint64(len(m.NodeID)))
|
||||
i += copy(data[i:], m.NodeID)
|
||||
}
|
||||
if m.Force {
|
||||
data[i] = 0x10
|
||||
i++
|
||||
if m.Force {
|
||||
data[i] = 1
|
||||
} else {
|
||||
data[i] = 0
|
||||
}
|
||||
i++
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
|
|
@ -4692,6 +4705,9 @@ func (m *RemoveNodeRequest) Size() (n int) {
|
|||
if l > 0 {
|
||||
n += 1 + l + sovControl(uint64(l))
|
||||
}
|
||||
if m.Force {
|
||||
n += 2
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
|
|
@ -5286,6 +5302,7 @@ func (this *RemoveNodeRequest) String() string {
|
|||
}
|
||||
s := strings.Join([]string{`&RemoveNodeRequest{`,
|
||||
`NodeID:` + fmt.Sprintf("%v", this.NodeID) + `,`,
|
||||
`Force:` + fmt.Sprintf("%v", this.Force) + `,`,
|
||||
`}`,
|
||||
}, "")
|
||||
return s
|
||||
|
|
@ -6617,6 +6634,26 @@ func (m *RemoveNodeRequest) Unmarshal(data []byte) error {
|
|||
}
|
||||
m.NodeID = string(data[iNdEx:postIndex])
|
||||
iNdEx = postIndex
|
||||
case 2:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Force", wireType)
|
||||
}
|
||||
var v int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowControl
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := data[iNdEx]
|
||||
iNdEx++
|
||||
v |= (int(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
m.Force = bool(v != 0)
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipControl(data[iNdEx:])
|
||||
|
|
@ -10521,99 +10558,100 @@ var (
|
|||
)
|
||||
|
||||
var fileDescriptorControl = []byte{
|
||||
// 1498 bytes of a gzipped FileDescriptorProto
|
||||
// 1512 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x59, 0xcf, 0x6f, 0x1b, 0xc5,
|
||||
0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x09, 0xda,
|
||||
0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x0d, 0xda,
|
||||
0xd2, 0x34, 0x91, 0x82, 0x03, 0x8e, 0x2a, 0x02, 0x48, 0x20, 0x9c, 0xd0, 0xca, 0xd0, 0x86, 0x6a,
|
||||
0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
|
||||
0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0x27, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
|
||||
0x20, 0xf1, 0x27, 0x70, 0xe5, 0xca, 0x81, 0x7f, 0x81, 0x6b, 0xc4, 0x89, 0x0b, 0x12, 0xa7, 0x88,
|
||||
0xf6, 0xc4, 0x09, 0xf1, 0x17, 0x20, 0xe6, 0xc7, 0x9b, 0xdd, 0xf5, 0x7a, 0x76, 0x6d, 0x27, 0x41,
|
||||
0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xfd, 0x98, 0x79, 0x9f, 0xf7, 0xf6, 0xcd, 0x04, 0xf2, 0x4d,
|
||||
0xbb, 0xe7, 0x39, 0x76, 0xa7, 0xd2, 0x77, 0x6c, 0xcf, 0x26, 0xa4, 0x65, 0x37, 0xdb, 0xd4, 0xa9,
|
||||
0xb8, 0x4f, 0x1b, 0x4e, 0xb7, 0x6d, 0x79, 0x95, 0x83, 0x37, 0xca, 0x39, 0xb7, 0x4f, 0x9b, 0xae,
|
||||
0x04, 0x94, 0xf3, 0xf6, 0xee, 0x17, 0xb4, 0xe9, 0xa9, 0xd7, 0x9c, 0x77, 0xd8, 0xa7, 0xea, 0xa5,
|
||||
0xb4, 0x67, 0xef, 0xd9, 0xe2, 0x71, 0x95, 0x3f, 0xe1, 0xe8, 0xd5, 0x7e, 0x67, 0x7f, 0xcf, 0xea,
|
||||
0xad, 0xca, 0x3f, 0x72, 0xd0, 0xb8, 0x03, 0x85, 0x7b, 0xd4, 0xdb, 0xb2, 0x5b, 0xd4, 0xa4, 0x5f,
|
||||
0xee, 0x53, 0xd7, 0x23, 0x37, 0x21, 0xd3, 0x63, 0xaf, 0x3b, 0x56, 0x6b, 0x2e, 0xf5, 0x4a, 0x6a,
|
||||
0x29, 0x5b, 0x83, 0x17, 0xc7, 0x0b, 0x33, 0x1c, 0x51, 0xdf, 0x34, 0x67, 0xf8, 0x54, 0xbd, 0x65,
|
||||
0xbc, 0x07, 0x97, 0x7d, 0x31, 0xb7, 0x6f, 0xf7, 0x5c, 0x4a, 0x56, 0x60, 0x9a, 0x4f, 0x0a, 0xa1,
|
||||
0x5c, 0x75, 0xae, 0x32, 0xbc, 0x80, 0x8a, 0xc0, 0x0b, 0x94, 0x71, 0x3c, 0x05, 0x57, 0xee, 0x5b,
|
||||
0xae, 0x50, 0xe1, 0x2a, 0xd3, 0x77, 0x21, 0xf3, 0xc4, 0xea, 0x78, 0xd4, 0x71, 0x51, 0xcb, 0x8a,
|
||||
0x4e, 0x4b, 0x54, 0xac, 0x72, 0x57, 0xca, 0x98, 0x4a, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0x70, 0x90,
|
||||
0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x5c, 0xe3, 0xd4, 0x52, 0xd6, 0x94, 0x2f, 0x64, 0x15, 0x72,
|
||||
0x56, 0x6b, 0xa7, 0xef, 0xd0, 0x27, 0xd6, 0x33, 0x36, 0x97, 0xe6, 0x73, 0xb5, 0x02, 0x5b, 0x28,
|
||||
0xd4, 0x37, 0x1f, 0xe2, 0xa8, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x87, 0x30, 0xd3, 0x69, 0xec, 0xd2,
|
||||
0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xfa, 0x24, 0x9e, 0x55, 0xee, 0x0b, 0xd1, 0x0f, 0x58,
|
||||
0x80, 0x0f, 0x4d, 0xd4, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x2e, 0x9b, 0xfe, 0xdc, 0xea, 0xbb,
|
||||
0x73, 0xd3, 0x4c, 0x6d, 0xa1, 0x7a, 0x3b, 0x6e, 0xdb, 0xb6, 0x59, 0xe8, 0x2b, 0x0f, 0x7c, 0xbc,
|
||||
0x19, 0x96, 0x25, 0x55, 0xb8, 0xc8, 0x98, 0xc3, 0xd6, 0x71, 0x51, 0x28, 0xb9, 0x11, 0xbb, 0xf7,
|
||||
0x0c, 0x64, 0x4a, 0x28, 0x0b, 0x73, 0x9e, 0x6f, 0x45, 0xb0, 0x07, 0x33, 0x62, 0x7f, 0x2e, 0xf1,
|
||||
0x41, 0xb5, 0xea, 0xf2, 0x5b, 0x90, 0x0b, 0xb9, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a, 0x5a,
|
||||
0x98, 0xfc, 0x91, 0xef, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0e, 0xf2, 0x31, 0xf9, 0xf2, 0x76,
|
||||
0x7a, 0x3d, 0x65, 0x6c, 0x40, 0x31, 0xb4, 0x1d, 0xc8, 0x91, 0x0a, 0x0b, 0x06, 0x1f, 0x10, 0xc1,
|
||||
0x48, 0x22, 0x89, 0x84, 0x19, 0x3f, 0xa6, 0xa0, 0xf8, 0xb8, 0xdf, 0x6a, 0x78, 0x74, 0x52, 0x86,
|
||||
0x92, 0x77, 0xe1, 0x92, 0x00, 0x1d, 0xb0, 0x4d, 0xb2, 0xec, 0x9e, 0x70, 0x30, 0x57, 0xbd, 0xae,
|
||||
0xb3, 0xf8, 0x89, 0x84, 0x98, 0x39, 0x2e, 0x80, 0x2f, 0xe4, 0x75, 0x98, 0xe6, 0xe9, 0xc6, 0xc2,
|
||||
0xcd, 0xe5, 0x6e, 0x24, 0xc5, 0xc5, 0x14, 0x48, 0xa3, 0x06, 0x24, 0xec, 0xeb, 0x89, 0xd2, 0x62,
|
||||
0x1d, 0x8a, 0x26, 0xed, 0xda, 0x07, 0x13, 0xaf, 0xd7, 0x28, 0x01, 0x09, 0x4b, 0x4a, 0xeb, 0x98,
|
||||
0xde, 0x8f, 0x1a, 0x6e, 0x3b, 0xa4, 0xcc, 0x63, 0xaf, 0x11, 0x65, 0x1c, 0xc1, 0x95, 0xf1, 0x29,
|
||||
0x3f, 0xbd, 0xa5, 0x58, 0xb0, 0x0e, 0x3e, 0x99, 0xb4, 0x0e, 0x81, 0x17, 0xa8, 0x60, 0x1d, 0x13,
|
||||
0x9b, 0xf6, 0xd7, 0x11, 0xb6, 0x6e, 0xfc, 0x83, 0xe5, 0x82, 0x0f, 0x9e, 0xa0, 0x5c, 0x84, 0xc5,
|
||||
0x86, 0xcb, 0xc5, 0x0f, 0xe7, 0x58, 0x2e, 0x74, 0x9e, 0x69, 0xcb, 0x05, 0x73, 0xc1, 0xa5, 0xce,
|
||||
0x81, 0xd5, 0xe4, 0x3c, 0x90, 0xe5, 0x02, 0x5d, 0xd8, 0x96, 0xc3, 0xf5, 0x4d, 0xe6, 0x02, 0x42,
|
||||
0xea, 0x2d, 0x97, 0x2c, 0xc2, 0x2c, 0xb2, 0x46, 0xd6, 0x85, 0x6c, 0x2d, 0xc7, 0xd0, 0x19, 0x49,
|
||||
0x1b, 0xb6, 0x7a, 0xc9, 0x1b, 0x97, 0x6c, 0x42, 0x81, 0xa5, 0x9a, 0xe5, 0xd0, 0xd6, 0x8e, 0xeb,
|
||||
0x31, 0xf6, 0xca, 0x4a, 0x50, 0xa8, 0xfe, 0x3f, 0x2e, 0xc4, 0xdb, 0x1c, 0x65, 0xe6, 0x51, 0x48,
|
||||
0xbc, 0x69, 0xca, 0x49, 0xe6, 0x3f, 0x29, 0x27, 0xb8, 0x5d, 0x41, 0x39, 0xe1, 0xac, 0x49, 0x2c,
|
||||
0x27, 0x82, 0x46, 0x12, 0x66, 0x7c, 0x04, 0xa5, 0x0d, 0x87, 0x32, 0x7f, 0x71, 0xcb, 0x14, 0x91,
|
||||
0xd6, 0x30, 0xd7, 0x25, 0x8b, 0x16, 0x74, 0x6a, 0x50, 0x22, 0x94, 0xee, 0x5b, 0x70, 0x2d, 0xa2,
|
||||
0x0c, 0xbd, 0xba, 0x03, 0x19, 0x0c, 0x03, 0x2a, 0xbc, 0x9e, 0xa0, 0xd0, 0x54, 0x58, 0xe3, 0x7d,
|
||||
0x28, 0xb2, 0x9c, 0x8b, 0x78, 0xb6, 0x02, 0x10, 0x44, 0x1d, 0xb3, 0x26, 0xcf, 0xc2, 0x98, 0xf5,
|
||||
0x83, 0x6e, 0x66, 0xfd, 0x98, 0xb3, 0xf5, 0x91, 0xb0, 0x8a, 0xd3, 0xf9, 0xf3, 0x73, 0x0a, 0x4a,
|
||||
0xb2, 0x9e, 0x9d, 0xc6, 0x27, 0x46, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x29, 0x2e, 0xa0, 0x8c, 0xaa,
|
||||
0xc6, 0x6b, 0x03, 0xd5, 0x78, 0xfc, 0x08, 0x45, 0x16, 0x70, 0xba, 0x1d, 0xd9, 0x84, 0x92, 0x2c,
|
||||
0x4d, 0xa7, 0x0a, 0xd2, 0xff, 0xe0, 0x5a, 0x44, 0x0b, 0xd6, 0xb8, 0x3f, 0xd3, 0x70, 0x95, 0x73,
|
||||
0x1c, 0xc7, 0xfd, 0x32, 0x57, 0x8f, 0x96, 0xb9, 0xd5, 0xb8, 0x62, 0x12, 0x91, 0x1c, 0xae, 0x74,
|
||||
0xdf, 0xa6, 0xcf, 0xbc, 0xd2, 0x6d, 0x47, 0x2a, 0xdd, 0x3b, 0x13, 0x3a, 0xa7, 0x2d, 0x76, 0x43,
|
||||
0xd5, 0x64, 0xfa, 0x6c, 0xab, 0xc9, 0xc7, 0x50, 0x1a, 0x74, 0x09, 0x89, 0xf1, 0x26, 0xcc, 0x62,
|
||||
0xa0, 0x54, 0x4d, 0x49, 0x64, 0x86, 0x0f, 0x0e, 0x2a, 0xcb, 0x16, 0xf5, 0x9e, 0xda, 0x4e, 0x7b,
|
||||
0x82, 0xca, 0x82, 0x12, 0xba, 0xca, 0xe2, 0x2b, 0x0b, 0x78, 0xdb, 0x93, 0x43, 0x49, 0xbc, 0x55,
|
||||
0x52, 0x0a, 0x6b, 0x3c, 0x16, 0x95, 0x25, 0xe2, 0x19, 0x61, 0x7d, 0x09, 0xdb, 0x4d, 0xdc, 0x2f,
|
||||
0xf1, 0xcc, 0x89, 0x8c, 0x32, 0x9c, 0xc8, 0xe9, 0x80, 0xc8, 0x28, 0xcb, 0x89, 0x8c, 0x00, 0xbf,
|
||||
0xda, 0x9c, 0x91, 0x8f, 0x9f, 0xa9, 0xdc, 0x3a, 0x73, 0x37, 0xfd, 0x7c, 0x8b, 0x78, 0xea, 0xe7,
|
||||
0x1b, 0x8e, 0x9f, 0x20, 0xdf, 0x22, 0x92, 0x2f, 0x57, 0xbe, 0xc5, 0x38, 0x77, 0x9e, 0xf9, 0x16,
|
||||
0xb8, 0x14, 0xe4, 0x1b, 0x06, 0x2a, 0x31, 0xdf, 0x54, 0xe4, 0x7c, 0x30, 0x7e, 0x2c, 0x37, 0x3a,
|
||||
0xfb, 0x2e, 0x5b, 0x53, 0xa8, 0x0e, 0x37, 0xe5, 0x48, 0xa4, 0x0e, 0x23, 0x8e, 0xf3, 0x02, 0x01,
|
||||
0x3e, 0x7d, 0x7d, 0x15, 0x01, 0x7d, 0x11, 0x92, 0x44, 0x5f, 0x25, 0xa5, 0xb0, 0x3e, 0x97, 0x70,
|
||||
0xe2, 0x04, 0x5c, 0x8a, 0x48, 0xbe, 0x5c, 0x5c, 0x8a, 0x71, 0xee, 0x3c, 0xb9, 0x14, 0xb8, 0x14,
|
||||
0x70, 0x09, 0xa3, 0x91, 0xc8, 0x25, 0x15, 0x3a, 0x1f, 0x6c, 0xec, 0x43, 0xf1, 0x43, 0xdb, 0xea,
|
||||
0x3d, 0xb2, 0xdb, 0xb4, 0x67, 0xda, 0xac, 0x9d, 0xe5, 0x0d, 0x47, 0x05, 0xae, 0x3a, 0xfc, 0x99,
|
||||
0xee, 0x70, 0xc2, 0x31, 0x46, 0x79, 0x7c, 0x5a, 0x78, 0x38, 0x6b, 0x16, 0xe5, 0xd4, 0xa7, 0x62,
|
||||
0x46, 0xc8, 0xb1, 0xe3, 0x62, 0x09, 0xf1, 0xdd, 0x46, 0xaf, 0xb1, 0xe7, 0x0b, 0xa4, 0x85, 0x00,
|
||||
0x91, 0x73, 0x0f, 0xe4, 0x94, 0x90, 0x30, 0xbe, 0x4b, 0xab, 0xfe, 0xea, 0x34, 0x34, 0xe6, 0xfd,
|
||||
0x95, 0x42, 0x4f, 0xd2, 0x5f, 0xa1, 0xcc, 0x04, 0xfd, 0x15, 0x5a, 0x0f, 0xbe, 0x53, 0xe4, 0x1e,
|
||||
0xcc, 0x3a, 0xb8, 0x5f, 0x2c, 0xc8, 0x5c, 0xf0, 0x96, 0x4e, 0x70, 0x68, 0x73, 0x6b, 0xd3, 0x47,
|
||||
0xc7, 0x0b, 0x17, 0x4c, 0x5f, 0x38, 0x68, 0xd4, 0xce, 0x26, 0x1b, 0xab, 0xbf, 0x15, 0x21, 0xb3,
|
||||
0x21, 0xaf, 0xd3, 0x88, 0x05, 0x19, 0xbc, 0xa9, 0x22, 0x86, 0x4e, 0x78, 0xf0, 0xf6, 0xab, 0x7c,
|
||||
0x33, 0x11, 0x83, 0x5f, 0x8e, 0x6b, 0xbf, 0xfc, 0xf4, 0xd7, 0xf7, 0xe9, 0xcb, 0x90, 0x17, 0xa0,
|
||||
0xd7, 0x30, 0xe2, 0xc4, 0x86, 0xac, 0x7f, 0xe5, 0x41, 0x5e, 0x1d, 0xe7, 0x82, 0xa8, 0x7c, 0x6b,
|
||||
0x04, 0x2a, 0xd9, 0xa0, 0x03, 0x10, 0xdc, 0x38, 0x10, 0xad, 0xae, 0xa1, 0xdb, 0x93, 0xf2, 0xe2,
|
||||
0x28, 0xd8, 0x48, 0x9b, 0xc1, 0x3d, 0x83, 0xde, 0xe6, 0xd0, 0x0d, 0x86, 0xde, 0xa6, 0xe6, 0xba,
|
||||
0x22, 0xc6, 0xa6, 0x8c, 0x21, 0x3f, 0xc9, 0xc5, 0xc6, 0x30, 0x74, 0xcf, 0x10, 0x1b, 0xc3, 0x81,
|
||||
0x1b, 0x85, 0xe4, 0x18, 0x8a, 0x73, 0x66, 0x7c, 0x0c, 0xc3, 0xa7, 0xf6, 0xf8, 0x18, 0x0e, 0x1c,
|
||||
0x56, 0x47, 0xee, 0xa7, 0x58, 0x5e, 0xc2, 0x7e, 0x86, 0x57, 0xb8, 0x38, 0x0a, 0x36, 0xd2, 0x66,
|
||||
0x70, 0x4e, 0xd4, 0xdb, 0x1c, 0x3a, 0x8a, 0xea, 0x6d, 0x0e, 0x1f, 0x37, 0xe3, 0x6c, 0x3e, 0x83,
|
||||
0x4b, 0xe1, 0x96, 0x9b, 0xdc, 0x1e, 0xf3, 0x9c, 0x50, 0x5e, 0x1a, 0x0d, 0x4c, 0xb6, 0xfc, 0x15,
|
||||
0xe4, 0x07, 0x0e, 0xea, 0x44, 0xab, 0x51, 0x77, 0x31, 0x50, 0x5e, 0x1e, 0x03, 0x39, 0xd2, 0xf8,
|
||||
0xc0, 0x19, 0x54, 0x6f, 0x5c, 0x77, 0xce, 0xd6, 0x1b, 0xd7, 0x1e, 0x68, 0x13, 0x8c, 0x0f, 0x1c,
|
||||
0x35, 0xf5, 0xc6, 0x75, 0x67, 0x5a, 0xbd, 0x71, 0xfd, 0xb9, 0x35, 0x91, 0x64, 0xd8, 0xba, 0xc5,
|
||||
0x92, 0x6c, 0xb0, 0xdd, 0x8f, 0x25, 0x59, 0xb4, 0x77, 0x4f, 0x26, 0x99, 0xea, 0x33, 0xe3, 0x49,
|
||||
0x16, 0x69, 0x8e, 0xe3, 0x49, 0x16, 0x6d, 0x59, 0x47, 0x92, 0x4c, 0x2d, 0x38, 0x81, 0x64, 0x91,
|
||||
0x35, 0x2f, 0x8f, 0x81, 0x1c, 0x33, 0xce, 0x89, 0xc6, 0x75, 0xe7, 0xab, 0xa4, 0x38, 0x8f, 0x69,
|
||||
0x5c, 0xc6, 0x19, 0xbf, 0xc1, 0xb1, 0x71, 0x1e, 0xec, 0x71, 0x62, 0xe3, 0x1c, 0x69, 0x00, 0x46,
|
||||
0xc4, 0x59, 0xf5, 0x80, 0xf1, 0x71, 0x8e, 0x34, 0xae, 0xf1, 0x71, 0x8e, 0xb6, 0x93, 0x23, 0xf3,
|
||||
0x59, 0x2d, 0x38, 0x21, 0x9f, 0x23, 0x6b, 0x5e, 0x1e, 0x03, 0x99, 0x68, 0xbc, 0x76, 0xe3, 0xe8,
|
||||
0xf9, 0xfc, 0x85, 0xdf, 0xd9, 0xef, 0xef, 0xe7, 0xf3, 0xa9, 0xaf, 0x5f, 0xcc, 0xa7, 0x8e, 0xd8,
|
||||
0xef, 0x57, 0xf6, 0xfb, 0x83, 0xfd, 0x76, 0x67, 0xc4, 0x7f, 0xf4, 0xd6, 0xfe, 0x0d, 0x00, 0x00,
|
||||
0xff, 0xff, 0xf3, 0xcc, 0x22, 0xcd, 0x4a, 0x1c, 0x00, 0x00,
|
||||
0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xcd, 0x7b, 0x33, 0x9f, 0xcf, 0xbc, 0x7d, 0x33, 0x81, 0x7c,
|
||||
0xd3, 0xee, 0x79, 0x8e, 0xdd, 0xa9, 0xf4, 0x1d, 0xdb, 0xb3, 0x09, 0x69, 0xd9, 0xcd, 0x36, 0x75,
|
||||
0x2a, 0xee, 0xd3, 0x86, 0xd3, 0x6d, 0x5b, 0x5e, 0xe5, 0xe0, 0x8d, 0x72, 0xce, 0xed, 0xd3, 0xa6,
|
||||
0x2b, 0x01, 0xe5, 0xbc, 0xbd, 0xf3, 0x05, 0x6d, 0x7a, 0xea, 0x35, 0xe7, 0x1d, 0xf6, 0xa9, 0x7a,
|
||||
0x29, 0xed, 0xd9, 0x7b, 0xb6, 0x78, 0x5c, 0xe1, 0x4f, 0xd8, 0x7a, 0xb5, 0xdf, 0xd9, 0xdf, 0xb3,
|
||||
0x7a, 0x2b, 0xf2, 0x8f, 0x6c, 0x34, 0xee, 0x42, 0xe1, 0x3e, 0xf5, 0x36, 0xed, 0x16, 0x35, 0xe9,
|
||||
0x97, 0xfb, 0xd4, 0xf5, 0xc8, 0x2d, 0xc8, 0xf4, 0xd8, 0xeb, 0xb6, 0xd5, 0x9a, 0x4b, 0xbd, 0x92,
|
||||
0x5a, 0xcc, 0xd6, 0xe0, 0xc5, 0xf1, 0xfc, 0x0c, 0x47, 0xd4, 0x37, 0xcc, 0x19, 0xde, 0x55, 0x6f,
|
||||
0x19, 0xef, 0xc1, 0x65, 0xdf, 0xcc, 0xed, 0xdb, 0x3d, 0x97, 0x92, 0x65, 0x98, 0xe6, 0x9d, 0xc2,
|
||||
0x28, 0x57, 0x9d, 0xab, 0x0c, 0x4f, 0xa0, 0x22, 0xf0, 0x02, 0x65, 0x1c, 0x4f, 0xc1, 0x95, 0x07,
|
||||
0x96, 0x2b, 0x86, 0x70, 0x95, 0xeb, 0x7b, 0x90, 0xd9, 0xb5, 0x3a, 0x1e, 0x75, 0x5c, 0x1c, 0x65,
|
||||
0x59, 0x37, 0x4a, 0xd4, 0xac, 0x72, 0x4f, 0xda, 0x98, 0xca, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0xb0,
|
||||
0x91, 0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x7c, 0xc4, 0xa9, 0xc5, 0xac, 0x29, 0x5f, 0xc8, 0x0a,
|
||||
0xe4, 0xac, 0xd6, 0x76, 0xdf, 0xa1, 0xbb, 0xd6, 0x33, 0xd6, 0x97, 0xe6, 0x7d, 0xb5, 0x02, 0x9b,
|
||||
0x28, 0xd4, 0x37, 0x1e, 0x61, 0xab, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x47, 0x30, 0xd3, 0x69, 0xec,
|
||||
0xd0, 0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xda, 0x24, 0x91, 0x55, 0x1e, 0x08, 0xd3, 0x0f,
|
||||
0x18, 0xc1, 0x87, 0x26, 0x8e, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x0e, 0xeb, 0xfe, 0xdc, 0xea,
|
||||
0xbb, 0x73, 0xd3, 0x6c, 0xd8, 0x42, 0xf5, 0x4e, 0xdc, 0xb2, 0x6d, 0x31, 0xea, 0x2b, 0x0f, 0x7d,
|
||||
0xbc, 0x19, 0xb6, 0x25, 0x55, 0xb8, 0xc8, 0x94, 0xc3, 0xe6, 0x71, 0x51, 0x0c, 0x72, 0x23, 0x76,
|
||||
0xed, 0x19, 0xc8, 0x94, 0x50, 0x46, 0x73, 0x9e, 0x2f, 0x45, 0xb0, 0x06, 0x33, 0x62, 0x7d, 0x2e,
|
||||
0xf1, 0x46, 0x35, 0xeb, 0xf2, 0x5b, 0x90, 0x0b, 0x85, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a,
|
||||
0x59, 0x98, 0xfc, 0x91, 0xaf, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0a, 0xf2, 0x36, 0xf9, 0xf2,
|
||||
0x76, 0x7a, 0x2d, 0x65, 0xac, 0x43, 0x31, 0xb4, 0x1c, 0xa8, 0x91, 0x0a, 0x23, 0x83, 0x37, 0x08,
|
||||
0x32, 0x92, 0x44, 0x22, 0x61, 0xc6, 0x8f, 0x29, 0x28, 0x3e, 0xe9, 0xb7, 0x1a, 0x1e, 0x9d, 0x54,
|
||||
0xa1, 0xe4, 0x5d, 0xb8, 0x24, 0x40, 0x07, 0x6c, 0x91, 0x2c, 0xbb, 0x27, 0x02, 0xcc, 0x55, 0xaf,
|
||||
0xeb, 0x3c, 0x7e, 0x22, 0x21, 0x66, 0x8e, 0x1b, 0xe0, 0x0b, 0x79, 0x1d, 0xa6, 0xf9, 0x76, 0x63,
|
||||
0x74, 0x73, 0xbb, 0x1b, 0x49, 0xbc, 0x98, 0x02, 0x69, 0xd4, 0x80, 0x84, 0x63, 0x3d, 0xd1, 0xb6,
|
||||
0xd8, 0x84, 0xa2, 0x49, 0xbb, 0xf6, 0xc1, 0xe4, 0xf3, 0x65, 0x4c, 0xec, 0xda, 0x4e, 0x53, 0x32,
|
||||
0x31, 0x6b, 0xca, 0x17, 0xa3, 0x04, 0x24, 0x3c, 0x9e, 0x8c, 0x09, 0x37, 0xfd, 0xe3, 0x86, 0xdb,
|
||||
0x0e, 0xb9, 0xf0, 0xd8, 0x6b, 0xc4, 0x05, 0x47, 0x70, 0x17, 0xbc, 0xcb, 0xdf, 0xf4, 0xd2, 0x2c,
|
||||
0x98, 0x1d, 0xef, 0x4c, 0x9a, 0x9d, 0xc0, 0x0b, 0x94, 0xb1, 0xa6, 0x66, 0x37, 0xb1, 0x6b, 0x7f,
|
||||
0x1e, 0x61, 0xef, 0xc6, 0x3f, 0x98, 0x44, 0x78, 0xe3, 0x09, 0x92, 0x48, 0xd8, 0x6c, 0x38, 0x89,
|
||||
0xfc, 0x70, 0x8e, 0x49, 0x44, 0x17, 0x99, 0x36, 0x89, 0xb0, 0x10, 0x5c, 0xea, 0x1c, 0x58, 0x4d,
|
||||
0xae, 0x0e, 0x99, 0x44, 0x30, 0x84, 0x2d, 0xd9, 0x5c, 0xdf, 0x60, 0x21, 0x20, 0xa4, 0xde, 0x72,
|
||||
0xc9, 0x02, 0xcc, 0xa2, 0x96, 0x64, 0xb6, 0xc8, 0xd6, 0x72, 0x0c, 0x9d, 0x91, 0x62, 0x62, 0xb3,
|
||||
0x97, 0x6a, 0x72, 0xc9, 0x06, 0x14, 0xd8, 0x06, 0xb4, 0x1c, 0xda, 0xda, 0x76, 0x3d, 0xa6, 0x69,
|
||||
0x99, 0x1f, 0x0a, 0xd5, 0xff, 0xc7, 0x51, 0xbc, 0xc5, 0x51, 0x66, 0x1e, 0x8d, 0xc4, 0x9b, 0x26,
|
||||
0xc9, 0x64, 0xfe, 0x93, 0x24, 0x83, 0xcb, 0x15, 0x24, 0x19, 0xae, 0x9a, 0xc4, 0x24, 0x23, 0x64,
|
||||
0x24, 0x61, 0xc6, 0x47, 0x50, 0x5a, 0x77, 0x28, 0x8b, 0x17, 0x97, 0x4c, 0x09, 0x69, 0x15, 0x33,
|
||||
0x80, 0x54, 0xd1, 0xbc, 0x6e, 0x18, 0xb4, 0x08, 0x25, 0x81, 0x4d, 0xb8, 0x16, 0x19, 0x0c, 0xa3,
|
||||
0xba, 0x0b, 0x19, 0xa4, 0x01, 0x07, 0xbc, 0x9e, 0x30, 0xa0, 0xa9, 0xb0, 0xc6, 0xfb, 0x50, 0x64,
|
||||
0x7b, 0x2e, 0x12, 0xd9, 0x32, 0x40, 0xc0, 0x3a, 0xee, 0x9a, 0x3c, 0xa3, 0x31, 0xeb, 0x93, 0x6e,
|
||||
0x66, 0x7d, 0xce, 0xd9, 0xfc, 0x48, 0x78, 0x88, 0xd3, 0xc5, 0xf3, 0x73, 0x0a, 0x4a, 0x32, 0xcb,
|
||||
0x9d, 0x26, 0x26, 0x26, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x82, 0x2e, 0xa0, 0x8d, 0xca, 0xd1, 0xab,
|
||||
0x03, 0x39, 0x7a, 0x7c, 0x86, 0x22, 0x13, 0x38, 0xdd, 0x8a, 0x6c, 0x40, 0x49, 0xa6, 0xa6, 0x53,
|
||||
0x91, 0xf4, 0x3f, 0xb8, 0x16, 0x19, 0x05, 0x73, 0xdc, 0x9f, 0x69, 0xb8, 0xca, 0x35, 0x8e, 0xed,
|
||||
0x7e, 0x9a, 0xab, 0x47, 0xd3, 0xdc, 0x4a, 0x5c, 0x32, 0x89, 0x58, 0x0e, 0x67, 0xba, 0x6f, 0xd3,
|
||||
0x67, 0x9e, 0xe9, 0xb6, 0x22, 0x99, 0xee, 0x9d, 0x09, 0x83, 0xd3, 0x26, 0xbb, 0xa1, 0x6c, 0x32,
|
||||
0x7d, 0xb6, 0xd9, 0xe4, 0x63, 0x28, 0x0d, 0x86, 0x84, 0xc2, 0x78, 0x13, 0x66, 0x91, 0x28, 0x95,
|
||||
0x53, 0x12, 0x95, 0xe1, 0x83, 0x83, 0xcc, 0xb2, 0x49, 0xbd, 0xa7, 0xb6, 0xd3, 0x9e, 0x20, 0xb3,
|
||||
0xa0, 0x85, 0x2e, 0xb3, 0xf8, 0x83, 0x05, 0xba, 0xed, 0xc9, 0xa6, 0x24, 0xdd, 0x2a, 0x2b, 0x85,
|
||||
0x35, 0x9e, 0x88, 0xcc, 0x12, 0x89, 0x8c, 0xb0, 0x6a, 0x85, 0xad, 0x26, 0xae, 0x97, 0x78, 0xe6,
|
||||
0x42, 0x46, 0x1b, 0x2e, 0xe4, 0x74, 0x20, 0x64, 0xb4, 0xe5, 0x42, 0x46, 0x80, 0x9f, 0x6d, 0xce,
|
||||
0x28, 0xc6, 0xcf, 0xd4, 0xde, 0x3a, 0xf3, 0x30, 0xfd, 0xfd, 0x16, 0x89, 0xd4, 0xdf, 0x6f, 0xd8,
|
||||
0x7e, 0x82, 0xfd, 0x16, 0xb1, 0x7c, 0xb9, 0xf6, 0x5b, 0x4c, 0x70, 0xe7, 0xb9, 0xdf, 0x82, 0x90,
|
||||
0x82, 0xfd, 0x86, 0x44, 0x25, 0xee, 0x37, 0xc5, 0x9c, 0x0f, 0xc6, 0x8f, 0xe5, 0x7a, 0x67, 0xdf,
|
||||
0x65, 0x73, 0x0a, 0xe5, 0xe1, 0xa6, 0x6c, 0x89, 0xe4, 0x61, 0xc4, 0x71, 0x5d, 0x20, 0xc0, 0x97,
|
||||
0xaf, 0x3f, 0x44, 0x20, 0x5f, 0x84, 0x24, 0xc9, 0x57, 0x59, 0x29, 0xac, 0xaf, 0x25, 0xec, 0x38,
|
||||
0x81, 0x96, 0x22, 0x96, 0x2f, 0x97, 0x96, 0x62, 0x82, 0x3b, 0x4f, 0x2d, 0x05, 0x21, 0x05, 0x5a,
|
||||
0x42, 0x36, 0x12, 0xb5, 0xa4, 0xa8, 0xf3, 0xc1, 0xc6, 0x3e, 0x14, 0x3f, 0xb4, 0xad, 0xde, 0x63,
|
||||
0xbb, 0x4d, 0x7b, 0xa6, 0xcd, 0xca, 0x59, 0x5e, 0x70, 0x54, 0xe0, 0xaa, 0xc3, 0x9f, 0xe9, 0x36,
|
||||
0x17, 0x1c, 0x53, 0x94, 0xc7, 0xbb, 0x45, 0x84, 0xb3, 0x66, 0x51, 0x76, 0x7d, 0x2a, 0x7a, 0x84,
|
||||
0x1d, 0x3b, 0x44, 0x96, 0x10, 0xdf, 0x6d, 0xf4, 0x1a, 0x7b, 0xbe, 0x81, 0x3c, 0xa3, 0x11, 0xd9,
|
||||
0xf7, 0x50, 0x76, 0x09, 0x0b, 0xe3, 0xbb, 0xb4, 0xaa, 0xaf, 0x4e, 0x23, 0x63, 0x5e, 0x5f, 0x29,
|
||||
0xf4, 0x24, 0xf5, 0x15, 0xda, 0x4c, 0x50, 0x5f, 0xa1, 0xf7, 0xe0, 0x3b, 0x45, 0xee, 0xc3, 0xac,
|
||||
0x83, 0xeb, 0xc5, 0x48, 0xe6, 0x86, 0xb7, 0x75, 0x86, 0x43, 0x8b, 0x5b, 0x9b, 0x3e, 0x3a, 0x9e,
|
||||
0xbf, 0x60, 0xfa, 0xc6, 0x41, 0xa1, 0x76, 0x36, 0xbb, 0xb1, 0xfa, 0x5b, 0x11, 0x32, 0xeb, 0xf2,
|
||||
0x92, 0x8d, 0x58, 0x90, 0xc1, 0xfb, 0x2b, 0x62, 0xe8, 0x8c, 0x07, 0xef, 0xc4, 0xca, 0xb7, 0x12,
|
||||
0x31, 0xf8, 0xe5, 0xb8, 0xf6, 0xcb, 0x4f, 0x7f, 0x7d, 0x9f, 0xbe, 0x0c, 0x79, 0x01, 0x7a, 0x0d,
|
||||
0x19, 0x27, 0x36, 0x64, 0xfd, 0x8b, 0x10, 0xf2, 0xea, 0x38, 0xd7, 0x46, 0xe5, 0xdb, 0x23, 0x50,
|
||||
0xc9, 0x0e, 0x1d, 0x80, 0xe0, 0x1e, 0x82, 0x68, 0xc7, 0x1a, 0xba, 0x53, 0x29, 0x2f, 0x8c, 0x82,
|
||||
0x8d, 0xf4, 0x19, 0xdc, 0x33, 0xe8, 0x7d, 0x0e, 0xdd, 0x6b, 0xe8, 0x7d, 0x6a, 0xae, 0x2b, 0x62,
|
||||
0x7c, 0x4a, 0x0e, 0xf9, 0x49, 0x2e, 0x96, 0xc3, 0xd0, 0x3d, 0x43, 0x2c, 0x87, 0x03, 0x37, 0x0a,
|
||||
0xc9, 0x1c, 0x8a, 0x73, 0x66, 0x3c, 0x87, 0xe1, 0x53, 0x7b, 0x3c, 0x87, 0x03, 0x87, 0xd5, 0x91,
|
||||
0xeb, 0x29, 0xa6, 0x97, 0xb0, 0x9e, 0xe1, 0x19, 0x2e, 0x8c, 0x82, 0x8d, 0xf4, 0x19, 0x9c, 0x13,
|
||||
0xf5, 0x3e, 0x87, 0x8e, 0xa2, 0x7a, 0x9f, 0xc3, 0xc7, 0xcd, 0x38, 0x9f, 0xcf, 0xe0, 0x52, 0xb8,
|
||||
0xe4, 0x26, 0x77, 0xc6, 0x3c, 0x27, 0x94, 0x17, 0x47, 0x03, 0x93, 0x3d, 0x7f, 0x05, 0xf9, 0x81,
|
||||
0x83, 0x3a, 0xd1, 0x8e, 0xa8, 0xbb, 0x18, 0x28, 0x2f, 0x8d, 0x81, 0x1c, 0xe9, 0x7c, 0xe0, 0x0c,
|
||||
0xaa, 0x77, 0xae, 0x3b, 0x67, 0xeb, 0x9d, 0x6b, 0x0f, 0xb4, 0x09, 0xce, 0x07, 0x8e, 0x9a, 0x7a,
|
||||
0xe7, 0xba, 0x33, 0xad, 0xde, 0xb9, 0xfe, 0xdc, 0x9a, 0x28, 0x32, 0x2c, 0xdd, 0x62, 0x45, 0x36,
|
||||
0x58, 0xee, 0xc7, 0x8a, 0x2c, 0x5a, 0xbb, 0x27, 0x8b, 0x4c, 0xd5, 0x99, 0xf1, 0x22, 0x8b, 0x14,
|
||||
0xc7, 0xf1, 0x22, 0x8b, 0x96, 0xac, 0x23, 0x45, 0xa6, 0x26, 0x9c, 0x20, 0xb2, 0xc8, 0x9c, 0x97,
|
||||
0xc6, 0x40, 0x8e, 0xc9, 0x73, 0xa2, 0x73, 0xdd, 0xf9, 0x2a, 0x89, 0xe7, 0x31, 0x9d, 0x4b, 0x9e,
|
||||
0xf1, 0x1b, 0x1c, 0xcb, 0xf3, 0x60, 0x8d, 0x13, 0xcb, 0x73, 0xa4, 0x00, 0x18, 0xc1, 0xb3, 0xaa,
|
||||
0x01, 0xe3, 0x79, 0x8e, 0x14, 0xae, 0xf1, 0x3c, 0x47, 0xcb, 0xc9, 0x91, 0xfb, 0x59, 0x4d, 0x38,
|
||||
0x61, 0x3f, 0x47, 0xe6, 0xbc, 0x34, 0x06, 0x32, 0xd1, 0x79, 0xed, 0xc6, 0xd1, 0xf3, 0x9b, 0x17,
|
||||
0x7e, 0x67, 0xbf, 0xbf, 0x9f, 0xdf, 0x4c, 0x7d, 0xfd, 0xe2, 0x66, 0xea, 0x88, 0xfd, 0x7e, 0x65,
|
||||
0xbf, 0x3f, 0xd8, 0x6f, 0x67, 0x46, 0xfc, 0x9f, 0x6f, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff,
|
||||
0xbb, 0x97, 0x43, 0x36, 0x60, 0x1c, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ message UpdateNodeResponse {
|
|||
// RemoveNodeRequest requests to delete the specified node from store.
|
||||
message RemoveNodeRequest {
|
||||
string node_id = 1 [(gogoproto.customname) = "NodeID"];
|
||||
bool force = 2;
|
||||
}
|
||||
|
||||
message RemoveNodeResponse {
|
||||
|
|
|
|||
|
|
@ -309,15 +309,6 @@ func (s *Server) Run(ctx context.Context) error {
|
|||
logger := log.G(ctx).WithField("module", "ca")
|
||||
ctx = log.WithLogger(ctx, logger)
|
||||
|
||||
// Run() should never be called twice, but just in case, we're
|
||||
// attempting to close the started channel in a safe way
|
||||
select {
|
||||
case <-s.started:
|
||||
return fmt.Errorf("CA server cannot be started more than once")
|
||||
default:
|
||||
close(s.started)
|
||||
}
|
||||
|
||||
// Retrieve the channels to keep track of changes in the cluster
|
||||
// Retrieve all the currently registered nodes
|
||||
var nodes []*api.Node
|
||||
|
|
@ -346,6 +337,7 @@ func (s *Server) Run(ctx context.Context) error {
|
|||
s.mu.Lock()
|
||||
s.ctx, s.cancel = context.WithCancel(ctx)
|
||||
s.mu.Unlock()
|
||||
close(s.started)
|
||||
|
||||
if err != nil {
|
||||
log.G(ctx).WithFields(logrus.Fields{
|
||||
|
|
|
|||
|
|
@ -377,7 +377,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
|
|||
|
||||
node.Attachment.Network = nc.ingressNetwork.Copy()
|
||||
if err := a.allocateNode(ctx, nc, node); err != nil {
|
||||
log.G(ctx).Errorf("Fauled to allocate network resources for node %s: %v", node.ID, err)
|
||||
log.G(ctx).Errorf("Failed to allocate network resources for node %s: %v", node.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -283,7 +283,7 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest)
|
|||
return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
|
||||
}
|
||||
}
|
||||
if node.Status.State == api.NodeStatus_READY {
|
||||
if !request.Force && node.Status.State == api.NodeStatus_READY {
|
||||
return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
|
||||
}
|
||||
return store.DeleteNode(tx, request.NodeID)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"github.com/docker/swarmkit/manager/state"
|
||||
"github.com/docker/swarmkit/manager/state/store"
|
||||
"github.com/docker/swarmkit/manager/state/watch"
|
||||
"github.com/docker/swarmkit/picker"
|
||||
"github.com/docker/swarmkit/protobuf/ptypes"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
|
@ -60,8 +61,6 @@ var (
|
|||
// Config is configuration for Dispatcher. For default you should use
|
||||
// DefautConfig.
|
||||
type Config struct {
|
||||
// Addr configures the address the dispatcher reports to agents.
|
||||
Addr string
|
||||
HeartbeatPeriod time.Duration
|
||||
HeartbeatEpsilon time.Duration
|
||||
// RateLimitPeriod specifies how often node with same ID can try to register
|
||||
|
|
@ -90,7 +89,6 @@ type Cluster interface {
|
|||
// Dispatcher is responsible for dispatching tasks and tracking agent health.
|
||||
type Dispatcher struct {
|
||||
mu sync.Mutex
|
||||
addr string
|
||||
nodes *nodeStore
|
||||
store *store.MemoryStore
|
||||
mgrQueue *watch.Queue
|
||||
|
|
@ -121,7 +119,6 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
|
|||
// NOTE: each handler which does something with raft must add to Dispatcher.wg
|
||||
func New(cluster Cluster, c *Config) *Dispatcher {
|
||||
return &Dispatcher{
|
||||
addr: c.Addr,
|
||||
nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod),
|
||||
store: cluster.MemoryStore(),
|
||||
cluster: cluster,
|
||||
|
|
@ -142,7 +139,11 @@ func getWeightedPeers(cluster Cluster) []*api.WeightedPeer {
|
|||
NodeID: m.NodeID,
|
||||
Addr: m.Addr,
|
||||
},
|
||||
Weight: 1,
|
||||
|
||||
// TODO(stevvooe): Calculate weight of manager selection based on
|
||||
// cluster-level observations, such as number of connections and
|
||||
// load.
|
||||
Weight: picker.DefaultObservationWeight,
|
||||
})
|
||||
}
|
||||
return mgrs
|
||||
|
|
@ -574,14 +575,18 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
|
|||
}
|
||||
|
||||
// bursty events should be processed in batches and sent out snapshot
|
||||
const modificationBatchLimit = 200
|
||||
const eventPausedGap = 50 * time.Millisecond
|
||||
var modificationCnt int
|
||||
// eventPaused is true when there have been modifications
|
||||
// but next event has not arrived within eventPausedGap
|
||||
eventPaused := false
|
||||
const (
|
||||
modificationBatchLimit = 200
|
||||
eventPausedGap = 50 * time.Millisecond
|
||||
)
|
||||
var (
|
||||
modificationCnt int
|
||||
eventPausedTimer *time.Timer
|
||||
eventPausedTimeout <-chan time.Time
|
||||
)
|
||||
|
||||
for modificationCnt < modificationBatchLimit && !eventPaused {
|
||||
batchingLoop:
|
||||
for modificationCnt < modificationBatchLimit {
|
||||
select {
|
||||
case event := <-nodeTasks:
|
||||
switch v := event.(type) {
|
||||
|
|
@ -602,16 +607,24 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
|
|||
delete(tasksMap, v.Task.ID)
|
||||
modificationCnt++
|
||||
}
|
||||
case <-time.After(eventPausedGap):
|
||||
if modificationCnt > 0 {
|
||||
eventPaused = true
|
||||
if eventPausedTimer != nil {
|
||||
eventPausedTimer.Reset(eventPausedGap)
|
||||
} else {
|
||||
eventPausedTimer = time.NewTimer(eventPausedGap)
|
||||
eventPausedTimeout = eventPausedTimer.C
|
||||
}
|
||||
case <-eventPausedTimeout:
|
||||
break batchingLoop
|
||||
case <-stream.Context().Done():
|
||||
return stream.Context().Err()
|
||||
case <-d.ctx.Done():
|
||||
return d.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
if eventPausedTimer != nil {
|
||||
eventPausedTimer.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -43,26 +43,29 @@ func (rn *registeredNode) checkSessionID(sessionID string) error {
|
|||
}
|
||||
|
||||
type nodeStore struct {
|
||||
periodChooser *periodChooser
|
||||
gracePeriodMultiplier time.Duration
|
||||
rateLimitPeriod time.Duration
|
||||
nodes map[string]*registeredNode
|
||||
mu sync.RWMutex
|
||||
periodChooser *periodChooser
|
||||
gracePeriodMultiplierNormal time.Duration
|
||||
gracePeriodMultiplierUnknown time.Duration
|
||||
rateLimitPeriod time.Duration
|
||||
nodes map[string]*registeredNode
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int, rateLimitPeriod time.Duration) *nodeStore {
|
||||
return &nodeStore{
|
||||
nodes: make(map[string]*registeredNode),
|
||||
periodChooser: newPeriodChooser(hbPeriod, hbEpsilon),
|
||||
gracePeriodMultiplier: time.Duration(graceMultiplier),
|
||||
rateLimitPeriod: rateLimitPeriod,
|
||||
nodes: make(map[string]*registeredNode),
|
||||
periodChooser: newPeriodChooser(hbPeriod, hbEpsilon),
|
||||
gracePeriodMultiplierNormal: time.Duration(graceMultiplier),
|
||||
gracePeriodMultiplierUnknown: time.Duration(graceMultiplier) * 2,
|
||||
rateLimitPeriod: rateLimitPeriod,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *nodeStore) updatePeriod(hbPeriod, hbEpsilon time.Duration, gracePeriodMultiplier int) {
|
||||
s.mu.Lock()
|
||||
s.periodChooser = newPeriodChooser(hbPeriod, hbEpsilon)
|
||||
s.gracePeriodMultiplier = time.Duration(gracePeriodMultiplier)
|
||||
s.gracePeriodMultiplierNormal = time.Duration(gracePeriodMultiplier)
|
||||
s.gracePeriodMultiplierUnknown = s.gracePeriodMultiplierNormal * 2
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
|
|
@ -79,7 +82,7 @@ func (s *nodeStore) AddUnknown(n *api.Node, expireFunc func()) error {
|
|||
Node: n,
|
||||
}
|
||||
s.nodes[n.ID] = rn
|
||||
rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
|
||||
rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierUnknown, expireFunc)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +127,7 @@ func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode {
|
|||
Disconnect: make(chan struct{}),
|
||||
}
|
||||
s.nodes[n.ID] = rn
|
||||
rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
|
||||
rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierNormal, expireFunc)
|
||||
return rn
|
||||
}
|
||||
|
||||
|
|
@ -154,7 +157,7 @@ func (s *nodeStore) Heartbeat(id, sid string) (time.Duration, error) {
|
|||
return 0, err
|
||||
}
|
||||
period := s.periodChooser.Choose() // base period for node
|
||||
grace := period * time.Duration(s.gracePeriodMultiplier)
|
||||
grace := period * time.Duration(s.gracePeriodMultiplierNormal)
|
||||
rn.mu.Lock()
|
||||
rn.Heartbeat.Update(grace)
|
||||
rn.Heartbeat.Beat()
|
||||
|
|
|
|||
|
|
@ -89,9 +89,11 @@ type Manager struct {
|
|||
server *grpc.Server
|
||||
localserver *grpc.Server
|
||||
RaftNode *raft.Node
|
||||
connSelector *raftpicker.ConnSelector
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
started chan struct{}
|
||||
stopped chan struct{}
|
||||
}
|
||||
|
||||
|
|
@ -139,9 +141,6 @@ func New(config *Config) (*Manager, error) {
|
|||
tcpAddr = net.JoinHostPort("0.0.0.0", tcpAddrPort)
|
||||
}
|
||||
|
||||
// FIXME(aaronl): Remove this. It appears to be unused.
|
||||
dispatcherConfig.Addr = tcpAddr
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(config.ProtoAddr["unix"]), 0700)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create socket directory: %v", err)
|
||||
|
|
@ -220,6 +219,7 @@ func New(config *Config) (*Manager, error) {
|
|||
server: grpc.NewServer(opts...),
|
||||
localserver: grpc.NewServer(opts...),
|
||||
RaftNode: RaftNode,
|
||||
started: make(chan struct{}),
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
|
||||
|
|
@ -428,11 +428,12 @@ func (m *Manager) Run(parent context.Context) error {
|
|||
}()
|
||||
|
||||
proxyOpts := []grpc.DialOption{
|
||||
grpc.WithBackoffMaxDelay(2 * time.Second),
|
||||
grpc.WithBackoffMaxDelay(time.Second),
|
||||
grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
|
||||
}
|
||||
|
||||
cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...)
|
||||
m.connSelector = cs
|
||||
|
||||
authorize := func(ctx context.Context, roles []string) error {
|
||||
// Authorize the remote roles, ensure they can only be forwarded by managers
|
||||
|
|
@ -506,6 +507,8 @@ func (m *Manager) Run(parent context.Context) error {
|
|||
return fmt.Errorf("can't initialize raft node: %v", err)
|
||||
}
|
||||
|
||||
close(m.started)
|
||||
|
||||
go func() {
|
||||
err := m.RaftNode.Run(ctx)
|
||||
if err != nil {
|
||||
|
|
@ -560,12 +563,15 @@ func (m *Manager) Run(parent context.Context) error {
|
|||
func (m *Manager) Stop(ctx context.Context) {
|
||||
log.G(ctx).Info("Stopping manager")
|
||||
|
||||
// It's not safe to start shutting down while the manager is still
|
||||
// starting up.
|
||||
<-m.started
|
||||
|
||||
// the mutex stops us from trying to stop while we're alrady stopping, or
|
||||
// from returning before we've finished stopping.
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
select {
|
||||
|
||||
// check to see that we've already stopped
|
||||
case <-m.stopped:
|
||||
return
|
||||
|
|
@ -600,6 +606,9 @@ func (m *Manager) Stop(ctx context.Context) {
|
|||
m.keyManager.Stop()
|
||||
}
|
||||
|
||||
if m.connSelector != nil {
|
||||
m.connSelector.Stop()
|
||||
}
|
||||
m.RaftNode.Shutdown()
|
||||
// some time after this point, Run will receive an error from one of these
|
||||
m.server.Stop()
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package raftpicker
|
|||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc"
|
||||
|
|
@ -14,46 +15,37 @@ type picker struct {
|
|||
addr string
|
||||
raft AddrSelector
|
||||
conn *grpc.Conn
|
||||
cc *grpc.ClientConn
|
||||
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newPicker(raft AddrSelector, addr string) *picker {
|
||||
return &picker{
|
||||
raft: raft,
|
||||
addr: addr,
|
||||
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Init does initial processing for the Picker, e.g., initiate some connections.
|
||||
func (p *picker) Init(cc *grpc.ClientConn) error {
|
||||
p.cc = cc
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *picker) initConn() error {
|
||||
if p.conn == nil {
|
||||
conn, err := grpc.NewConn(p.cc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.conn = conn
|
||||
conn, err := grpc.NewConn(cc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.conn = conn
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
|
||||
// or some error happens.
|
||||
func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
|
||||
p.mu.Lock()
|
||||
if err := p.initConn(); err != nil {
|
||||
p.mu.Unlock()
|
||||
if err := p.updateConn(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
addr, err := p.raft.LeaderAddr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.mu.Lock()
|
||||
if p.addr != addr {
|
||||
p.addr = addr
|
||||
p.conn.NotifyReset()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
return p.conn.Wait(ctx)
|
||||
}
|
||||
|
||||
|
|
@ -89,15 +81,46 @@ func (p *picker) Reset() error {
|
|||
|
||||
// Close closes all the Conn's owned by this Picker.
|
||||
func (p *picker) Close() error {
|
||||
close(p.stop)
|
||||
<-p.done
|
||||
return p.conn.Close()
|
||||
}
|
||||
|
||||
func (p *picker) updateConn() error {
|
||||
addr, err := p.raft.LeaderAddr()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.mu.Lock()
|
||||
if p.addr != addr {
|
||||
p.addr = addr
|
||||
p.Reset()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *picker) updateLoop() {
|
||||
defer close(p.done)
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
p.updateConn()
|
||||
case <-p.stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ConnSelector is struct for obtaining connection with raftpicker.
|
||||
type ConnSelector struct {
|
||||
mu sync.Mutex
|
||||
cc *grpc.ClientConn
|
||||
cluster RaftCluster
|
||||
opts []grpc.DialOption
|
||||
picker *picker
|
||||
}
|
||||
|
||||
// NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which
|
||||
|
|
@ -122,8 +145,9 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
picker := &picker{raft: c.cluster, addr: addr}
|
||||
opts := append(c.opts, grpc.WithPicker(picker))
|
||||
c.picker = newPicker(c.cluster, addr)
|
||||
go c.picker.updateLoop()
|
||||
opts := append(c.opts, grpc.WithPicker(c.picker))
|
||||
cc, err := grpc.Dial(addr, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -131,3 +155,13 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
|
|||
c.cc = cc
|
||||
return c.cc, nil
|
||||
}
|
||||
|
||||
// Stop cancels tracking loop for raftpicker and closes it.
|
||||
func (c *ConnSelector) Stop() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.picker == nil {
|
||||
return
|
||||
}
|
||||
c.picker.Close()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -451,6 +451,17 @@ func (n *Node) Shutdown() {
|
|||
}
|
||||
}
|
||||
|
||||
// isShutdown indicates if node was shut down.
|
||||
// This method should be called under n.stopMu to avoid races with n.stop().
|
||||
func (n *Node) isShutdown() bool {
|
||||
select {
|
||||
case <-n.Ctx.Done():
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Node) stop() {
|
||||
n.stopMu.Lock()
|
||||
defer n.stopMu.Unlock()
|
||||
|
|
@ -763,7 +774,10 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
|
|||
func (n *Node) LeaderAddr() (string, error) {
|
||||
n.stopMu.RLock()
|
||||
defer n.stopMu.RUnlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if n.isShutdown() {
|
||||
return "", fmt.Errorf("raft node is shut down")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := WaitForLeader(ctx, n); err != nil {
|
||||
return "", ErrNoClusterLeader
|
||||
|
|
@ -1288,7 +1302,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership.
|
|||
|
||||
// SubscribeLeadership returns channel to which events about leadership change
|
||||
// will be sent in form of raft.LeadershipState. Also cancel func is returned -
|
||||
// it should be called when listener is not longer interested in events.
|
||||
// it should be called when listener is no longer interested in events.
|
||||
func (n *Node) SubscribeLeadership() (q chan events.Event, cancel func()) {
|
||||
ch := events.NewChannel(0)
|
||||
sink := events.Sink(events.NewQueue(ch))
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ import (
|
|||
|
||||
var errRemotesUnavailable = fmt.Errorf("no remote hosts provided")
|
||||
|
||||
// DefaultObservationWeight provides a weight to use for positive observations
|
||||
// that will balance well under repeated observations.
|
||||
const DefaultObservationWeight = 10
|
||||
|
||||
// Remotes keeps track of remote addresses by weight, informed by
|
||||
// observations.
|
||||
type Remotes interface {
|
||||
|
|
@ -49,7 +53,7 @@ func NewRemotes(peers ...api.Peer) Remotes {
|
|||
}
|
||||
|
||||
for _, peer := range peers {
|
||||
mwr.Observe(peer, 1)
|
||||
mwr.Observe(peer, DefaultObservationWeight)
|
||||
}
|
||||
|
||||
return mwr
|
||||
|
|
@ -96,7 +100,7 @@ func (mwr *remotesWeightedRandom) Select(excludes ...string) (api.Peer, error) {
|
|||
|
||||
// bias to zero-weighted remotes have same probability. otherwise, we
|
||||
// always select first entry when all are zero.
|
||||
const bias = 0.1
|
||||
const bias = 0.001
|
||||
|
||||
// clear out workspace
|
||||
mwr.cdf = mwr.cdf[:0]
|
||||
|
|
@ -165,7 +169,7 @@ const (
|
|||
// See
|
||||
// https://en.wikipedia.org/wiki/Exponential_smoothing#Basic_exponential_smoothing
|
||||
// for details.
|
||||
remoteWeightSmoothingFactor = 0.7
|
||||
remoteWeightSmoothingFactor = 0.5
|
||||
remoteWeightMax = 1 << 8
|
||||
)
|
||||
|
||||
|
|
@ -228,7 +232,7 @@ func (p *Picker) Init(cc *grpc.ClientConn) error {
|
|||
peer := p.peer
|
||||
p.mu.Unlock()
|
||||
|
||||
p.r.ObserveIfExists(peer, 1)
|
||||
p.r.ObserveIfExists(peer, DefaultObservationWeight)
|
||||
c, err := grpc.NewConn(cc)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -248,7 +252,7 @@ func (p *Picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
|
|||
p.mu.Unlock()
|
||||
transport, err := p.conn.Wait(ctx)
|
||||
if err != nil {
|
||||
p.r.ObserveIfExists(peer, -1)
|
||||
p.r.ObserveIfExists(peer, -DefaultObservationWeight)
|
||||
}
|
||||
|
||||
return transport, err
|
||||
|
|
@ -261,7 +265,7 @@ func (p *Picker) PickAddr() (string, error) {
|
|||
peer := p.peer
|
||||
p.mu.Unlock()
|
||||
|
||||
p.r.ObserveIfExists(peer, -1) // downweight the current addr
|
||||
p.r.ObserveIfExists(peer, -DefaultObservationWeight) // downweight the current addr
|
||||
|
||||
var err error
|
||||
peer, err = p.r.Select()
|
||||
|
|
@ -299,15 +303,15 @@ func (p *Picker) WaitForStateChange(ctx context.Context, sourceState grpc.Connec
|
|||
// TODO(stevvooe): This is questionable, but we'll see how it works.
|
||||
switch state {
|
||||
case grpc.Idle:
|
||||
p.r.ObserveIfExists(peer, 1)
|
||||
p.r.ObserveIfExists(peer, DefaultObservationWeight)
|
||||
case grpc.Connecting:
|
||||
p.r.ObserveIfExists(peer, 1)
|
||||
p.r.ObserveIfExists(peer, DefaultObservationWeight)
|
||||
case grpc.Ready:
|
||||
p.r.ObserveIfExists(peer, 1)
|
||||
p.r.ObserveIfExists(peer, DefaultObservationWeight)
|
||||
case grpc.TransientFailure:
|
||||
p.r.ObserveIfExists(peer, -1)
|
||||
p.r.ObserveIfExists(peer, -DefaultObservationWeight)
|
||||
case grpc.Shutdown:
|
||||
p.r.ObserveIfExists(peer, -1)
|
||||
p.r.ObserveIfExists(peer, -DefaultObservationWeight)
|
||||
}
|
||||
|
||||
return state, err
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue