diff --git a/vendor.conf b/vendor.conf index 120dd9d378..8d1bed0745 100644 --- a/vendor.conf +++ b/vendor.conf @@ -114,7 +114,7 @@ github.com/dmcgowan/go-tar go1.10 github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f # cluster -github.com/docker/swarmkit de950a7ed842c7b7e47e9451cde9bf8f96031894 +github.com/docker/swarmkit 4429c763170d9ca96929249353c3270c19e7d39e github.com/gogo/protobuf v0.4 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e diff --git a/vendor/github.com/docker/swarmkit/agent/exec/controller.go b/vendor/github.com/docker/swarmkit/agent/exec/controller.go index 9b4fc7bca1..c9e9343fd7 100644 --- a/vendor/github.com/docker/swarmkit/agent/exec/controller.go +++ b/vendor/github.com/docker/swarmkit/agent/exec/controller.go @@ -288,7 +288,9 @@ func Do(ctx context.Context, task *api.Task, ctlr Controller) (*api.TaskStatus, status.PortStatus = portStatus }() - if task.DesiredState == api.TaskStateShutdown { + // this branch bounds the largest state achievable in the agent as SHUTDOWN, which + // is exactly the correct behavior for the agent. + if task.DesiredState >= api.TaskStateShutdown { if status.State >= api.TaskStateCompleted { return noop() } diff --git a/vendor/github.com/docker/swarmkit/api/ca.pb.go b/vendor/github.com/docker/swarmkit/api/ca.pb.go index caaa06cc83..7d289126a1 100644 --- a/vendor/github.com/docker/swarmkit/api/ca.pb.go +++ b/vendor/github.com/docker/swarmkit/api/ca.pb.go @@ -131,6 +131,8 @@ LeaveResponse ProcessRaftMessageRequest ProcessRaftMessageResponse + StreamRaftMessageRequest + StreamRaftMessageResponse ResolveAddressRequest ResolveAddressResponse InternalRaftRequest @@ -235,6 +237,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -984,12 +987,12 @@ func NewRaftProxyCAServer(local CAServer, connSelector raftselector.ConnProvider redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1126,12 +1129,12 @@ func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/control.pb.go b/vendor/github.com/docker/swarmkit/api/control.pb.go index 13ef4822cb..fb28c5d330 100644 --- a/vendor/github.com/docker/swarmkit/api/control.pb.go +++ b/vendor/github.com/docker/swarmkit/api/control.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -5859,12 +5860,12 @@ func NewRaftProxyControlServer(local ControlServer, connSelector raftselector.Co redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go b/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go index 8e9b038cad..120df8811f 100644 --- a/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go +++ b/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go @@ -24,6 +24,7 @@ import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1602,12 +1603,12 @@ func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector raftselec redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/health.pb.go b/vendor/github.com/docker/swarmkit/api/health.pb.go index 757db6acbf..ed7df73054 100644 --- a/vendor/github.com/docker/swarmkit/api/health.pb.go +++ b/vendor/github.com/docker/swarmkit/api/health.pb.go @@ -17,6 +17,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -286,12 +287,12 @@ func NewRaftProxyHealthServer(local HealthServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/logbroker.pb.go b/vendor/github.com/docker/swarmkit/api/logbroker.pb.go index 58515aa11e..1108088fba 100644 --- a/vendor/github.com/docker/swarmkit/api/logbroker.pb.go +++ b/vendor/github.com/docker/swarmkit/api/logbroker.pb.go @@ -20,6 +20,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1273,12 +1274,12 @@ func NewRaftProxyLogsServer(local LogsServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1396,12 +1397,12 @@ func NewRaftProxyLogBrokerServer(local LogBrokerServer, connSelector raftselecto redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/raft.pb.go b/vendor/github.com/docker/swarmkit/api/raft.pb.go index 4710ee6b01..ddee3e18a6 100644 --- a/vendor/github.com/docker/swarmkit/api/raft.pb.go +++ b/vendor/github.com/docker/swarmkit/api/raft.pb.go @@ -21,6 +21,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -133,6 +134,23 @@ func (m *ProcessRaftMessageResponse) Reset() { *m = ProcessRa func (*ProcessRaftMessageResponse) ProtoMessage() {} func (*ProcessRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} } +// Raft message streaming request. +type StreamRaftMessageRequest struct { + Message *raftpb.Message `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` +} + +func (m *StreamRaftMessageRequest) Reset() { *m = StreamRaftMessageRequest{} } +func (*StreamRaftMessageRequest) ProtoMessage() {} +func (*StreamRaftMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } + +// Raft message streaming response. +type StreamRaftMessageResponse struct { +} + +func (m *StreamRaftMessageResponse) Reset() { *m = StreamRaftMessageResponse{} } +func (*StreamRaftMessageResponse) ProtoMessage() {} +func (*StreamRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } + type ResolveAddressRequest struct { // raft_id is the ID to resolve to an address. RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"` @@ -140,7 +158,7 @@ type ResolveAddressRequest struct { func (m *ResolveAddressRequest) Reset() { *m = ResolveAddressRequest{} } func (*ResolveAddressRequest) ProtoMessage() {} -func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } +func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } type ResolveAddressResponse struct { // Addr specifies the address of the member @@ -149,7 +167,7 @@ type ResolveAddressResponse struct { func (m *ResolveAddressResponse) Reset() { *m = ResolveAddressResponse{} } func (*ResolveAddressResponse) ProtoMessage() {} -func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } +func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } // Contains one of many protobuf encoded objects to replicate // over the raft backend with a request ID to track when the @@ -161,7 +179,7 @@ type InternalRaftRequest struct { func (m *InternalRaftRequest) Reset() { *m = InternalRaftRequest{} } func (*InternalRaftRequest) ProtoMessage() {} -func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } +func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{11} } // StoreAction defines a target and operation to apply on the storage system. type StoreAction struct { @@ -181,7 +199,7 @@ type StoreAction struct { func (m *StoreAction) Reset() { *m = StoreAction{} } func (*StoreAction) ProtoMessage() {} -func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } +func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{12} } type isStoreAction_Target interface { isStoreAction_Target() @@ -512,6 +530,8 @@ func init() { proto.RegisterType((*LeaveResponse)(nil), "docker.swarmkit.v1.LeaveResponse") proto.RegisterType((*ProcessRaftMessageRequest)(nil), "docker.swarmkit.v1.ProcessRaftMessageRequest") proto.RegisterType((*ProcessRaftMessageResponse)(nil), "docker.swarmkit.v1.ProcessRaftMessageResponse") + proto.RegisterType((*StreamRaftMessageRequest)(nil), "docker.swarmkit.v1.StreamRaftMessageRequest") + proto.RegisterType((*StreamRaftMessageResponse)(nil), "docker.swarmkit.v1.StreamRaftMessageResponse") proto.RegisterType((*ResolveAddressRequest)(nil), "docker.swarmkit.v1.ResolveAddressRequest") proto.RegisterType((*ResolveAddressResponse)(nil), "docker.swarmkit.v1.ResolveAddressResponse") proto.RegisterType((*InternalRaftRequest)(nil), "docker.swarmkit.v1.InternalRaftRequest") @@ -539,6 +559,14 @@ func (p *authenticatedWrapperRaftServer) ProcessRaftMessage(ctx context.Context, return p.local.ProcessRaftMessage(ctx, r) } +func (p *authenticatedWrapperRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + + if err := p.authorize(stream.Context(), []string{"swarm-manager"}); err != nil { + return err + } + return p.local.StreamRaftMessage(stream) +} + func (p *authenticatedWrapperRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { @@ -673,6 +701,16 @@ func (m *ProcessRaftMessageResponse) Copy() *ProcessRaftMessageResponse { } func (m *ProcessRaftMessageResponse) CopyFrom(src interface{}) {} +func (m *StreamRaftMessageResponse) Copy() *StreamRaftMessageResponse { + if m == nil { + return nil + } + o := &StreamRaftMessageResponse{} + o.CopyFrom(m) + return o +} + +func (m *StreamRaftMessageResponse) CopyFrom(src interface{}) {} func (m *ResolveAddressRequest) Copy() *ResolveAddressRequest { if m == nil { return nil @@ -813,6 +851,12 @@ type RaftClient interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMessageRequest, opts ...grpc.CallOption) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) } @@ -834,6 +878,40 @@ func (c *raftClient) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMess return out, nil } +func (c *raftClient) StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Raft_serviceDesc.Streams[0], c.cc, "/docker.swarmkit.v1.Raft/StreamRaftMessage", opts...) + if err != nil { + return nil, err + } + x := &raftStreamRaftMessageClient{stream} + return x, nil +} + +type Raft_StreamRaftMessageClient interface { + Send(*StreamRaftMessageRequest) error + CloseAndRecv() (*StreamRaftMessageResponse, error) + grpc.ClientStream +} + +type raftStreamRaftMessageClient struct { + grpc.ClientStream +} + +func (x *raftStreamRaftMessageClient) Send(m *StreamRaftMessageRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageClient) CloseAndRecv() (*StreamRaftMessageResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(StreamRaftMessageResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *raftClient) ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) { out := new(ResolveAddressResponse) err := grpc.Invoke(ctx, "/docker.swarmkit.v1.Raft/ResolveAddress", in, out, c.cc, opts...) @@ -849,6 +927,12 @@ type RaftServer interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(context.Context, *ProcessRaftMessageRequest) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(Raft_StreamRaftMessageServer) error // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(context.Context, *ResolveAddressRequest) (*ResolveAddressResponse, error) } @@ -875,6 +959,32 @@ func _Raft_ProcessRaftMessage_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Raft_StreamRaftMessage_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RaftServer).StreamRaftMessage(&raftStreamRaftMessageServer{stream}) +} + +type Raft_StreamRaftMessageServer interface { + SendAndClose(*StreamRaftMessageResponse) error + Recv() (*StreamRaftMessageRequest, error) + grpc.ServerStream +} + +type raftStreamRaftMessageServer struct { + grpc.ServerStream +} + +func (x *raftStreamRaftMessageServer) SendAndClose(m *StreamRaftMessageResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageServer) Recv() (*StreamRaftMessageRequest, error) { + m := new(StreamRaftMessageRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func _Raft_ResolveAddress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ResolveAddressRequest) if err := dec(in); err != nil { @@ -906,7 +1016,13 @@ var _Raft_serviceDesc = grpc.ServiceDesc{ Handler: _Raft_ResolveAddress_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamRaftMessage", + Handler: _Raft_StreamRaftMessage_Handler, + ClientStreams: true, + }, + }, Metadata: "github.com/docker/swarmkit/api/raft.proto", } @@ -1212,6 +1328,52 @@ func (m *ProcessRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *StreamRaftMessageRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Message != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.Message.Size())) + n4, err := m.Message.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + return i, nil +} + +func (m *StreamRaftMessageResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func (m *ResolveAddressRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1315,11 +1477,11 @@ func (m *StoreAction) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRaft(dAtA, i, uint64(m.Action)) } if m.Target != nil { - nn4, err := m.Target.MarshalTo(dAtA[i:]) + nn5, err := m.Target.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn4 + i += nn5 } return i, nil } @@ -1330,11 +1492,11 @@ func (m *StoreAction_Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Node.Size())) - n5, err := m.Node.MarshalTo(dAtA[i:]) + n6, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n6 } return i, nil } @@ -1344,11 +1506,11 @@ func (m *StoreAction_Service) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Service.Size())) - n6, err := m.Service.MarshalTo(dAtA[i:]) + n7, err := m.Service.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n7 } return i, nil } @@ -1358,11 +1520,11 @@ func (m *StoreAction_Task) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Task.Size())) - n7, err := m.Task.MarshalTo(dAtA[i:]) + n8, err := m.Task.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n8 } return i, nil } @@ -1372,11 +1534,11 @@ func (m *StoreAction_Network) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Network.Size())) - n8, err := m.Network.MarshalTo(dAtA[i:]) + n9, err := m.Network.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n8 + i += n9 } return i, nil } @@ -1386,11 +1548,11 @@ func (m *StoreAction_Cluster) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Cluster.Size())) - n9, err := m.Cluster.MarshalTo(dAtA[i:]) + n10, err := m.Cluster.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } return i, nil } @@ -1400,11 +1562,11 @@ func (m *StoreAction_Secret) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x3a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Secret.Size())) - n10, err := m.Secret.MarshalTo(dAtA[i:]) + n11, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } return i, nil } @@ -1414,11 +1576,11 @@ func (m *StoreAction_Resource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Resource.Size())) - n11, err := m.Resource.MarshalTo(dAtA[i:]) + n12, err := m.Resource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n12 } return i, nil } @@ -1428,11 +1590,11 @@ func (m *StoreAction_Extension) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x4a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Extension.Size())) - n12, err := m.Extension.MarshalTo(dAtA[i:]) + n13, err := m.Extension.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n13 } return i, nil } @@ -1442,11 +1604,11 @@ func (m *StoreAction_Config) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Config.Size())) - n13, err := m.Config.MarshalTo(dAtA[i:]) + n14, err := m.Config.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } return i, nil } @@ -1488,12 +1650,12 @@ func NewRaftProxyRaftServer(local RaftServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1585,6 +1747,63 @@ func (p *raftProxyRaftServer) ProcessRaftMessage(ctx context.Context, r *Process return resp, err } +type Raft_StreamRaftMessageServerWrapper struct { + Raft_StreamRaftMessageServer + ctx context.Context +} + +func (s Raft_StreamRaftMessageServerWrapper) Context() context.Context { + return s.ctx +} + +func (p *raftProxyRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + ctx := stream.Context() + conn, err := p.connSelector.LeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + ctx, err = p.runCtxMods(ctx, p.localCtxMods) + if err != nil { + return err + } + streamWrapper := Raft_StreamRaftMessageServerWrapper{ + Raft_StreamRaftMessageServer: stream, + ctx: ctx, + } + return p.local.StreamRaftMessage(streamWrapper) + } + return err + } + ctx, err = p.runCtxMods(ctx, p.remoteCtxMods) + if err != nil { + return err + } + clientStream, err := NewRaftClient(conn).StreamRaftMessage(ctx) + + if err != nil { + return err + } + + for { + msg, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if err := clientStream.Send(msg); err != nil { + return err + } + } + + reply, err := clientStream.CloseAndRecv() + if err != nil { + return err + } + + return stream.SendAndClose(reply) +} + func (p *raftProxyRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { conn, err := p.connSelector.LeaderConn(ctx) @@ -1630,12 +1849,12 @@ func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector r redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1843,6 +2062,22 @@ func (m *ProcessRaftMessageResponse) Size() (n int) { return n } +func (m *StreamRaftMessageRequest) Size() (n int) { + var l int + _ = l + if m.Message != nil { + l = m.Message.Size() + n += 1 + l + sovRaft(uint64(l)) + } + return n +} + +func (m *StreamRaftMessageResponse) Size() (n int) { + var l int + _ = l + return n +} + func (m *ResolveAddressRequest) Size() (n int) { var l int _ = l @@ -2057,6 +2292,25 @@ func (this *ProcessRaftMessageResponse) String() string { }, "") return s } +func (this *StreamRaftMessageRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageRequest{`, + `Message:` + strings.Replace(fmt.Sprintf("%v", this.Message), "Message", "raftpb.Message", 1) + `,`, + `}`, + }, "") + return s +} +func (this *StreamRaftMessageResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageResponse{`, + `}`, + }, "") + return s +} func (this *ResolveAddressRequest) String() string { if this == nil { return "nil" @@ -2861,6 +3115,139 @@ func (m *ProcessRaftMessageResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *StreamRaftMessageRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Message == nil { + m.Message = &raftpb.Message{} + } + if err := m.Message.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StreamRaftMessageResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ResolveAddressRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3574,66 +3961,69 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/raft.proto", fileDescriptorRaft) } var fileDescriptorRaft = []byte{ - // 974 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x96, 0x4f, 0x6f, 0x1b, 0x45, - 0x18, 0xc6, 0x77, 0xd7, 0x5b, 0x27, 0x79, 0xd3, 0x26, 0xd1, 0x94, 0x84, 0xed, 0x52, 0x1c, 0x77, - 0x8b, 0x84, 0x13, 0x9a, 0xb5, 0x30, 0x48, 0x45, 0x85, 0x1e, 0x62, 0xc7, 0x92, 0x4d, 0x5b, 0xa7, - 0xda, 0x24, 0xd0, 0x5b, 0x58, 0xef, 0x4e, 0xdc, 0xc5, 0xf6, 0x8e, 0x99, 0x19, 0x3b, 0x70, 0x41, - 0x3d, 0xa2, 0x5c, 0x39, 0x80, 0x90, 0x7a, 0x82, 0x73, 0x3f, 0x00, 0x1f, 0x00, 0x45, 0x9c, 0xb8, - 0xc1, 0x29, 0xa2, 0xfe, 0x00, 0xf0, 0x15, 0xd0, 0xcc, 0xee, 0x3a, 0xc6, 0x59, 0x3b, 0xb9, 0x24, - 0xa3, 0x9d, 0xdf, 0xf3, 0x3e, 0xef, 0x3b, 0x7f, 0xde, 0x31, 0x6c, 0xb4, 0x02, 0xfe, 0xbc, 0xdf, - 0xb4, 0x3d, 0xd2, 0x2d, 0xfa, 0xc4, 0x6b, 0x63, 0x5a, 0x64, 0xc7, 0x2e, 0xed, 0xb6, 0x03, 0x5e, - 0x74, 0x7b, 0x41, 0x91, 0xba, 0x47, 0xdc, 0xee, 0x51, 0xc2, 0x09, 0x42, 0xd1, 0xbc, 0x9d, 0xcc, - 0xdb, 0x83, 0xf7, 0xcd, 0x7b, 0x97, 0xc8, 0x49, 0xf3, 0x4b, 0xec, 0x71, 0x16, 0x45, 0x30, 0x37, - 0x2f, 0xa1, 0xf9, 0x37, 0x3d, 0x9c, 0xb0, 0x5b, 0x63, 0xac, 0x47, 0x28, 0x26, 0xac, 0x88, 0xb9, - 0xe7, 0xcb, 0x84, 0xe4, 0x9f, 0x5e, 0x73, 0x2c, 0x39, 0xf3, 0x8d, 0x16, 0x69, 0x11, 0x39, 0x2c, - 0x8a, 0x51, 0xfc, 0xf5, 0xfe, 0x0c, 0x43, 0x49, 0x34, 0xfb, 0x47, 0xc5, 0x5e, 0xa7, 0xdf, 0x0a, - 0xc2, 0xf8, 0x5f, 0x24, 0xb4, 0x5e, 0xa9, 0x00, 0x8e, 0x7b, 0xc4, 0x9f, 0xe0, 0x6e, 0x13, 0x53, - 0x74, 0x17, 0xe6, 0x84, 0xd7, 0x61, 0xe0, 0x1b, 0x6a, 0x5e, 0x2d, 0xe8, 0x65, 0x18, 0x9e, 0xad, - 0x67, 0x05, 0x50, 0xdf, 0x71, 0xb2, 0x62, 0xaa, 0xee, 0x0b, 0x28, 0x24, 0x3e, 0x16, 0x90, 0x96, - 0x57, 0x0b, 0x0b, 0x11, 0xd4, 0x20, 0x3e, 0x16, 0x90, 0x98, 0xaa, 0xfb, 0x08, 0x81, 0xee, 0xfa, - 0x3e, 0x35, 0x32, 0x82, 0x70, 0xe4, 0x18, 0x95, 0x21, 0xcb, 0xb8, 0xcb, 0xfb, 0xcc, 0xd0, 0xf3, - 0x6a, 0x61, 0xb1, 0xf4, 0x8e, 0x7d, 0x71, 0xa5, 0xed, 0xf3, 0x6c, 0xf6, 0x24, 0x5b, 0xd6, 0x4f, - 0xcf, 0xd6, 0x15, 0x27, 0x56, 0x5a, 0x77, 0x60, 0xf1, 0x53, 0x12, 0x84, 0x0e, 0xfe, 0xaa, 0x8f, - 0x19, 0x1f, 0xd9, 0xa8, 0xe7, 0x36, 0xd6, 0x4f, 0x2a, 0x5c, 0x8f, 0x18, 0xd6, 0x23, 0x21, 0xc3, - 0x57, 0xab, 0xea, 0x23, 0x98, 0xeb, 0x4a, 0x5b, 0x66, 0x68, 0xf9, 0x4c, 0x61, 0xb1, 0x94, 0x9b, - 0x9d, 0x9d, 0x93, 0xe0, 0xe8, 0x3d, 0x58, 0xa6, 0xb8, 0x4b, 0x06, 0xd8, 0x3f, 0x4c, 0x22, 0x64, - 0xf2, 0x99, 0x82, 0x5e, 0xd6, 0x56, 0x14, 0x67, 0x29, 0x9e, 0x8a, 0x44, 0xcc, 0x2a, 0xc3, 0xf5, - 0xc7, 0xd8, 0x1d, 0xe0, 0xa4, 0x80, 0x12, 0xe8, 0x62, 0xc5, 0x64, 0x62, 0x97, 0x7b, 0x4a, 0xd6, - 0x5a, 0x86, 0x1b, 0x71, 0x8c, 0xa8, 0x40, 0xeb, 0x31, 0xdc, 0x7a, 0x4a, 0x89, 0x87, 0x19, 0x8b, - 0x58, 0xc6, 0xdc, 0xd6, 0xc8, 0x61, 0x43, 0x14, 0x26, 0xbf, 0xc4, 0x26, 0xcb, 0x76, 0x74, 0xac, - 0xec, 0x04, 0x4c, 0xe6, 0x1f, 0xe8, 0x2f, 0x7e, 0xb0, 0x14, 0xeb, 0x36, 0x98, 0x69, 0xd1, 0x62, - 0xaf, 0x4f, 0x60, 0xd5, 0xc1, 0x8c, 0x74, 0x06, 0x78, 0xdb, 0xf7, 0xa9, 0x80, 0x62, 0x9f, 0xab, - 0xac, 0xb2, 0x75, 0x0f, 0xd6, 0x26, 0xd5, 0xf1, 0x26, 0xa5, 0xed, 0x64, 0x07, 0x6e, 0xd6, 0x43, - 0x8e, 0x69, 0xe8, 0x76, 0x44, 0x9c, 0xc4, 0x69, 0x0d, 0xb4, 0x91, 0x49, 0x76, 0x78, 0xb6, 0xae, - 0xd5, 0x77, 0x1c, 0x2d, 0xf0, 0xd1, 0x43, 0xc8, 0xba, 0x1e, 0x0f, 0x48, 0x18, 0xef, 0xe0, 0x7a, - 0xda, 0x6a, 0xee, 0x71, 0x42, 0xf1, 0xb6, 0xc4, 0x92, 0xa3, 0x15, 0x89, 0xac, 0xdf, 0x74, 0x58, - 0x1c, 0x9b, 0x45, 0x1f, 0x8f, 0xc2, 0x09, 0xab, 0xa5, 0xd2, 0xdd, 0x4b, 0xc2, 0x3d, 0x0a, 0x42, - 0x3f, 0x09, 0x86, 0xec, 0x78, 0x5f, 0x35, 0xb9, 0xe4, 0x46, 0x9a, 0x54, 0xdc, 0x98, 0x9a, 0x12, - 0xed, 0x29, 0xba, 0x0f, 0x73, 0x0c, 0xd3, 0x41, 0xe0, 0x61, 0x79, 0x65, 0x16, 0x4b, 0x6f, 0xa5, - 0xba, 0x45, 0x48, 0x4d, 0x71, 0x12, 0x5a, 0x18, 0x71, 0x97, 0xb5, 0xe3, 0x2b, 0x95, 0x6a, 0xb4, - 0xef, 0xb2, 0xb6, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x31, 0x3f, 0x26, 0xb4, 0x6d, 0x5c, 0x9b, 0x6e, - 0xd4, 0x88, 0x10, 0x61, 0x14, 0xd3, 0x42, 0xe8, 0x75, 0xfa, 0x8c, 0x63, 0x6a, 0x64, 0xa7, 0x0b, - 0x2b, 0x11, 0x22, 0x84, 0x31, 0x8d, 0x3e, 0x84, 0x2c, 0xc3, 0x1e, 0xc5, 0xdc, 0x98, 0x93, 0x3a, - 0x33, 0xbd, 0x32, 0x41, 0xd4, 0xc4, 0x45, 0x97, 0x23, 0xf4, 0x00, 0xe6, 0x29, 0x66, 0xa4, 0x4f, - 0x3d, 0x6c, 0xcc, 0x4b, 0xdd, 0xed, 0xd4, 0xcb, 0x11, 0x33, 0x35, 0xc5, 0x19, 0xf1, 0xe8, 0x21, - 0x2c, 0xe0, 0xaf, 0x39, 0x0e, 0x99, 0xd8, 0xbc, 0x05, 0x29, 0x7e, 0x3b, 0x4d, 0x5c, 0x4d, 0xa0, - 0x9a, 0xe2, 0x9c, 0x2b, 0x44, 0xc2, 0x1e, 0x09, 0x8f, 0x82, 0x96, 0x01, 0xd3, 0x13, 0xae, 0x48, - 0x42, 0x24, 0x1c, 0xb1, 0xe5, 0x79, 0xc8, 0x72, 0x97, 0xb6, 0x30, 0xdf, 0xfc, 0x57, 0x85, 0xe5, - 0x89, 0x73, 0x81, 0xde, 0x85, 0xb9, 0x83, 0xc6, 0xa3, 0xc6, 0xee, 0xe7, 0x8d, 0x15, 0xc5, 0x34, - 0x4f, 0x5e, 0xe6, 0xd7, 0x26, 0x88, 0x83, 0xb0, 0x1d, 0x92, 0xe3, 0x10, 0x95, 0xe0, 0xe6, 0xde, - 0xfe, 0xae, 0x53, 0x3d, 0xdc, 0xae, 0xec, 0xd7, 0x77, 0x1b, 0x87, 0x15, 0xa7, 0xba, 0xbd, 0x5f, - 0x5d, 0x51, 0xcd, 0x5b, 0x27, 0x2f, 0xf3, 0xab, 0x13, 0xa2, 0x0a, 0xc5, 0x2e, 0xc7, 0x17, 0x34, - 0x07, 0x4f, 0x77, 0x84, 0x46, 0x4b, 0xd5, 0x1c, 0xf4, 0xfc, 0x34, 0x8d, 0x53, 0x7d, 0xb2, 0xfb, - 0x59, 0x75, 0x25, 0x93, 0xaa, 0x71, 0x64, 0x13, 0x33, 0xdf, 0xfc, 0xee, 0xe7, 0x9c, 0xf2, 0xeb, - 0x2f, 0xb9, 0xc9, 0xea, 0x4a, 0xdf, 0x6b, 0xa0, 0x8b, 0x1b, 0x8a, 0x4e, 0x54, 0x40, 0x17, 0x9b, - 0x07, 0xda, 0x4a, 0x5b, 0xc1, 0xa9, 0x2d, 0xcb, 0xb4, 0xaf, 0x8a, 0xc7, 0x3d, 0x69, 0xf5, 0xf7, - 0x57, 0xff, 0xfc, 0xa8, 0x2d, 0xc3, 0x0d, 0xc9, 0x6f, 0x75, 0xdd, 0xd0, 0x6d, 0x61, 0x8a, 0xbe, - 0x85, 0xa5, 0xff, 0x37, 0x1b, 0xb4, 0x31, 0xed, 0x08, 0x5d, 0x68, 0x67, 0xe6, 0xe6, 0x55, 0xd0, - 0x99, 0xfe, 0xa5, 0x3f, 0x55, 0x58, 0x3a, 0x6f, 0xde, 0xec, 0x79, 0xd0, 0x43, 0x5f, 0x80, 0x2e, - 0x9e, 0x26, 0x94, 0xda, 0x9a, 0xc6, 0x1e, 0x36, 0x33, 0x3f, 0x1d, 0x98, 0x5d, 0xb4, 0x07, 0xd7, - 0xe4, 0xe3, 0x80, 0x52, 0x23, 0x8c, 0xbf, 0x3d, 0xe6, 0x9d, 0x19, 0xc4, 0x4c, 0x93, 0xb2, 0x71, - 0xfa, 0x3a, 0xa7, 0xfc, 0xf5, 0x3a, 0xa7, 0xbc, 0x18, 0xe6, 0xd4, 0xd3, 0x61, 0x4e, 0xfd, 0x63, - 0x98, 0x53, 0xff, 0x1e, 0xe6, 0xd4, 0x67, 0x99, 0x67, 0x7a, 0x33, 0x2b, 0x7f, 0x5b, 0x7c, 0xf0, - 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x01, 0xa3, 0x4f, 0x74, 0x09, 0x00, 0x00, + // 1015 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xc1, 0x6e, 0x1b, 0x45, + 0x18, 0xc7, 0x77, 0xed, 0xad, 0xd3, 0x7c, 0x69, 0x93, 0x30, 0x25, 0x61, 0xb3, 0x2d, 0x8e, 0xbb, + 0x45, 0xc2, 0x09, 0xc9, 0x5a, 0x18, 0xa4, 0xa2, 0x42, 0x0f, 0x71, 0x62, 0x29, 0x26, 0xad, 0x53, + 0x6d, 0x12, 0xe8, 0x2d, 0xac, 0x77, 0x27, 0xee, 0x62, 0x7b, 0xc7, 0xcc, 0x8c, 0x1d, 0xb8, 0xa0, + 0x1e, 0x21, 0x2f, 0x00, 0x42, 0xaa, 0x38, 0xc0, 0xb9, 0x0f, 0xc0, 0x03, 0xa0, 0x88, 0x13, 0x37, + 0x38, 0x45, 0xd4, 0x0f, 0x00, 0xaf, 0x80, 0x66, 0x76, 0xd7, 0x31, 0xf6, 0xda, 0xf1, 0x81, 0x4b, + 0x32, 0xda, 0xf9, 0xfd, 0xbf, 0xff, 0x37, 0x33, 0xdf, 0x7c, 0x63, 0x58, 0xab, 0xfb, 0xfc, 0x59, + 0xa7, 0x66, 0xb9, 0xa4, 0x55, 0xf0, 0x88, 0xdb, 0xc0, 0xb4, 0xc0, 0x4e, 0x1d, 0xda, 0x6a, 0xf8, + 0xbc, 0xe0, 0xb4, 0xfd, 0x02, 0x75, 0x4e, 0xb8, 0xd5, 0xa6, 0x84, 0x13, 0x84, 0xc2, 0x79, 0x2b, + 0x9e, 0xb7, 0xba, 0xef, 0x1a, 0x1b, 0x57, 0xc8, 0x49, 0xed, 0x73, 0xec, 0x72, 0x16, 0x46, 0x30, + 0xd6, 0xaf, 0xa0, 0xf9, 0x57, 0x6d, 0x1c, 0xb3, 0x9b, 0x03, 0xac, 0x4b, 0x28, 0x26, 0xac, 0x80, + 0xb9, 0xeb, 0xc9, 0x84, 0xe4, 0x9f, 0x76, 0x6d, 0x20, 0x39, 0xe3, 0xf5, 0x3a, 0xa9, 0x13, 0x39, + 0x2c, 0x88, 0x51, 0xf4, 0xf5, 0xfe, 0x04, 0x43, 0x49, 0xd4, 0x3a, 0x27, 0x85, 0x76, 0xb3, 0x53, + 0xf7, 0x83, 0xe8, 0x5f, 0x28, 0x34, 0x5f, 0xaa, 0x00, 0xb6, 0x73, 0xc2, 0x1f, 0xe3, 0x56, 0x0d, + 0x53, 0x74, 0x0f, 0x66, 0x84, 0xd7, 0xb1, 0xef, 0xe9, 0x6a, 0x4e, 0xcd, 0x6b, 0x25, 0xe8, 0x5d, + 0xac, 0x66, 0x04, 0x50, 0xd9, 0xb1, 0x33, 0x62, 0xaa, 0xe2, 0x09, 0x28, 0x20, 0x1e, 0x16, 0x50, + 0x2a, 0xa7, 0xe6, 0x67, 0x43, 0xa8, 0x4a, 0x3c, 0x2c, 0x20, 0x31, 0x55, 0xf1, 0x10, 0x02, 0xcd, + 0xf1, 0x3c, 0xaa, 0xa7, 0x05, 0x61, 0xcb, 0x31, 0x2a, 0x41, 0x86, 0x71, 0x87, 0x77, 0x98, 0xae, + 0xe5, 0xd4, 0xfc, 0x5c, 0xf1, 0x2d, 0x6b, 0x74, 0xa7, 0xad, 0xcb, 0x6c, 0x0e, 0x24, 0x5b, 0xd2, + 0xce, 0x2f, 0x56, 0x15, 0x3b, 0x52, 0x9a, 0x77, 0x61, 0xee, 0x63, 0xe2, 0x07, 0x36, 0xfe, 0xa2, + 0x83, 0x19, 0xef, 0xdb, 0xa8, 0x97, 0x36, 0xe6, 0x0f, 0x2a, 0xdc, 0x08, 0x19, 0xd6, 0x26, 0x01, + 0xc3, 0xd3, 0xad, 0xea, 0x03, 0x98, 0x69, 0x49, 0x5b, 0xa6, 0xa7, 0x72, 0xe9, 0xfc, 0x5c, 0x31, + 0x3b, 0x39, 0x3b, 0x3b, 0xc6, 0xd1, 0x3b, 0xb0, 0x40, 0x71, 0x8b, 0x74, 0xb1, 0x77, 0x1c, 0x47, + 0x48, 0xe7, 0xd2, 0x79, 0xad, 0x94, 0x5a, 0x54, 0xec, 0xf9, 0x68, 0x2a, 0x14, 0x31, 0xb3, 0x04, + 0x37, 0x1e, 0x61, 0xa7, 0x8b, 0xe3, 0x05, 0x14, 0x41, 0x13, 0x3b, 0x26, 0x13, 0xbb, 0xda, 0x53, + 0xb2, 0xe6, 0x02, 0xdc, 0x8c, 0x62, 0x84, 0x0b, 0x34, 0x1f, 0xc1, 0xca, 0x13, 0x4a, 0x5c, 0xcc, + 0x58, 0xc8, 0x32, 0xe6, 0xd4, 0xfb, 0x0e, 0x6b, 0x62, 0x61, 0xf2, 0x4b, 0x64, 0xb2, 0x60, 0x85, + 0x65, 0x65, 0xc5, 0x60, 0x3c, 0xff, 0x40, 0x7b, 0xfe, 0x9d, 0xa9, 0x98, 0x77, 0xc0, 0x48, 0x8a, + 0x16, 0x79, 0xed, 0x81, 0x7e, 0xc0, 0x29, 0x76, 0x5a, 0xff, 0x87, 0xd5, 0x6d, 0x58, 0x49, 0x08, + 0x16, 0x39, 0x7d, 0x04, 0x4b, 0x36, 0x66, 0xa4, 0xd9, 0xc5, 0x5b, 0x9e, 0x47, 0x45, 0x3a, 0x91, + 0xcd, 0x34, 0xe7, 0x69, 0x6e, 0xc0, 0xf2, 0xb0, 0x3a, 0x2a, 0x87, 0xa4, 0x9a, 0x69, 0xc2, 0xad, + 0x4a, 0xc0, 0x31, 0x0d, 0x9c, 0xa6, 0x88, 0x13, 0x3b, 0x2d, 0x43, 0xaa, 0x6f, 0x92, 0xe9, 0x5d, + 0xac, 0xa6, 0x2a, 0x3b, 0x76, 0xca, 0xf7, 0xd0, 0x43, 0xc8, 0x38, 0x2e, 0xf7, 0x49, 0x10, 0xd5, + 0xca, 0x6a, 0xd2, 0xb9, 0x1d, 0x70, 0x42, 0xf1, 0x96, 0xc4, 0xe2, 0x22, 0x0e, 0x45, 0xe6, 0xaf, + 0x1a, 0xcc, 0x0d, 0xcc, 0xa2, 0x0f, 0xfb, 0xe1, 0x84, 0xd5, 0x7c, 0xf1, 0xde, 0x15, 0xe1, 0xf6, + 0xfc, 0xc0, 0x8b, 0x83, 0x21, 0x2b, 0xaa, 0xa0, 0x94, 0xdc, 0x71, 0x3d, 0x49, 0x2a, 0xee, 0xe6, + 0xae, 0x12, 0x56, 0x0f, 0xba, 0x0f, 0x33, 0x0c, 0xd3, 0xae, 0xef, 0x62, 0x79, 0x39, 0xe7, 0x8a, + 0xb7, 0x13, 0xdd, 0x42, 0x64, 0x57, 0xb1, 0x63, 0x5a, 0x18, 0x71, 0x87, 0x35, 0xa2, 0xcb, 0x9b, + 0x68, 0x74, 0xe8, 0xb0, 0x86, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x30, 0x3f, 0x25, 0xb4, 0xa1, 0x5f, + 0x1b, 0x6f, 0x54, 0x0d, 0x11, 0x61, 0x14, 0xd1, 0x42, 0xe8, 0x36, 0x3b, 0x8c, 0x63, 0xaa, 0x67, + 0xc6, 0x0b, 0xb7, 0x43, 0x44, 0x08, 0x23, 0x1a, 0xbd, 0x0f, 0x19, 0x86, 0x5d, 0x8a, 0xb9, 0x3e, + 0x23, 0x75, 0x46, 0xf2, 0xca, 0x04, 0xb1, 0x2b, 0x5a, 0x8a, 0x1c, 0xa1, 0x07, 0x70, 0x9d, 0x62, + 0x46, 0x3a, 0xd4, 0xc5, 0xfa, 0x75, 0xa9, 0xbb, 0x93, 0x78, 0x0d, 0x23, 0x66, 0x57, 0xb1, 0xfb, + 0x3c, 0x7a, 0x08, 0xb3, 0xf8, 0x4b, 0x8e, 0x03, 0x26, 0x0e, 0x6f, 0x56, 0x8a, 0xdf, 0x4c, 0x12, + 0x97, 0x63, 0x68, 0x57, 0xb1, 0x2f, 0x15, 0x22, 0x61, 0x97, 0x04, 0x27, 0x7e, 0x5d, 0x87, 0xf1, + 0x09, 0x6f, 0x4b, 0x42, 0x24, 0x1c, 0xb2, 0xa5, 0xeb, 0x90, 0xe1, 0x0e, 0xad, 0x63, 0xbe, 0xfe, + 0x8f, 0x0a, 0x0b, 0x43, 0x75, 0x81, 0xde, 0x86, 0x99, 0xa3, 0xea, 0x5e, 0x75, 0xff, 0xd3, 0xea, + 0xa2, 0x62, 0x18, 0x67, 0x2f, 0x72, 0xcb, 0x43, 0xc4, 0x51, 0xd0, 0x08, 0xc8, 0x69, 0x80, 0x8a, + 0x70, 0xeb, 0xe0, 0x70, 0xdf, 0x2e, 0x1f, 0x6f, 0x6d, 0x1f, 0x56, 0xf6, 0xab, 0xc7, 0xdb, 0x76, + 0x79, 0xeb, 0xb0, 0xbc, 0xa8, 0x1a, 0x2b, 0x67, 0x2f, 0x72, 0x4b, 0x43, 0xa2, 0x6d, 0x8a, 0x1d, + 0x8e, 0x47, 0x34, 0x47, 0x4f, 0x76, 0x84, 0x26, 0x95, 0xa8, 0x39, 0x6a, 0x7b, 0x49, 0x1a, 0xbb, + 0xfc, 0x78, 0xff, 0x93, 0xf2, 0x62, 0x3a, 0x51, 0x63, 0xcb, 0x76, 0x69, 0xbc, 0xf1, 0xcd, 0x4f, + 0x59, 0xe5, 0x97, 0x9f, 0xb3, 0xc3, 0xab, 0x2b, 0xfe, 0x98, 0x06, 0x4d, 0xdc, 0x50, 0x74, 0xa6, + 0x02, 0x1a, 0x6d, 0x53, 0x68, 0x33, 0x69, 0x07, 0xc7, 0x36, 0x47, 0xc3, 0x9a, 0x16, 0x8f, 0x7a, + 0xd2, 0xd2, 0x6f, 0x2f, 0xff, 0xfe, 0x3e, 0xb5, 0x00, 0x37, 0x25, 0xbf, 0xd9, 0x72, 0x02, 0xa7, + 0x8e, 0x29, 0xfa, 0x56, 0x85, 0xd7, 0x46, 0x1a, 0x19, 0xda, 0x48, 0xbe, 0xc6, 0xc9, 0xcd, 0xd3, + 0xd8, 0x9c, 0x92, 0x9e, 0x98, 0x49, 0x5e, 0x45, 0x5f, 0xc3, 0xfc, 0x7f, 0x1b, 0x1f, 0x5a, 0x1b, + 0x57, 0xce, 0x23, 0xad, 0xd5, 0x58, 0x9f, 0x06, 0x9d, 0x98, 0x41, 0xf1, 0x0f, 0x15, 0xe6, 0x2f, + 0x9f, 0x2c, 0xf6, 0xcc, 0x6f, 0xa3, 0xcf, 0x40, 0x13, 0x0f, 0x32, 0x4a, 0x6c, 0x93, 0x03, 0xcf, + 0xb9, 0x91, 0x1b, 0x0f, 0x4c, 0x3e, 0x00, 0x17, 0xae, 0xc9, 0x27, 0x11, 0x25, 0x46, 0x18, 0x7c, + 0x71, 0x8d, 0xbb, 0x13, 0x88, 0x89, 0x26, 0x25, 0xfd, 0xfc, 0x55, 0x56, 0xf9, 0xf3, 0x55, 0x56, + 0x79, 0xde, 0xcb, 0xaa, 0xe7, 0xbd, 0xac, 0xfa, 0x7b, 0x2f, 0xab, 0xfe, 0xd5, 0xcb, 0xaa, 0x4f, + 0xd3, 0x4f, 0xb5, 0x5a, 0x46, 0xfe, 0xa2, 0x7a, 0xef, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3e, + 0x7a, 0x8b, 0xe7, 0x6a, 0x0a, 0x00, 0x00, } diff --git a/vendor/github.com/docker/swarmkit/api/raft.proto b/vendor/github.com/docker/swarmkit/api/raft.proto index b39831584d..b351c15bde 100644 --- a/vendor/github.com/docker/swarmkit/api/raft.proto +++ b/vendor/github.com/docker/swarmkit/api/raft.proto @@ -16,6 +16,15 @@ service Raft { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; }; + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + rpc StreamRaftMessage(stream StreamRaftMessageRequest) returns (StreamRaftMessageResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; + }; + // ResolveAddress returns the address where the node with the given ID can be reached. rpc ResolveAddress(ResolveAddressRequest) returns (ResolveAddressResponse) { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; @@ -82,6 +91,15 @@ message ProcessRaftMessageRequest { message ProcessRaftMessageResponse {} +// Raft message streaming request. +message StreamRaftMessageRequest { + option (docker.protobuf.plugin.deepcopy) = false; + raftpb.Message message = 1; +} + +// Raft message streaming response. +message StreamRaftMessageResponse {} + message ResolveAddressRequest { // raft_id is the ID to resolve to an address. uint64 raft_id = 1; diff --git a/vendor/github.com/docker/swarmkit/api/resource.pb.go b/vendor/github.com/docker/swarmkit/api/resource.pb.go index ead5d27ef4..d4e27f3362 100644 --- a/vendor/github.com/docker/swarmkit/api/resource.pb.go +++ b/vendor/github.com/docker/swarmkit/api/resource.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -403,12 +404,12 @@ func NewRaftProxyResourceAllocatorServer(local ResourceAllocatorServer, connSele redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/api/types.pb.go b/vendor/github.com/docker/swarmkit/api/types.pb.go index 33e2281b67..67243dc88e 100644 --- a/vendor/github.com/docker/swarmkit/api/types.pb.go +++ b/vendor/github.com/docker/swarmkit/api/types.pb.go @@ -72,7 +72,16 @@ const ( TaskStateShutdown TaskState = 640 TaskStateFailed TaskState = 704 TaskStateRejected TaskState = 768 - TaskStateOrphaned TaskState = 832 + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + TaskStateRemove TaskState = 800 + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + TaskStateOrphaned TaskState = 832 ) var TaskState_name = map[int32]string{ @@ -88,6 +97,7 @@ var TaskState_name = map[int32]string{ 640: "SHUTDOWN", 704: "FAILED", 768: "REJECTED", + 800: "REMOVE", 832: "ORPHANED", } var TaskState_value = map[string]int32{ @@ -103,6 +113,7 @@ var TaskState_value = map[string]int32{ "SHUTDOWN": 640, "FAILED": 704, "REJECTED": 768, + "REMOVE": 800, "ORPHANED": 832, } @@ -17029,319 +17040,321 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 5020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4d, 0x6c, 0x24, 0x49, - 0x56, 0x76, 0xfd, 0xba, 0xea, 0x55, 0xd9, 0x4e, 0x47, 0x7b, 0x7b, 0xdc, 0xb5, 0xdd, 0x76, 0x4d, - 0xce, 0xf4, 0xce, 0x6c, 0x6f, 0x53, 0xfd, 0xb7, 0xbb, 0xea, 0x99, 0x61, 0x77, 0xa6, 0xfe, 0x6c, - 0xd7, 0xb6, 0x5d, 0x55, 0x8a, 0x2a, 0x77, 0xef, 0x22, 0x41, 0x2a, 0x9d, 0x19, 0x2e, 0xe7, 0x38, - 0x2b, 0xa3, 0xc8, 0xcc, 0xb2, 0xbb, 0x58, 0x10, 0x2d, 0x0e, 0x80, 0x7c, 0x82, 0xdb, 0x22, 0x64, - 0x2e, 0x70, 0x42, 0x48, 0x1c, 0x40, 0x42, 0x70, 0x1a, 0x24, 0x0e, 0x7b, 0x83, 0x05, 0x09, 0xad, - 0x40, 0x32, 0xac, 0x0f, 0xdc, 0x56, 0x70, 0x59, 0x71, 0x01, 0x09, 0xc5, 0x4f, 0x66, 0xa5, 0xab, - 0xd3, 0x76, 0x0f, 0xb3, 0x17, 0xbb, 0xe2, 0xbd, 0xef, 0xbd, 0x78, 0xf1, 0x22, 0xe2, 0xc5, 0x7b, - 0x11, 0x09, 0xf7, 0x06, 0x96, 0x7f, 0x30, 0xde, 0xab, 0x18, 0x74, 0xf8, 0xc0, 0xa4, 0xc6, 0x21, - 0x71, 0x1f, 0x78, 0xc7, 0xba, 0x3b, 0x3c, 0xb4, 0xfc, 0x07, 0xfa, 0xc8, 0x7a, 0xe0, 0x4f, 0x46, - 0xc4, 0xab, 0x8c, 0x5c, 0xea, 0x53, 0x84, 0x04, 0xa0, 0x12, 0x00, 0x2a, 0x47, 0x8f, 0x4a, 0xeb, - 0x03, 0x4a, 0x07, 0x36, 0x79, 0xc0, 0x11, 0x7b, 0xe3, 0xfd, 0x07, 0xbe, 0x35, 0x24, 0x9e, 0xaf, - 0x0f, 0x47, 0x42, 0xa8, 0xb4, 0x36, 0x0b, 0x30, 0xc7, 0xae, 0xee, 0x5b, 0xd4, 0x91, 0xfc, 0x95, - 0x01, 0x1d, 0x50, 0xfe, 0xf3, 0x01, 0xfb, 0x25, 0xa8, 0xea, 0x3a, 0xcc, 0x3f, 0x27, 0xae, 0x67, - 0x51, 0x07, 0xad, 0x40, 0xc6, 0x72, 0x4c, 0xf2, 0x72, 0x35, 0x51, 0x4e, 0xbc, 0x9f, 0xc6, 0xa2, - 0xa1, 0x3e, 0x04, 0x68, 0xb1, 0x1f, 0x4d, 0xc7, 0x77, 0x27, 0x48, 0x81, 0xd4, 0x21, 0x99, 0x70, - 0x44, 0x1e, 0xb3, 0x9f, 0x8c, 0x72, 0xa4, 0xdb, 0xab, 0x49, 0x41, 0x39, 0xd2, 0x6d, 0xf5, 0x27, - 0x09, 0x28, 0x54, 0x1d, 0x87, 0xfa, 0xbc, 0x77, 0x0f, 0x21, 0x48, 0x3b, 0xfa, 0x90, 0x48, 0x21, - 0xfe, 0x1b, 0xd5, 0x21, 0x6b, 0xeb, 0x7b, 0xc4, 0xf6, 0x56, 0x93, 0xe5, 0xd4, 0xfb, 0x85, 0xc7, - 0x5f, 0xab, 0xbc, 0x3e, 0xe4, 0x4a, 0x44, 0x49, 0x65, 0x9b, 0xa3, 0xb9, 0x11, 0x58, 0x8a, 0xa2, - 0x6f, 0xc3, 0xbc, 0xe5, 0x98, 0x96, 0x41, 0xbc, 0xd5, 0x34, 0xd7, 0xb2, 0x16, 0xa7, 0x65, 0x6a, - 0x7d, 0x2d, 0xfd, 0xc3, 0xb3, 0xf5, 0x39, 0x1c, 0x08, 0x95, 0x3e, 0x80, 0x42, 0x44, 0x6d, 0xcc, - 0xd8, 0x56, 0x20, 0x73, 0xa4, 0xdb, 0x63, 0x22, 0x47, 0x27, 0x1a, 0x1f, 0x26, 0x9f, 0x26, 0xd4, - 0x4f, 0x60, 0xa5, 0xad, 0x0f, 0x89, 0xb9, 0x49, 0x1c, 0xe2, 0x5a, 0x06, 0x26, 0x1e, 0x1d, 0xbb, - 0x06, 0x61, 0x63, 0x3d, 0xb4, 0x1c, 0x33, 0x18, 0x2b, 0xfb, 0x1d, 0xaf, 0x45, 0xad, 0xc3, 0x5b, - 0x0d, 0xcb, 0x33, 0x5c, 0xe2, 0x93, 0xcf, 0xad, 0x24, 0x15, 0x28, 0x39, 0x4b, 0xc0, 0xd2, 0xac, - 0xf4, 0x2f, 0xc1, 0x0d, 0xe6, 0x62, 0x53, 0x73, 0x25, 0x45, 0xf3, 0x46, 0xc4, 0xe0, 0xca, 0x0a, - 0x8f, 0xdf, 0x8f, 0xf3, 0x50, 0xdc, 0x48, 0xb6, 0xe6, 0xf0, 0x32, 0x57, 0x13, 0x10, 0x7a, 0x23, - 0x62, 0x20, 0x03, 0x6e, 0x9a, 0xd2, 0xe8, 0x19, 0xf5, 0x49, 0xae, 0x3e, 0x76, 0x1a, 0x2f, 0x19, - 0xe6, 0xd6, 0x1c, 0x5e, 0x09, 0x94, 0x45, 0x3b, 0xa9, 0x01, 0xe4, 0x02, 0xdd, 0xea, 0x0f, 0x12, - 0x90, 0x0f, 0x98, 0x1e, 0xfa, 0x2a, 0xe4, 0x1d, 0xdd, 0xa1, 0x9a, 0x31, 0x1a, 0x7b, 0x7c, 0x40, - 0xa9, 0x5a, 0xf1, 0xfc, 0x6c, 0x3d, 0xd7, 0xd6, 0x1d, 0x5a, 0xef, 0xee, 0x7a, 0x38, 0xc7, 0xd8, - 0xf5, 0xd1, 0xd8, 0x43, 0x6f, 0x43, 0x71, 0x48, 0x86, 0xd4, 0x9d, 0x68, 0x7b, 0x13, 0x9f, 0x78, - 0xd2, 0x6d, 0x05, 0x41, 0xab, 0x31, 0x12, 0xfa, 0x16, 0xcc, 0x0f, 0x84, 0x49, 0xab, 0x29, 0xbe, - 0x7c, 0xde, 0x89, 0xb3, 0x7e, 0xc6, 0x6a, 0x1c, 0xc8, 0xa8, 0xbf, 0x97, 0x80, 0x95, 0x90, 0x4a, - 0x7e, 0x75, 0x6c, 0xb9, 0x64, 0x48, 0x1c, 0xdf, 0x43, 0xdf, 0x80, 0xac, 0x6d, 0x0d, 0x2d, 0xdf, - 0x93, 0x3e, 0xbf, 0x13, 0xa7, 0x36, 0x1c, 0x14, 0x96, 0x60, 0x54, 0x85, 0xa2, 0x4b, 0x3c, 0xe2, - 0x1e, 0x89, 0x15, 0x2f, 0x3d, 0x7a, 0x8d, 0xf0, 0x05, 0x11, 0x75, 0x03, 0x72, 0x5d, 0x5b, 0xf7, - 0xf7, 0xa9, 0x3b, 0x44, 0x2a, 0x14, 0x75, 0xd7, 0x38, 0xb0, 0x7c, 0x62, 0xf8, 0x63, 0x37, 0xd8, - 0x7d, 0x17, 0x68, 0xe8, 0x26, 0x24, 0xa9, 0xe8, 0x28, 0x5f, 0xcb, 0x9e, 0x9f, 0xad, 0x27, 0x3b, - 0x3d, 0x9c, 0xa4, 0x9e, 0xfa, 0x11, 0x2c, 0x77, 0xed, 0xf1, 0xc0, 0x72, 0x1a, 0xc4, 0x33, 0x5c, - 0x6b, 0xc4, 0xb4, 0xb3, 0x55, 0xc9, 0x62, 0x54, 0xb0, 0x2a, 0xd9, 0xef, 0x70, 0x6b, 0x27, 0xa7, - 0x5b, 0x5b, 0xfd, 0x9d, 0x24, 0x2c, 0x37, 0x9d, 0x81, 0xe5, 0x90, 0xa8, 0xf4, 0x5d, 0x58, 0x24, - 0x9c, 0xa8, 0x1d, 0x89, 0x70, 0x23, 0xf5, 0x2c, 0x08, 0x6a, 0x10, 0x83, 0x5a, 0x33, 0x71, 0xe1, - 0x51, 0xdc, 0xf0, 0x5f, 0xd3, 0x1e, 0x1b, 0x1d, 0x9a, 0x30, 0x3f, 0xe2, 0x83, 0xf0, 0xe4, 0xf4, - 0xde, 0x8d, 0xd3, 0xf5, 0xda, 0x38, 0x83, 0x20, 0x21, 0x65, 0xbf, 0x48, 0x90, 0xf8, 0xb3, 0x24, - 0x2c, 0xb5, 0xa9, 0x79, 0xc1, 0x0f, 0x25, 0xc8, 0x1d, 0x50, 0xcf, 0x8f, 0x04, 0xc4, 0xb0, 0x8d, - 0x9e, 0x42, 0x6e, 0x24, 0xa7, 0x4f, 0xce, 0xfe, 0xed, 0x78, 0x93, 0x05, 0x06, 0x87, 0x68, 0xf4, - 0x11, 0xe4, 0x83, 0x2d, 0xc3, 0x46, 0xfb, 0x06, 0x0b, 0x67, 0x8a, 0x47, 0xdf, 0x82, 0xac, 0x98, - 0x84, 0xd5, 0x34, 0x97, 0xbc, 0xfb, 0x46, 0x3e, 0xc7, 0x52, 0x08, 0x6d, 0x42, 0xce, 0xb7, 0x3d, - 0xcd, 0x72, 0xf6, 0xe9, 0x6a, 0x86, 0x2b, 0x58, 0x8f, 0x0d, 0x32, 0xd4, 0x24, 0xfd, 0xed, 0x5e, - 0xcb, 0xd9, 0xa7, 0xb5, 0xc2, 0xf9, 0xd9, 0xfa, 0xbc, 0x6c, 0xe0, 0x79, 0xdf, 0xf6, 0xd8, 0x0f, - 0xf5, 0xf7, 0x13, 0x50, 0x88, 0xa0, 0xd0, 0x1d, 0x00, 0xdf, 0x1d, 0x7b, 0xbe, 0xe6, 0x52, 0xea, - 0x73, 0x67, 0x15, 0x71, 0x9e, 0x53, 0x30, 0xa5, 0x3e, 0xaa, 0xc0, 0x0d, 0x83, 0xb8, 0xbe, 0x66, - 0x79, 0xde, 0x98, 0xb8, 0x9a, 0x37, 0xde, 0xfb, 0x94, 0x18, 0x3e, 0x77, 0x5c, 0x11, 0x2f, 0x33, - 0x56, 0x8b, 0x73, 0x7a, 0x82, 0x81, 0x9e, 0xc0, 0xcd, 0x28, 0x7e, 0x34, 0xde, 0xb3, 0x2d, 0x43, - 0x63, 0x93, 0x99, 0xe2, 0x22, 0x37, 0xa6, 0x22, 0x5d, 0xce, 0x7b, 0x46, 0x26, 0xea, 0x8f, 0x13, - 0xa0, 0x60, 0x7d, 0xdf, 0xdf, 0x21, 0xc3, 0x3d, 0xe2, 0xf6, 0x7c, 0xdd, 0x1f, 0x7b, 0xe8, 0x26, - 0x64, 0x6d, 0xa2, 0x9b, 0xc4, 0xe5, 0x46, 0xe5, 0xb0, 0x6c, 0xa1, 0x5d, 0xb6, 0x83, 0x75, 0xe3, - 0x40, 0xdf, 0xb3, 0x6c, 0xcb, 0x9f, 0x70, 0x53, 0x16, 0xe3, 0x97, 0xf0, 0xac, 0xce, 0x0a, 0x8e, - 0x08, 0xe2, 0x0b, 0x6a, 0xd0, 0x2a, 0xcc, 0x0f, 0x89, 0xe7, 0xe9, 0x03, 0xc2, 0x2d, 0xcd, 0xe3, - 0xa0, 0xa9, 0x7e, 0x04, 0xc5, 0xa8, 0x1c, 0x2a, 0xc0, 0xfc, 0x6e, 0xfb, 0x59, 0xbb, 0xf3, 0xa2, - 0xad, 0xcc, 0xa1, 0x25, 0x28, 0xec, 0xb6, 0x71, 0xb3, 0x5a, 0xdf, 0xaa, 0xd6, 0xb6, 0x9b, 0x4a, - 0x02, 0x2d, 0x40, 0x7e, 0xda, 0x4c, 0xaa, 0x7f, 0x91, 0x00, 0x60, 0xee, 0x96, 0x83, 0xfa, 0x10, - 0x32, 0x9e, 0xaf, 0xfb, 0x62, 0x55, 0x2e, 0x3e, 0x7e, 0xf7, 0xb2, 0x39, 0x94, 0xf6, 0xb2, 0x7f, - 0x04, 0x0b, 0x91, 0xa8, 0x85, 0xc9, 0x0b, 0x16, 0xb2, 0x00, 0xa1, 0x9b, 0xa6, 0x2b, 0x0d, 0xe7, - 0xbf, 0xd5, 0x8f, 0x20, 0xc3, 0xa5, 0x2f, 0x9a, 0x9b, 0x83, 0x74, 0x83, 0xfd, 0x4a, 0xa0, 0x3c, - 0x64, 0x70, 0xb3, 0xda, 0xf8, 0x9e, 0x92, 0x44, 0x0a, 0x14, 0x1b, 0xad, 0x5e, 0xbd, 0xd3, 0x6e, - 0x37, 0xeb, 0xfd, 0x66, 0x43, 0x49, 0xa9, 0x77, 0x21, 0xd3, 0x1a, 0x32, 0xcd, 0xb7, 0xd9, 0x92, - 0xdf, 0x27, 0x2e, 0x71, 0x8c, 0x60, 0x27, 0x4d, 0x09, 0xea, 0x4f, 0x0b, 0x90, 0xd9, 0xa1, 0x63, - 0xc7, 0x47, 0x8f, 0x23, 0x61, 0x6b, 0x31, 0x3e, 0x43, 0xe0, 0xc0, 0x4a, 0x7f, 0x32, 0x22, 0x32, - 0xac, 0xdd, 0x84, 0xac, 0xd8, 0x1c, 0x72, 0x38, 0xb2, 0xc5, 0xe8, 0xbe, 0xee, 0x0e, 0x88, 0x2f, - 0xc7, 0x23, 0x5b, 0xe8, 0x7d, 0x76, 0x62, 0xe9, 0x26, 0x75, 0xec, 0x09, 0xdf, 0x43, 0x39, 0x71, - 0x2c, 0x61, 0xa2, 0x9b, 0x1d, 0xc7, 0x9e, 0xe0, 0x90, 0x8b, 0xb6, 0xa0, 0xb8, 0x67, 0x39, 0xa6, - 0x46, 0x47, 0x22, 0xc8, 0x67, 0x2e, 0xdf, 0x71, 0xc2, 0xaa, 0x9a, 0xe5, 0x98, 0x1d, 0x01, 0xc6, - 0x85, 0xbd, 0x69, 0x03, 0xb5, 0x61, 0xf1, 0x88, 0xda, 0xe3, 0x21, 0x09, 0x75, 0x65, 0xb9, 0xae, - 0xf7, 0x2e, 0xd7, 0xf5, 0x9c, 0xe3, 0x03, 0x6d, 0x0b, 0x47, 0xd1, 0x26, 0x7a, 0x06, 0x0b, 0xfe, - 0x70, 0xb4, 0xef, 0x85, 0xea, 0xe6, 0xb9, 0xba, 0xaf, 0x5c, 0xe1, 0x30, 0x06, 0x0f, 0xb4, 0x15, - 0xfd, 0x48, 0x0b, 0x6d, 0x42, 0xc1, 0xa0, 0x8e, 0x67, 0x79, 0x3e, 0x71, 0x8c, 0xc9, 0x6a, 0x8e, - 0xfb, 0xfe, 0x8a, 0x51, 0xd6, 0xa7, 0x60, 0x1c, 0x95, 0x2c, 0xfd, 0x56, 0x0a, 0x0a, 0x11, 0x17, - 0xa0, 0x1e, 0x14, 0x46, 0x2e, 0x1d, 0xe9, 0x03, 0x7e, 0xe2, 0xc9, 0x49, 0x7d, 0xf4, 0x46, 0xee, - 0xab, 0x74, 0xa7, 0x82, 0x38, 0xaa, 0x45, 0x3d, 0x4d, 0x42, 0x21, 0xc2, 0x44, 0xf7, 0x20, 0x87, - 0xbb, 0xb8, 0xf5, 0xbc, 0xda, 0x6f, 0x2a, 0x73, 0xa5, 0xdb, 0x27, 0xa7, 0xe5, 0x55, 0xae, 0x2d, - 0xaa, 0xa0, 0xeb, 0x5a, 0x47, 0x6c, 0x0d, 0xbf, 0x0f, 0xf3, 0x01, 0x34, 0x51, 0xfa, 0xf2, 0xc9, - 0x69, 0xf9, 0xad, 0x59, 0x68, 0x04, 0x89, 0x7b, 0x5b, 0x55, 0xdc, 0x6c, 0x28, 0xc9, 0x78, 0x24, - 0xee, 0x1d, 0xe8, 0x2e, 0x31, 0xd1, 0x57, 0x20, 0x2b, 0x81, 0xa9, 0x52, 0xe9, 0xe4, 0xb4, 0x7c, - 0x73, 0x16, 0x38, 0xc5, 0xe1, 0xde, 0x76, 0xf5, 0x79, 0x53, 0x49, 0xc7, 0xe3, 0x70, 0xcf, 0xd6, - 0x8f, 0x08, 0x7a, 0x17, 0x32, 0x02, 0x96, 0x29, 0xdd, 0x3a, 0x39, 0x2d, 0x7f, 0xe9, 0x35, 0x75, - 0x0c, 0x55, 0x5a, 0xfd, 0xdd, 0x3f, 0x5e, 0x9b, 0xfb, 0x9b, 0x3f, 0x59, 0x53, 0x66, 0xd9, 0xa5, - 0xff, 0x49, 0xc0, 0xc2, 0x85, 0xb5, 0x83, 0x54, 0xc8, 0x3a, 0xd4, 0xa0, 0x23, 0x71, 0x10, 0xe6, - 0x6a, 0x70, 0x7e, 0xb6, 0x9e, 0x6d, 0xd3, 0x3a, 0x1d, 0x4d, 0xb0, 0xe4, 0xa0, 0x67, 0x33, 0x47, - 0xf9, 0x93, 0x37, 0x5c, 0x98, 0xb1, 0x87, 0xf9, 0xc7, 0xb0, 0x60, 0xba, 0xd6, 0x11, 0x71, 0x35, - 0x83, 0x3a, 0xfb, 0xd6, 0x40, 0x1e, 0x72, 0xa5, 0xd8, 0x7c, 0x93, 0x03, 0x71, 0x51, 0x08, 0xd4, - 0x39, 0xfe, 0x0b, 0x1c, 0xe3, 0xa5, 0xe7, 0x50, 0x8c, 0x2e, 0x75, 0x76, 0x2e, 0x79, 0xd6, 0xaf, - 0x11, 0x99, 0x58, 0xf2, 0x34, 0x14, 0xe7, 0x19, 0x45, 0xa4, 0x95, 0xef, 0x41, 0x7a, 0x48, 0x4d, - 0xa1, 0x67, 0xa1, 0x76, 0x83, 0x65, 0x13, 0xff, 0x72, 0xb6, 0x5e, 0xa0, 0x5e, 0x65, 0xc3, 0xb2, - 0xc9, 0x0e, 0x35, 0x09, 0xe6, 0x00, 0xf5, 0x08, 0xd2, 0x2c, 0xe6, 0xa0, 0x2f, 0x43, 0xba, 0xd6, - 0x6a, 0x37, 0x94, 0xb9, 0xd2, 0xf2, 0xc9, 0x69, 0x79, 0x81, 0xbb, 0x84, 0x31, 0xd8, 0xda, 0x45, - 0xeb, 0x90, 0x7d, 0xde, 0xd9, 0xde, 0xdd, 0x61, 0xcb, 0xeb, 0xc6, 0xc9, 0x69, 0x79, 0x29, 0x64, - 0x0b, 0xa7, 0xa1, 0x3b, 0x90, 0xe9, 0xef, 0x74, 0x37, 0x7a, 0x4a, 0xb2, 0x84, 0x4e, 0x4e, 0xcb, - 0x8b, 0x21, 0x9f, 0xdb, 0x5c, 0x5a, 0x96, 0xb3, 0x9a, 0x0f, 0xe9, 0xea, 0x8f, 0x12, 0x50, 0x88, - 0x6c, 0x38, 0xb6, 0x30, 0x1b, 0xcd, 0x8d, 0xea, 0xee, 0x76, 0x5f, 0x99, 0x8b, 0x2c, 0xcc, 0x08, - 0xa4, 0x41, 0xf6, 0xf5, 0xb1, 0xcd, 0xe2, 0x1c, 0xd4, 0x3b, 0xed, 0x5e, 0xab, 0xd7, 0x6f, 0xb6, - 0xfb, 0x4a, 0xa2, 0xb4, 0x7a, 0x72, 0x5a, 0x5e, 0x99, 0x05, 0x6f, 0x8c, 0x6d, 0x9b, 0x2d, 0xcd, - 0x7a, 0xb5, 0xbe, 0xc5, 0xd7, 0xfa, 0x74, 0x69, 0x46, 0x50, 0x75, 0xdd, 0x38, 0x20, 0x26, 0xba, - 0x0f, 0xf9, 0x46, 0x73, 0xbb, 0xb9, 0x59, 0xe5, 0xd1, 0xbd, 0x74, 0xe7, 0xe4, 0xb4, 0x7c, 0xeb, - 0xf5, 0xde, 0x6d, 0x32, 0xd0, 0x7d, 0x62, 0xce, 0x2c, 0xd1, 0x08, 0x44, 0xfd, 0x59, 0x12, 0x16, - 0x30, 0x2b, 0x87, 0x5d, 0xbf, 0x4b, 0x6d, 0xcb, 0x98, 0xa0, 0x2e, 0xe4, 0x0d, 0xea, 0x98, 0x56, - 0x24, 0x4e, 0x3c, 0xbe, 0x24, 0x25, 0x9a, 0x4a, 0x05, 0xad, 0x7a, 0x20, 0x89, 0xa7, 0x4a, 0xd0, - 0x03, 0xc8, 0x98, 0xc4, 0xd6, 0x27, 0x32, 0x37, 0xbb, 0x55, 0x11, 0x05, 0x77, 0x25, 0x28, 0xb8, - 0x2b, 0x0d, 0x59, 0x70, 0x63, 0x81, 0xe3, 0x35, 0x88, 0xfe, 0x52, 0xd3, 0x7d, 0x9f, 0x0c, 0x47, - 0xbe, 0x48, 0xcc, 0xd2, 0xb8, 0x30, 0xd4, 0x5f, 0x56, 0x25, 0x09, 0x3d, 0x82, 0xec, 0xb1, 0xe5, - 0x98, 0xf4, 0x58, 0xe6, 0x5e, 0x57, 0x28, 0x95, 0x40, 0xf5, 0x84, 0xa5, 0x24, 0x33, 0x66, 0xb2, - 0x35, 0xd4, 0xee, 0xb4, 0x9b, 0xc1, 0x1a, 0x92, 0xfc, 0x8e, 0xd3, 0xa6, 0x0e, 0xdb, 0xff, 0xd0, - 0x69, 0x6b, 0x1b, 0xd5, 0xd6, 0xf6, 0x2e, 0x66, 0xeb, 0x68, 0xe5, 0xe4, 0xb4, 0xac, 0x84, 0x90, - 0x0d, 0xdd, 0xb2, 0x59, 0x31, 0x70, 0x0b, 0x52, 0xd5, 0xf6, 0xf7, 0x94, 0x64, 0x49, 0x39, 0x39, - 0x2d, 0x17, 0x43, 0x76, 0xd5, 0x99, 0x4c, 0xfd, 0x3e, 0xdb, 0xaf, 0xfa, 0xf7, 0x29, 0x28, 0xee, - 0x8e, 0x4c, 0xdd, 0x27, 0x62, 0x9f, 0xa1, 0x32, 0x14, 0x46, 0xba, 0xab, 0xdb, 0x36, 0xb1, 0x2d, - 0x6f, 0x28, 0xaf, 0x12, 0xa2, 0x24, 0xf4, 0xc1, 0x9b, 0xba, 0xb1, 0x96, 0x63, 0x7b, 0xe7, 0x07, - 0xff, 0xb6, 0x9e, 0x08, 0x1c, 0xba, 0x0b, 0x8b, 0xfb, 0xc2, 0x5a, 0x4d, 0x37, 0xf8, 0xc4, 0xa6, - 0xf8, 0xc4, 0x56, 0xe2, 0x26, 0x36, 0x6a, 0x56, 0x45, 0x0e, 0xb2, 0xca, 0xa5, 0xf0, 0xc2, 0x7e, - 0xb4, 0x89, 0x9e, 0xc0, 0xfc, 0x90, 0x3a, 0x96, 0x4f, 0xdd, 0xeb, 0x67, 0x21, 0x40, 0xa2, 0x7b, - 0xb0, 0xcc, 0x26, 0x37, 0xb0, 0x87, 0xb3, 0xf9, 0x71, 0x9e, 0xc4, 0x4b, 0x43, 0xfd, 0xa5, 0xec, - 0x10, 0x33, 0x32, 0xaa, 0x41, 0x86, 0xba, 0x2c, 0x5f, 0xcc, 0x72, 0x73, 0xef, 0x5f, 0x6b, 0xae, - 0x68, 0x74, 0x98, 0x0c, 0x16, 0xa2, 0xea, 0x37, 0x61, 0xe1, 0xc2, 0x20, 0x58, 0x9a, 0xd4, 0xad, - 0xee, 0xf6, 0x9a, 0xca, 0x1c, 0x2a, 0x42, 0xae, 0xde, 0x69, 0xf7, 0x5b, 0xed, 0x5d, 0x96, 0xe7, - 0x15, 0x21, 0x87, 0x3b, 0xdb, 0xdb, 0xb5, 0x6a, 0xfd, 0x99, 0x92, 0x54, 0x2b, 0x50, 0x88, 0x68, - 0x43, 0x8b, 0x00, 0xbd, 0x7e, 0xa7, 0xab, 0x6d, 0xb4, 0x70, 0xaf, 0x2f, 0xb2, 0xc4, 0x5e, 0xbf, - 0x8a, 0xfb, 0x92, 0x90, 0x50, 0xff, 0x33, 0x19, 0xcc, 0xa8, 0x4c, 0x0c, 0x6b, 0x17, 0x13, 0xc3, - 0x2b, 0x8c, 0x97, 0xa9, 0xe1, 0xb4, 0x11, 0x26, 0x88, 0x1f, 0x00, 0xf0, 0x85, 0x43, 0x4c, 0x4d, - 0xf7, 0xe5, 0xc4, 0x97, 0x5e, 0x73, 0x72, 0x3f, 0xb8, 0xd1, 0xc2, 0x79, 0x89, 0xae, 0xfa, 0xe8, - 0x5b, 0x50, 0x34, 0xe8, 0x70, 0x64, 0x13, 0x29, 0x9c, 0xba, 0x56, 0xb8, 0x10, 0xe2, 0xab, 0x7e, - 0x34, 0x35, 0x4d, 0x5f, 0x4c, 0x9e, 0x7f, 0x3b, 0x11, 0x78, 0x26, 0x26, 0x1b, 0x2d, 0x42, 0x6e, - 0xb7, 0xdb, 0xa8, 0xf6, 0x5b, 0xed, 0x4d, 0x25, 0x81, 0x00, 0xb2, 0xdc, 0xd5, 0x0d, 0x25, 0xc9, - 0xb2, 0xe8, 0x7a, 0x67, 0xa7, 0xbb, 0xdd, 0xe4, 0x11, 0x0b, 0xad, 0x80, 0x12, 0x38, 0x5b, 0xe3, - 0x8e, 0x6c, 0x36, 0x94, 0x34, 0xba, 0x01, 0x4b, 0x21, 0x55, 0x4a, 0x66, 0xd0, 0x4d, 0x40, 0x21, - 0x71, 0xaa, 0x22, 0xab, 0xfe, 0x06, 0x2c, 0xd5, 0xa9, 0xe3, 0xeb, 0x96, 0x13, 0x56, 0x18, 0x8f, - 0xd9, 0xa0, 0x25, 0x49, 0xb3, 0xe4, 0x4d, 0x50, 0x6d, 0xe9, 0xfc, 0x6c, 0xbd, 0x10, 0x42, 0x5b, - 0x0d, 0x9e, 0x2a, 0xc9, 0x86, 0xc9, 0xf6, 0xef, 0xc8, 0x32, 0xb9, 0x73, 0x33, 0xb5, 0xf9, 0xf3, - 0xb3, 0xf5, 0x54, 0xb7, 0xd5, 0xc0, 0x8c, 0x86, 0xbe, 0x0c, 0x79, 0xf2, 0xd2, 0xf2, 0x35, 0x83, - 0x9d, 0x4b, 0xcc, 0x81, 0x19, 0x9c, 0x63, 0x84, 0x3a, 0x3b, 0x86, 0x6a, 0x00, 0x5d, 0xea, 0xfa, - 0xb2, 0xe7, 0xaf, 0x43, 0x66, 0x44, 0x5d, 0x7e, 0x77, 0x71, 0xe9, 0x8d, 0x1a, 0x83, 0x8b, 0x85, - 0x8a, 0x05, 0x58, 0xfd, 0x83, 0x14, 0x40, 0x5f, 0xf7, 0x0e, 0xa5, 0x92, 0xa7, 0x90, 0x0f, 0x6f, - 0x27, 0xe5, 0x25, 0xc8, 0x95, 0xb3, 0x1d, 0x82, 0xd1, 0x93, 0x60, 0xb1, 0x89, 0xda, 0x29, 0xb6, - 0x88, 0x0d, 0x3a, 0x8a, 0x2b, 0x3f, 0x2e, 0x16, 0x48, 0xec, 0x98, 0x27, 0xae, 0x2b, 0x67, 0x9e, - 0xfd, 0x44, 0x75, 0x7e, 0x2c, 0x08, 0xa7, 0xc9, 0xec, 0x3b, 0xf6, 0xda, 0x67, 0x66, 0x46, 0xb6, - 0xe6, 0xf0, 0x54, 0x0e, 0x7d, 0x0c, 0x05, 0x36, 0x6e, 0xcd, 0xe3, 0x3c, 0x99, 0x78, 0x5f, 0xea, - 0x2a, 0xa1, 0x01, 0xc3, 0x68, 0xea, 0xe5, 0x3b, 0x00, 0xfa, 0x68, 0x64, 0x5b, 0xc4, 0xd4, 0xf6, - 0x26, 0x3c, 0xd3, 0xce, 0xe3, 0xbc, 0xa4, 0xd4, 0x26, 0x6c, 0xbb, 0x04, 0x6c, 0xdd, 0xe7, 0xd9, - 0xf3, 0x35, 0x0e, 0x94, 0xe8, 0xaa, 0x5f, 0x53, 0x60, 0xd1, 0x1d, 0x3b, 0xcc, 0xa1, 0xd2, 0x3a, - 0xf5, 0xcf, 0x93, 0xf0, 0x56, 0x9b, 0xf8, 0xc7, 0xd4, 0x3d, 0xac, 0xfa, 0xbe, 0x6e, 0x1c, 0x0c, - 0x89, 0x23, 0xa7, 0x2f, 0x52, 0xd0, 0x24, 0x2e, 0x14, 0x34, 0xab, 0x30, 0xaf, 0xdb, 0x96, 0xee, - 0x11, 0x91, 0xbc, 0xe5, 0x71, 0xd0, 0x64, 0x65, 0x17, 0x2b, 0xe2, 0x88, 0xe7, 0x11, 0x71, 0xaf, - 0xc2, 0x0c, 0x0f, 0x08, 0xe8, 0xfb, 0x70, 0x53, 0xa6, 0x69, 0x7a, 0xd8, 0x15, 0x2b, 0x28, 0x82, - 0x0b, 0xda, 0x66, 0x6c, 0x55, 0x19, 0x6f, 0x9c, 0xcc, 0xe3, 0xa6, 0xe4, 0xce, 0xc8, 0x97, 0x59, - 0xe1, 0x8a, 0x19, 0xc3, 0x2a, 0x6d, 0xc2, 0xad, 0x4b, 0x45, 0x3e, 0xd7, 0xbd, 0xcd, 0x3f, 0x25, - 0x01, 0x5a, 0xdd, 0xea, 0x8e, 0x74, 0x52, 0x03, 0xb2, 0xfb, 0xfa, 0xd0, 0xb2, 0x27, 0x57, 0x45, - 0xc0, 0x29, 0xbe, 0x52, 0x15, 0xee, 0xd8, 0xe0, 0x32, 0x58, 0xca, 0xf2, 0x9a, 0x72, 0xbc, 0xe7, - 0x10, 0x3f, 0xac, 0x29, 0x79, 0x8b, 0x99, 0xe1, 0xea, 0x4e, 0xb8, 0x74, 0x45, 0x83, 0x4d, 0x00, - 0x4b, 0x79, 0x8e, 0xf5, 0x49, 0x10, 0xb6, 0x64, 0x13, 0x6d, 0xf1, 0xdb, 0x51, 0xe2, 0x1e, 0x11, - 0x73, 0x35, 0xc3, 0x9d, 0x7a, 0x9d, 0x3d, 0x58, 0xc2, 0x85, 0xef, 0x42, 0xe9, 0xd2, 0x47, 0x3c, - 0x65, 0x9a, 0xb2, 0x3e, 0x97, 0x8f, 0x1e, 0xc2, 0xc2, 0x85, 0x71, 0xbe, 0x56, 0xcc, 0xb7, 0xba, - 0xcf, 0xbf, 0xae, 0xa4, 0xe5, 0xaf, 0x6f, 0x2a, 0x59, 0xf5, 0x4f, 0x53, 0x22, 0xd0, 0x48, 0xaf, - 0xc6, 0xbf, 0x0a, 0xe4, 0xf8, 0xea, 0x36, 0xa8, 0x2d, 0x03, 0xc0, 0x7b, 0x57, 0xc7, 0x1f, 0x56, - 0xd3, 0x71, 0x38, 0x0e, 0x05, 0xd1, 0x3a, 0x14, 0xc4, 0x2a, 0xd6, 0xd8, 0x86, 0xe3, 0x6e, 0x5d, - 0xc0, 0x20, 0x48, 0x4c, 0x12, 0xdd, 0x85, 0x45, 0x7e, 0xf9, 0xe3, 0x1d, 0x10, 0x53, 0x60, 0xd2, - 0x1c, 0xb3, 0x10, 0x52, 0x39, 0x6c, 0x07, 0x8a, 0x92, 0xa0, 0xf1, 0x7c, 0x3e, 0xc3, 0x0d, 0xba, - 0x77, 0x9d, 0x41, 0x42, 0x84, 0xa7, 0xf9, 0x85, 0xd1, 0xb4, 0xa1, 0x36, 0x20, 0x17, 0x18, 0x8b, - 0x56, 0x21, 0xd5, 0xaf, 0x77, 0x95, 0xb9, 0xd2, 0xd2, 0xc9, 0x69, 0xb9, 0x10, 0x90, 0xfb, 0xf5, - 0x2e, 0xe3, 0xec, 0x36, 0xba, 0x4a, 0xe2, 0x22, 0x67, 0xb7, 0xd1, 0x2d, 0xa5, 0x59, 0x0e, 0xa6, - 0xee, 0x43, 0x21, 0xd2, 0x03, 0x7a, 0x07, 0xe6, 0x5b, 0xed, 0x4d, 0xdc, 0xec, 0xf5, 0x94, 0xb9, - 0xd2, 0xcd, 0x93, 0xd3, 0x32, 0x8a, 0x70, 0x5b, 0xce, 0x80, 0xcd, 0x0f, 0xba, 0x03, 0xe9, 0xad, - 0x0e, 0x3b, 0xdb, 0x45, 0x01, 0x11, 0x41, 0x6c, 0x51, 0xcf, 0x2f, 0xdd, 0x90, 0xc9, 0x5d, 0x54, - 0xb1, 0xfa, 0x87, 0x09, 0xc8, 0x8a, 0xcd, 0x14, 0x3b, 0x51, 0x55, 0x98, 0x0f, 0xae, 0x09, 0x44, - 0x71, 0xf7, 0xde, 0xe5, 0x85, 0x58, 0x45, 0xd6, 0x4d, 0x62, 0xf9, 0x05, 0x72, 0xa5, 0x0f, 0xa1, - 0x18, 0x65, 0x7c, 0xae, 0xc5, 0xf7, 0x7d, 0x28, 0xb0, 0xf5, 0x1d, 0x14, 0x64, 0x8f, 0x21, 0x2b, - 0x02, 0x42, 0x78, 0xd6, 0x5c, 0x5e, 0x15, 0x4a, 0x24, 0x7a, 0x0a, 0xf3, 0xa2, 0x92, 0x0c, 0x6e, - 0x87, 0xd7, 0xae, 0xde, 0x45, 0x38, 0x80, 0xab, 0x1f, 0x43, 0xba, 0x4b, 0x88, 0xcb, 0x7c, 0xef, - 0x50, 0x93, 0x4c, 0x8f, 0x67, 0x59, 0x04, 0x9b, 0xa4, 0xd5, 0x60, 0x45, 0xb0, 0x49, 0x5a, 0x66, - 0x78, 0xff, 0x95, 0x8c, 0xdc, 0x7f, 0xf5, 0xa1, 0xf8, 0x82, 0x58, 0x83, 0x03, 0x9f, 0x98, 0x5c, - 0xd1, 0x7d, 0x48, 0x8f, 0x48, 0x68, 0xfc, 0x6a, 0xec, 0x02, 0x23, 0xc4, 0xc5, 0x1c, 0xc5, 0xe2, - 0xc8, 0x31, 0x97, 0x96, 0x4f, 0x1a, 0xb2, 0xa5, 0xfe, 0x63, 0x12, 0x16, 0x5b, 0x9e, 0x37, 0xd6, - 0x1d, 0x23, 0xc8, 0xdc, 0xbe, 0x7d, 0x31, 0x73, 0x8b, 0x7d, 0xfb, 0xb9, 0x28, 0x72, 0xf1, 0x5a, - 0x4f, 0x9e, 0x9e, 0xc9, 0xf0, 0xf4, 0x54, 0x7f, 0x9a, 0x08, 0xee, 0xee, 0xee, 0x46, 0xb6, 0xbb, - 0xa8, 0x03, 0xa3, 0x9a, 0xc8, 0xae, 0x73, 0xe8, 0xd0, 0x63, 0x07, 0xbd, 0x0d, 0x19, 0xdc, 0x6c, - 0x37, 0x5f, 0x28, 0x09, 0xb1, 0x3c, 0x2f, 0x80, 0x30, 0x71, 0xc8, 0x31, 0xd3, 0xd4, 0x6d, 0xb6, - 0x1b, 0x2c, 0xd3, 0x4a, 0xc6, 0x68, 0xea, 0x12, 0xc7, 0xb4, 0x9c, 0x01, 0x7a, 0x07, 0xb2, 0xad, - 0x5e, 0x6f, 0x97, 0x97, 0x89, 0x6f, 0x9d, 0x9c, 0x96, 0x6f, 0x5c, 0x40, 0xf1, 0x7b, 0x5b, 0x93, - 0x81, 0x58, 0x99, 0xc3, 0x72, 0xb0, 0x18, 0x10, 0xcb, 0x9f, 0x05, 0x08, 0x77, 0xfa, 0xd5, 0x7e, - 0x53, 0xc9, 0xc4, 0x80, 0x30, 0x65, 0x7f, 0xe5, 0x76, 0xfb, 0xd7, 0x24, 0x28, 0x55, 0xc3, 0x20, - 0x23, 0x9f, 0xf1, 0x65, 0x65, 0xd9, 0x87, 0xdc, 0x88, 0xfd, 0xb2, 0x48, 0x90, 0x25, 0x3d, 0x8d, - 0x7d, 0xbd, 0x9c, 0x91, 0xab, 0x60, 0x6a, 0x93, 0xaa, 0x39, 0xb4, 0x3c, 0xcf, 0xa2, 0x8e, 0xa0, - 0xe1, 0x50, 0x53, 0xe9, 0xbf, 0x12, 0x70, 0x23, 0x06, 0x81, 0x1e, 0x42, 0xda, 0xa5, 0x76, 0x30, - 0x87, 0xb7, 0x2f, 0xbb, 0x96, 0x65, 0xa2, 0x98, 0x23, 0xd1, 0x1a, 0x80, 0x3e, 0xf6, 0xa9, 0xce, - 0xfb, 0xe7, 0xb3, 0x97, 0xc3, 0x11, 0x0a, 0x7a, 0x01, 0x59, 0x8f, 0x18, 0x2e, 0x09, 0x72, 0xe9, - 0x8f, 0xff, 0xbf, 0xd6, 0x57, 0x7a, 0x5c, 0x0d, 0x96, 0xea, 0x4a, 0x15, 0xc8, 0x0a, 0x0a, 0x5b, - 0xf6, 0xa6, 0xee, 0xeb, 0xf2, 0xd2, 0x9e, 0xff, 0x66, 0xab, 0x49, 0xb7, 0x07, 0xc1, 0x6a, 0xd2, - 0xed, 0x81, 0xfa, 0x77, 0x49, 0x80, 0xe6, 0x4b, 0x9f, 0xb8, 0x8e, 0x6e, 0xd7, 0xab, 0xa8, 0x19, - 0x89, 0xfe, 0x62, 0xb4, 0x5f, 0x8d, 0x7d, 0x89, 0x08, 0x25, 0x2a, 0xf5, 0x6a, 0x4c, 0xfc, 0xbf, - 0x05, 0xa9, 0xb1, 0x2b, 0x1f, 0xa4, 0x45, 0x1e, 0xbc, 0x8b, 0xb7, 0x31, 0xa3, 0xa1, 0xe6, 0x34, - 0x6c, 0xa5, 0x2e, 0x7f, 0x76, 0x8e, 0x74, 0x10, 0x1b, 0xba, 0xd8, 0xce, 0x37, 0x74, 0xcd, 0x20, - 0xf2, 0xe4, 0x28, 0x8a, 0x9d, 0x5f, 0xaf, 0xd6, 0x89, 0xeb, 0xe3, 0xac, 0xa1, 0xb3, 0xff, 0x5f, - 0x28, 0xbe, 0xdd, 0x07, 0x98, 0x0e, 0x0d, 0xad, 0x41, 0xa6, 0xbe, 0xd1, 0xeb, 0x6d, 0x2b, 0x73, - 0x22, 0x80, 0x4f, 0x59, 0x9c, 0xac, 0xfe, 0x75, 0x12, 0x72, 0xf5, 0xaa, 0x3c, 0x56, 0xeb, 0xa0, - 0xf0, 0xa8, 0xc4, 0x9f, 0x3a, 0xc8, 0xcb, 0x91, 0xe5, 0x4e, 0x64, 0x60, 0xb9, 0xa2, 0xa8, 0x5d, - 0x64, 0x22, 0xcc, 0xea, 0x26, 0x17, 0x40, 0x18, 0x8a, 0x44, 0x3a, 0x41, 0x33, 0xf4, 0x20, 0xc6, - 0xaf, 0x5d, 0xed, 0x2c, 0x51, 0x9e, 0x4c, 0xdb, 0x1e, 0x2e, 0x04, 0x4a, 0xea, 0xba, 0x87, 0x3e, - 0x80, 0x25, 0xcf, 0x1a, 0x38, 0x96, 0x33, 0xd0, 0x02, 0xe7, 0xf1, 0x77, 0x97, 0xda, 0xf2, 0xf9, - 0xd9, 0xfa, 0x42, 0x4f, 0xb0, 0xa4, 0x0f, 0x17, 0x24, 0xb2, 0xce, 0x5d, 0x89, 0xbe, 0x09, 0x8b, - 0x11, 0x51, 0xe6, 0x45, 0xe1, 0x76, 0xe5, 0xfc, 0x6c, 0xbd, 0x18, 0x4a, 0x3e, 0x23, 0x13, 0x5c, - 0x0c, 0x05, 0x9f, 0x11, 0x7e, 0xff, 0xb2, 0x4f, 0x5d, 0x83, 0x68, 0x2e, 0xdf, 0xd3, 0xfc, 0x04, - 0x4f, 0xe3, 0x02, 0xa7, 0x89, 0x6d, 0xae, 0x3e, 0x87, 0x1b, 0x1d, 0xd7, 0x38, 0x20, 0x9e, 0x2f, - 0x5c, 0x21, 0xbd, 0xf8, 0x31, 0xdc, 0xf6, 0x75, 0xef, 0x50, 0x3b, 0xb0, 0x3c, 0x9f, 0xba, 0x13, - 0xcd, 0x25, 0x3e, 0x71, 0x18, 0x5f, 0xe3, 0x8f, 0xb5, 0xf2, 0xd2, 0xef, 0x16, 0xc3, 0x6c, 0x09, - 0x08, 0x0e, 0x10, 0xdb, 0x0c, 0xa0, 0xb6, 0xa0, 0xc8, 0xca, 0x14, 0x79, 0x71, 0xc6, 0x46, 0x0f, - 0x36, 0x1d, 0x68, 0x6f, 0x7c, 0x4c, 0xe5, 0x6d, 0x3a, 0x10, 0x3f, 0xd5, 0xef, 0x82, 0xd2, 0xb0, - 0xbc, 0x91, 0xee, 0x1b, 0x07, 0xc1, 0x6d, 0x26, 0x6a, 0x80, 0x72, 0x40, 0x74, 0xd7, 0xdf, 0x23, - 0xba, 0xaf, 0x8d, 0x88, 0x6b, 0x51, 0xf3, 0xfa, 0x59, 0x5e, 0x0a, 0x45, 0xba, 0x5c, 0x42, 0xfd, - 0xef, 0x04, 0x00, 0xd6, 0xf7, 0x83, 0x8c, 0xec, 0x6b, 0xb0, 0xec, 0x39, 0xfa, 0xc8, 0x3b, 0xa0, - 0xbe, 0x66, 0x39, 0x3e, 0x71, 0x8f, 0x74, 0x5b, 0x5e, 0xe0, 0x28, 0x01, 0xa3, 0x25, 0xe9, 0xe8, - 0x3e, 0xa0, 0x43, 0x42, 0x46, 0x1a, 0xb5, 0x4d, 0x2d, 0x60, 0x8a, 0xa7, 0xe4, 0x34, 0x56, 0x18, - 0xa7, 0x63, 0x9b, 0xbd, 0x80, 0x8e, 0x6a, 0xb0, 0xc6, 0x86, 0x4f, 0x1c, 0xdf, 0xb5, 0x88, 0xa7, - 0xed, 0x53, 0x57, 0xf3, 0x6c, 0x7a, 0xac, 0xed, 0x53, 0xdb, 0xa6, 0xc7, 0xc4, 0x0d, 0xee, 0xc6, - 0x4a, 0x36, 0x1d, 0x34, 0x05, 0x68, 0x83, 0xba, 0x3d, 0x9b, 0x1e, 0x6f, 0x04, 0x08, 0x96, 0xb6, - 0x4d, 0xc7, 0xec, 0x5b, 0xc6, 0x61, 0x90, 0xb6, 0x85, 0xd4, 0xbe, 0x65, 0x1c, 0xa2, 0x77, 0x60, - 0x81, 0xd8, 0x84, 0x5f, 0x91, 0x08, 0x54, 0x86, 0xa3, 0x8a, 0x01, 0x91, 0x81, 0xd4, 0x4f, 0x40, - 0x69, 0x3a, 0x86, 0x3b, 0x19, 0x45, 0xe6, 0xfc, 0x3e, 0x20, 0x16, 0x24, 0x35, 0x9b, 0x1a, 0x87, - 0xda, 0x50, 0x77, 0xf4, 0x01, 0xb3, 0x4b, 0xbc, 0xf0, 0x29, 0x8c, 0xb3, 0x4d, 0x8d, 0xc3, 0x1d, - 0x49, 0x57, 0x3f, 0x00, 0xe8, 0x8d, 0x5c, 0xa2, 0x9b, 0x1d, 0x96, 0x4d, 0x30, 0xd7, 0xf1, 0x96, - 0x66, 0xca, 0x17, 0x52, 0xea, 0xca, 0xad, 0xae, 0x08, 0x46, 0x23, 0xa4, 0xab, 0xbf, 0x0c, 0x37, - 0xba, 0xb6, 0x6e, 0xf0, 0xaf, 0x05, 0xba, 0xe1, 0x93, 0x15, 0x7a, 0x0a, 0x59, 0x01, 0x95, 0x33, - 0x19, 0xbb, 0xdd, 0xa6, 0x7d, 0x6e, 0xcd, 0x61, 0x89, 0xaf, 0x15, 0x01, 0xa6, 0x7a, 0xd4, 0xbf, - 0x4c, 0x40, 0x3e, 0xd4, 0x8f, 0xca, 0xe2, 0x25, 0xc6, 0x77, 0x75, 0xcb, 0x91, 0x55, 0x7d, 0x1e, - 0x47, 0x49, 0xa8, 0x05, 0x85, 0x51, 0x28, 0x7d, 0x65, 0x3e, 0x17, 0x63, 0x35, 0x8e, 0xca, 0xa2, - 0x0f, 0x21, 0x1f, 0x3c, 0x49, 0x07, 0x11, 0xf6, 0xea, 0x17, 0xec, 0x29, 0x5c, 0xfd, 0x36, 0xc0, - 0x77, 0xa8, 0xe5, 0xf4, 0xe9, 0x21, 0x71, 0xf8, 0x13, 0x2b, 0xab, 0x09, 0x49, 0xe0, 0x45, 0xd9, - 0xe2, 0xa5, 0xbe, 0x98, 0x82, 0xf0, 0xa5, 0x51, 0x34, 0xd5, 0xbf, 0x4d, 0x42, 0x16, 0x53, 0xea, - 0xd7, 0xab, 0xa8, 0x0c, 0x59, 0x19, 0x27, 0xf8, 0xf9, 0x53, 0xcb, 0x9f, 0x9f, 0xad, 0x67, 0x44, - 0x80, 0xc8, 0x18, 0x3c, 0x32, 0x44, 0x22, 0x78, 0xf2, 0xb2, 0x08, 0x8e, 0x1e, 0x42, 0x51, 0x82, - 0xb4, 0x03, 0xdd, 0x3b, 0x10, 0x05, 0x5a, 0x6d, 0xf1, 0xfc, 0x6c, 0x1d, 0x04, 0x72, 0x4b, 0xf7, - 0x0e, 0x30, 0x08, 0x34, 0xfb, 0x8d, 0x9a, 0x50, 0xf8, 0x94, 0x5a, 0x8e, 0xe6, 0xf3, 0x41, 0xc8, - 0xcb, 0xc4, 0xd8, 0x79, 0x9c, 0x0e, 0x55, 0x7e, 0x6f, 0x00, 0x9f, 0x4e, 0x07, 0xdf, 0x84, 0x05, - 0x97, 0x52, 0x5f, 0x84, 0x2d, 0x8b, 0x3a, 0xf2, 0x9e, 0xa2, 0x1c, 0x7b, 0x7d, 0x4d, 0xa9, 0x8f, - 0x25, 0x0e, 0x17, 0xdd, 0x48, 0x0b, 0x3d, 0x84, 0x15, 0x5b, 0xf7, 0x7c, 0x8d, 0xc7, 0x3b, 0x73, - 0xaa, 0x2d, 0xcb, 0xb7, 0x1a, 0x62, 0xbc, 0x0d, 0xce, 0x0a, 0x24, 0xd4, 0x7f, 0x4e, 0x40, 0x81, - 0x0d, 0xc6, 0xda, 0xb7, 0x0c, 0x96, 0xe4, 0x7d, 0xfe, 0xdc, 0xe3, 0x16, 0xa4, 0x0c, 0xcf, 0x95, - 0x4e, 0xe5, 0x87, 0x6f, 0xbd, 0x87, 0x31, 0xa3, 0xa1, 0x4f, 0x20, 0x2b, 0xef, 0x4b, 0x44, 0xda, - 0xa1, 0x5e, 0x9f, 0x8e, 0x4a, 0xdf, 0x48, 0x39, 0xbe, 0x96, 0xa7, 0xd6, 0x89, 0x43, 0x00, 0x47, - 0x49, 0xe8, 0x26, 0x24, 0x0d, 0xe1, 0x2e, 0xf9, 0x41, 0x4b, 0xbd, 0x8d, 0x93, 0x86, 0xa3, 0xfe, - 0x28, 0x01, 0x0b, 0xd3, 0x0d, 0xcf, 0x56, 0xc0, 0x6d, 0xc8, 0x7b, 0xe3, 0x3d, 0x6f, 0xe2, 0xf9, - 0x64, 0x18, 0x3c, 0x1f, 0x87, 0x04, 0xd4, 0x82, 0xbc, 0x6e, 0x0f, 0xa8, 0x6b, 0xf9, 0x07, 0x43, - 0x59, 0x89, 0xc6, 0xa7, 0x0a, 0x51, 0x9d, 0x95, 0x6a, 0x20, 0x82, 0xa7, 0xd2, 0xc1, 0xb9, 0x2f, - 0xbe, 0x31, 0xe0, 0xe7, 0xfe, 0xdb, 0x50, 0xb4, 0xf5, 0x21, 0xbf, 0x40, 0xf2, 0xad, 0xa1, 0x18, - 0x47, 0x1a, 0x17, 0x24, 0xad, 0x6f, 0x0d, 0x89, 0xaa, 0x42, 0x3e, 0x54, 0x86, 0x96, 0xa0, 0x50, - 0x6d, 0xf6, 0xb4, 0x47, 0x8f, 0x9f, 0x6a, 0x9b, 0xf5, 0x1d, 0x65, 0x4e, 0xe6, 0xa6, 0x7f, 0x95, - 0x80, 0x05, 0x19, 0x8e, 0x64, 0xbe, 0xff, 0x0e, 0xcc, 0xbb, 0xfa, 0xbe, 0x1f, 0x54, 0x24, 0x69, - 0xb1, 0xaa, 0x59, 0x84, 0x67, 0x15, 0x09, 0x63, 0xc5, 0x57, 0x24, 0x91, 0x0f, 0x1a, 0x52, 0x57, - 0x7e, 0xd0, 0x90, 0xfe, 0xb9, 0x7c, 0xd0, 0xa0, 0xfe, 0x26, 0xc0, 0x86, 0x65, 0x93, 0xbe, 0xb8, - 0x6b, 0x8a, 0xab, 0x2f, 0x59, 0x0e, 0x27, 0xef, 0x32, 0x83, 0x1c, 0xae, 0xd5, 0xc0, 0x8c, 0xc6, - 0x58, 0x03, 0xcb, 0x94, 0x9b, 0x91, 0xb3, 0x36, 0x19, 0x6b, 0x60, 0x99, 0xe1, 0xcb, 0x5b, 0xfa, - 0xba, 0x97, 0xb7, 0xd3, 0x04, 0x2c, 0xc9, 0xdc, 0x35, 0x0c, 0xbf, 0x5f, 0x85, 0xbc, 0x48, 0x63, - 0xa7, 0x05, 0x1d, 0x7f, 0xc4, 0x17, 0xb8, 0x56, 0x03, 0xe7, 0x04, 0xbb, 0x65, 0xa2, 0x75, 0x28, - 0x48, 0x68, 0xe4, 0xe3, 0x27, 0x10, 0xa4, 0x36, 0x33, 0xff, 0xeb, 0x90, 0xde, 0xb7, 0x6c, 0x22, - 0x17, 0x7a, 0x6c, 0x00, 0x98, 0x3a, 0x60, 0x6b, 0x0e, 0x73, 0x74, 0x2d, 0x17, 0x5c, 0xc6, 0x71, - 0xfb, 0x64, 0xd9, 0x19, 0xb5, 0x4f, 0x54, 0xa0, 0x33, 0xf6, 0x09, 0x1c, 0xb3, 0x4f, 0xb0, 0x85, - 0x7d, 0x12, 0x1a, 0xb5, 0x4f, 0x90, 0x7e, 0x2e, 0xf6, 0x6d, 0xc3, 0xcd, 0x9a, 0xad, 0x1b, 0x87, - 0xb6, 0xe5, 0xf9, 0xc4, 0x8c, 0x46, 0x8c, 0xc7, 0x90, 0xbd, 0x90, 0x74, 0x5e, 0x75, 0x6b, 0x29, - 0x91, 0xea, 0x7f, 0x24, 0xa0, 0xb8, 0x45, 0x74, 0xdb, 0x3f, 0x98, 0x5e, 0x0d, 0xf9, 0xc4, 0xf3, - 0xe5, 0x61, 0xc5, 0x7f, 0xa3, 0x6f, 0x40, 0x2e, 0xcc, 0x49, 0xae, 0x7d, 0x7f, 0x0b, 0xa1, 0xe8, - 0x09, 0xcc, 0xb3, 0x3d, 0x46, 0xc7, 0x41, 0xb1, 0x73, 0xd5, 0xd3, 0x8e, 0x44, 0xb2, 0x43, 0xc6, - 0x25, 0x3c, 0x09, 0xe1, 0x4b, 0x29, 0x83, 0x83, 0x26, 0xfa, 0x45, 0x28, 0xf2, 0x97, 0x89, 0x20, - 0xe7, 0xca, 0x5c, 0xa7, 0xb3, 0x20, 0x1e, 0x17, 0x45, 0xbe, 0xf5, 0xbf, 0x09, 0x58, 0xd9, 0xd1, - 0x27, 0x7b, 0x44, 0x86, 0x0d, 0x62, 0x62, 0x62, 0x50, 0xd7, 0x44, 0xdd, 0x68, 0xb8, 0xb9, 0xe2, - 0xad, 0x32, 0x4e, 0x38, 0x3e, 0xea, 0x04, 0x05, 0x58, 0x32, 0x52, 0x80, 0xad, 0x40, 0xc6, 0xa1, - 0x8e, 0x41, 0x64, 0x2c, 0x12, 0x0d, 0xd5, 0x8a, 0x86, 0x9a, 0x52, 0xf8, 0x8c, 0xc8, 0x1f, 0x01, - 0xdb, 0xd4, 0x0f, 0x7b, 0x43, 0x9f, 0x40, 0xa9, 0xd7, 0xac, 0xe3, 0x66, 0xbf, 0xd6, 0xf9, 0xae, - 0xd6, 0xab, 0x6e, 0xf7, 0xaa, 0x8f, 0x1f, 0x6a, 0xdd, 0xce, 0xf6, 0xf7, 0x1e, 0x3d, 0x79, 0xf8, - 0x0d, 0x25, 0x51, 0x2a, 0x9f, 0x9c, 0x96, 0x6f, 0xb7, 0xab, 0xf5, 0x6d, 0xb1, 0x63, 0xf6, 0xe8, - 0xcb, 0x9e, 0x6e, 0x7b, 0xfa, 0xe3, 0x87, 0x5d, 0x6a, 0x4f, 0x18, 0x86, 0x2d, 0xeb, 0x62, 0xf4, - 0xbc, 0x8a, 0x1e, 0xc3, 0x89, 0x4b, 0x8f, 0xe1, 0xe9, 0x69, 0x9e, 0xbc, 0xe4, 0x34, 0xdf, 0x80, - 0x15, 0xc3, 0xa5, 0x9e, 0xa7, 0xb1, 0xec, 0x9f, 0x98, 0x33, 0xf5, 0xc5, 0x97, 0xce, 0xcf, 0xd6, - 0x97, 0xeb, 0x8c, 0xdf, 0xe3, 0x6c, 0xa9, 0x7e, 0xd9, 0x88, 0x90, 0x78, 0x4f, 0xea, 0x1f, 0xa5, - 0x58, 0x22, 0x65, 0x1d, 0x59, 0x36, 0x19, 0x10, 0x0f, 0x3d, 0x87, 0x25, 0xc3, 0x25, 0x26, 0x4b, - 0xeb, 0x75, 0x3b, 0xfa, 0x11, 0xed, 0x2f, 0xc4, 0xe6, 0x34, 0xa1, 0x60, 0xa5, 0x1e, 0x4a, 0xf5, - 0x46, 0xc4, 0xc0, 0x8b, 0xc6, 0x85, 0x36, 0xfa, 0x14, 0x96, 0x3c, 0x62, 0x5b, 0xce, 0xf8, 0xa5, - 0x66, 0x50, 0xc7, 0x27, 0x2f, 0x83, 0x17, 0xb1, 0xeb, 0xf4, 0xf6, 0x9a, 0xdb, 0x4c, 0xaa, 0x2e, - 0x84, 0x6a, 0xe8, 0xfc, 0x6c, 0x7d, 0xf1, 0x22, 0x0d, 0x2f, 0x4a, 0xcd, 0xb2, 0x5d, 0x6a, 0xc3, - 0xe2, 0x45, 0x6b, 0xd0, 0x8a, 0xdc, 0xfb, 0x3c, 0x84, 0x04, 0x7b, 0x1b, 0xdd, 0x86, 0x9c, 0x4b, - 0x06, 0x96, 0xe7, 0xbb, 0xc2, 0xcd, 0x8c, 0x13, 0x52, 0xd8, 0xce, 0x17, 0x5f, 0x40, 0x95, 0x7e, - 0x1d, 0x66, 0x7a, 0x64, 0x9b, 0xc5, 0xb4, 0x3c, 0x7d, 0x4f, 0xaa, 0xcc, 0xe1, 0xa0, 0xc9, 0xd6, - 0xe0, 0xd8, 0x0b, 0x13, 0x35, 0xfe, 0x9b, 0xd1, 0x78, 0x46, 0x21, 0xbf, 0x07, 0xe3, 0x39, 0x43, - 0xf0, 0x61, 0x69, 0x3a, 0xf2, 0x61, 0xe9, 0x0a, 0x64, 0x6c, 0x72, 0x44, 0x6c, 0x71, 0x96, 0x63, - 0xd1, 0xb8, 0xf7, 0x10, 0x8a, 0xc1, 0x17, 0x8c, 0xfc, 0xcb, 0x89, 0x1c, 0xa4, 0xfb, 0xd5, 0xde, - 0x33, 0x65, 0x0e, 0x01, 0x64, 0xc5, 0xe2, 0x14, 0xaf, 0x75, 0xf5, 0x4e, 0x7b, 0xa3, 0xb5, 0xa9, - 0x24, 0xef, 0xfd, 0x2c, 0x05, 0xf9, 0xf0, 0xbd, 0x88, 0x9d, 0x1d, 0xed, 0xe6, 0x8b, 0x60, 0x75, - 0x87, 0xf4, 0x36, 0x39, 0x46, 0x6f, 0x4f, 0x6f, 0xa1, 0x3e, 0x11, 0x0f, 0xe4, 0x21, 0x3b, 0xb8, - 0x81, 0x7a, 0x17, 0x72, 0xd5, 0x5e, 0xaf, 0xb5, 0xd9, 0x6e, 0x36, 0x94, 0xcf, 0x12, 0xa5, 0x2f, - 0x9d, 0x9c, 0x96, 0x97, 0x43, 0x50, 0xd5, 0x13, 0x8b, 0x8f, 0xa3, 0xea, 0xf5, 0x66, 0xb7, 0xdf, - 0x6c, 0x28, 0xaf, 0x92, 0xb3, 0x28, 0x7e, 0xab, 0xc2, 0x3f, 0xdd, 0xc9, 0x77, 0x71, 0xb3, 0x5b, - 0xc5, 0xac, 0xc3, 0xcf, 0x92, 0xe2, 0x72, 0x6c, 0xda, 0xa3, 0x4b, 0x46, 0xba, 0xcb, 0xfa, 0x5c, - 0x0b, 0xbe, 0x85, 0x7b, 0x95, 0x12, 0x9f, 0x77, 0x4c, 0x1f, 0xbf, 0x88, 0x6e, 0x4e, 0x58, 0x6f, - 0xfc, 0xd5, 0x91, 0xab, 0x49, 0xcd, 0xf4, 0xd6, 0x63, 0xb1, 0x87, 0x69, 0x51, 0x61, 0x1e, 0xef, - 0xb6, 0xdb, 0x0c, 0xf4, 0x2a, 0x3d, 0x33, 0x3a, 0x3c, 0x76, 0x58, 0xc5, 0x8c, 0xee, 0x42, 0x2e, - 0x78, 0x94, 0x54, 0x3e, 0x4b, 0xcf, 0x18, 0x54, 0x0f, 0x5e, 0x54, 0x79, 0x87, 0x5b, 0xbb, 0x7d, - 0xfe, 0xa9, 0xde, 0xab, 0xcc, 0x6c, 0x87, 0x07, 0x63, 0xdf, 0xa4, 0xc7, 0x0e, 0xdb, 0xb3, 0xf2, - 0x1e, 0xee, 0xb3, 0x8c, 0xb8, 0xb4, 0x08, 0x31, 0xf2, 0x12, 0xee, 0x5d, 0xc8, 0xe1, 0xe6, 0x77, - 0xc4, 0x57, 0x7d, 0xaf, 0xb2, 0x33, 0x7a, 0x30, 0xf9, 0x94, 0x18, 0xb2, 0xb7, 0x0e, 0xee, 0x6e, - 0x55, 0xb9, 0xcb, 0x67, 0x51, 0x1d, 0x77, 0x74, 0xa0, 0x3b, 0xc4, 0x9c, 0x7e, 0xe3, 0x12, 0xb2, - 0xee, 0xfd, 0x0a, 0xe4, 0x82, 0xcc, 0x14, 0xad, 0x41, 0xf6, 0x45, 0x07, 0x3f, 0x6b, 0x62, 0x65, - 0x4e, 0xf8, 0x30, 0xe0, 0xbc, 0x10, 0x35, 0x45, 0x19, 0xe6, 0x77, 0xaa, 0xed, 0xea, 0x66, 0x13, - 0x07, 0x57, 0xe4, 0x01, 0x40, 0xa6, 0x57, 0x25, 0x45, 0x76, 0x10, 0xea, 0xac, 0xad, 0xfe, 0xf0, - 0x27, 0x6b, 0x73, 0x3f, 0xfe, 0xc9, 0xda, 0xdc, 0xab, 0xf3, 0xb5, 0xc4, 0x0f, 0xcf, 0xd7, 0x12, - 0xff, 0x70, 0xbe, 0x96, 0xf8, 0xf7, 0xf3, 0xb5, 0xc4, 0x5e, 0x96, 0x1f, 0x02, 0x4f, 0xfe, 0x2f, - 0x00, 0x00, 0xff, 0xff, 0x4b, 0xdb, 0xdc, 0xec, 0xf0, 0x31, 0x00, 0x00, + // 5043 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7a, 0x4d, 0x6c, 0x24, 0x49, + 0x56, 0xbf, 0xeb, 0xd3, 0x55, 0xaf, 0xca, 0x76, 0x3a, 0xda, 0xdb, 0xe3, 0xae, 0xed, 0xb6, 0x6b, + 0x72, 0xa6, 0x77, 0x66, 0x7b, 0xfb, 0x5f, 0xfd, 0xb5, 0xbb, 0xea, 0x99, 0xf9, 0xef, 0xce, 0xd4, + 0x97, 0xed, 0xda, 0xb6, 0xab, 0x4a, 0x51, 0xe5, 0xee, 0x5d, 0x24, 0x48, 0xa5, 0x33, 0xc3, 0xe5, + 0x1c, 0x67, 0x65, 0x14, 0x99, 0x59, 0x76, 0x17, 0x0b, 0xa2, 0xc5, 0x01, 0x90, 0x4f, 0x70, 0x62, + 0x11, 0x32, 0x42, 0x82, 0x13, 0x42, 0xe2, 0x00, 0x12, 0x82, 0xd3, 0x20, 0x71, 0xd8, 0x1b, 0x2c, + 0x48, 0x68, 0x05, 0x92, 0x61, 0x7d, 0xe0, 0xb6, 0x82, 0x0b, 0xe2, 0x02, 0x12, 0x8a, 0x8f, 0xcc, + 0x4a, 0x57, 0xa7, 0xed, 0x1e, 0x66, 0x2f, 0x76, 0xc5, 0x7b, 0xbf, 0xf7, 0xe2, 0xc5, 0x8b, 0x88, + 0x17, 0xef, 0x45, 0x24, 0xdc, 0x1b, 0x58, 0xfe, 0xc1, 0x78, 0xaf, 0x62, 0xd0, 0xe1, 0x03, 0x93, + 0x1a, 0x87, 0xc4, 0x7d, 0xe0, 0x1d, 0xeb, 0xee, 0xf0, 0xd0, 0xf2, 0x1f, 0xe8, 0x23, 0xeb, 0x81, + 0x3f, 0x19, 0x11, 0xaf, 0x32, 0x72, 0xa9, 0x4f, 0x11, 0x12, 0x80, 0x4a, 0x00, 0xa8, 0x1c, 0x3d, + 0x2a, 0xad, 0x0f, 0x28, 0x1d, 0xd8, 0xe4, 0x01, 0x47, 0xec, 0x8d, 0xf7, 0x1f, 0xf8, 0xd6, 0x90, + 0x78, 0xbe, 0x3e, 0x1c, 0x09, 0xa1, 0xd2, 0xda, 0x2c, 0xc0, 0x1c, 0xbb, 0xba, 0x6f, 0x51, 0x47, + 0xf2, 0x57, 0x06, 0x74, 0x40, 0xf9, 0xcf, 0x07, 0xec, 0x97, 0xa0, 0xaa, 0xeb, 0x30, 0xff, 0x9c, + 0xb8, 0x9e, 0x45, 0x1d, 0xb4, 0x02, 0x19, 0xcb, 0x31, 0xc9, 0xcb, 0xd5, 0x44, 0x39, 0xf1, 0x7e, + 0x1a, 0x8b, 0x86, 0xfa, 0x10, 0xa0, 0xc5, 0x7e, 0x34, 0x1d, 0xdf, 0x9d, 0x20, 0x05, 0x52, 0x87, + 0x64, 0xc2, 0x11, 0x79, 0xcc, 0x7e, 0x32, 0xca, 0x91, 0x6e, 0xaf, 0x26, 0x05, 0xe5, 0x48, 0xb7, + 0xd5, 0x9f, 0x24, 0xa0, 0x50, 0x75, 0x1c, 0xea, 0xf3, 0xde, 0x3d, 0x84, 0x20, 0xed, 0xe8, 0x43, + 0x22, 0x85, 0xf8, 0x6f, 0x54, 0x87, 0xac, 0xad, 0xef, 0x11, 0xdb, 0x5b, 0x4d, 0x96, 0x53, 0xef, + 0x17, 0x1e, 0x7f, 0xad, 0xf2, 0xfa, 0x90, 0x2b, 0x11, 0x25, 0x95, 0x6d, 0x8e, 0xe6, 0x46, 0x60, + 0x29, 0x8a, 0xbe, 0x0d, 0xf3, 0x96, 0x63, 0x5a, 0x06, 0xf1, 0x56, 0xd3, 0x5c, 0xcb, 0x5a, 0x9c, + 0x96, 0xa9, 0xf5, 0xb5, 0xf4, 0x0f, 0xcf, 0xd6, 0xe7, 0x70, 0x20, 0x54, 0xfa, 0x00, 0x0a, 0x11, + 0xb5, 0x31, 0x63, 0x5b, 0x81, 0xcc, 0x91, 0x6e, 0x8f, 0x89, 0x1c, 0x9d, 0x68, 0x7c, 0x98, 0x7c, + 0x9a, 0x50, 0x3f, 0x81, 0x95, 0xb6, 0x3e, 0x24, 0xe6, 0x26, 0x71, 0x88, 0x6b, 0x19, 0x98, 0x78, + 0x74, 0xec, 0x1a, 0x84, 0x8d, 0xf5, 0xd0, 0x72, 0xcc, 0x60, 0xac, 0xec, 0x77, 0xbc, 0x16, 0xb5, + 0x0e, 0x6f, 0x35, 0x2c, 0xcf, 0x70, 0x89, 0x4f, 0x3e, 0xb7, 0x92, 0x54, 0xa0, 0xe4, 0x2c, 0x01, + 0x4b, 0xb3, 0xd2, 0x3f, 0x07, 0x37, 0x98, 0x8b, 0x4d, 0xcd, 0x95, 0x14, 0xcd, 0x1b, 0x11, 0x83, + 0x2b, 0x2b, 0x3c, 0x7e, 0x3f, 0xce, 0x43, 0x71, 0x23, 0xd9, 0x9a, 0xc3, 0xcb, 0x5c, 0x4d, 0x40, + 0xe8, 0x8d, 0x88, 0x81, 0x0c, 0xb8, 0x69, 0x4a, 0xa3, 0x67, 0xd4, 0x27, 0xb9, 0xfa, 0xd8, 0x69, + 0xbc, 0x64, 0x98, 0x5b, 0x73, 0x78, 0x25, 0x50, 0x16, 0xed, 0xa4, 0x06, 0x90, 0x0b, 0x74, 0xab, + 0x3f, 0x48, 0x40, 0x3e, 0x60, 0x7a, 0xe8, 0xab, 0x90, 0x77, 0x74, 0x87, 0x6a, 0xc6, 0x68, 0xec, + 0xf1, 0x01, 0xa5, 0x6a, 0xc5, 0xf3, 0xb3, 0xf5, 0x5c, 0x5b, 0x77, 0x68, 0xbd, 0xbb, 0xeb, 0xe1, + 0x1c, 0x63, 0xd7, 0x47, 0x63, 0x0f, 0xbd, 0x0d, 0xc5, 0x21, 0x19, 0x52, 0x77, 0xa2, 0xed, 0x4d, + 0x7c, 0xe2, 0x49, 0xb7, 0x15, 0x04, 0xad, 0xc6, 0x48, 0xe8, 0x5b, 0x30, 0x3f, 0x10, 0x26, 0xad, + 0xa6, 0xf8, 0xf2, 0x79, 0x27, 0xce, 0xfa, 0x19, 0xab, 0x71, 0x20, 0xa3, 0xfe, 0x56, 0x02, 0x56, + 0x42, 0x2a, 0xf9, 0xc5, 0xb1, 0xe5, 0x92, 0x21, 0x71, 0x7c, 0x0f, 0x7d, 0x03, 0xb2, 0xb6, 0x35, + 0xb4, 0x7c, 0x4f, 0xfa, 0xfc, 0x4e, 0x9c, 0xda, 0x70, 0x50, 0x58, 0x82, 0x51, 0x15, 0x8a, 0x2e, + 0xf1, 0x88, 0x7b, 0x24, 0x56, 0xbc, 0xf4, 0xe8, 0x35, 0xc2, 0x17, 0x44, 0xd4, 0x0d, 0xc8, 0x75, + 0x6d, 0xdd, 0xdf, 0xa7, 0xee, 0x10, 0xa9, 0x50, 0xd4, 0x5d, 0xe3, 0xc0, 0xf2, 0x89, 0xe1, 0x8f, + 0xdd, 0x60, 0xf7, 0x5d, 0xa0, 0xa1, 0x9b, 0x90, 0xa4, 0xa2, 0xa3, 0x7c, 0x2d, 0x7b, 0x7e, 0xb6, + 0x9e, 0xec, 0xf4, 0x70, 0x92, 0x7a, 0xea, 0x47, 0xb0, 0xdc, 0xb5, 0xc7, 0x03, 0xcb, 0x69, 0x10, + 0xcf, 0x70, 0xad, 0x11, 0xd3, 0xce, 0x56, 0x25, 0x8b, 0x51, 0xc1, 0xaa, 0x64, 0xbf, 0xc3, 0xad, + 0x9d, 0x9c, 0x6e, 0x6d, 0xf5, 0x37, 0x92, 0xb0, 0xdc, 0x74, 0x06, 0x96, 0x43, 0xa2, 0xd2, 0x77, + 0x61, 0x91, 0x70, 0xa2, 0x76, 0x24, 0xc2, 0x8d, 0xd4, 0xb3, 0x20, 0xa8, 0x41, 0x0c, 0x6a, 0xcd, + 0xc4, 0x85, 0x47, 0x71, 0xc3, 0x7f, 0x4d, 0x7b, 0x6c, 0x74, 0x68, 0xc2, 0xfc, 0x88, 0x0f, 0xc2, + 0x93, 0xd3, 0x7b, 0x37, 0x4e, 0xd7, 0x6b, 0xe3, 0x0c, 0x82, 0x84, 0x94, 0xfd, 0x22, 0x41, 0xe2, + 0x4f, 0x92, 0xb0, 0xd4, 0xa6, 0xe6, 0x05, 0x3f, 0x94, 0x20, 0x77, 0x40, 0x3d, 0x3f, 0x12, 0x10, + 0xc3, 0x36, 0x7a, 0x0a, 0xb9, 0x91, 0x9c, 0x3e, 0x39, 0xfb, 0xb7, 0xe3, 0x4d, 0x16, 0x18, 0x1c, + 0xa2, 0xd1, 0x47, 0x90, 0x0f, 0xb6, 0x0c, 0x1b, 0xed, 0x1b, 0x2c, 0x9c, 0x29, 0x1e, 0x7d, 0x0b, + 0xb2, 0x62, 0x12, 0x56, 0xd3, 0x5c, 0xf2, 0xee, 0x1b, 0xf9, 0x1c, 0x4b, 0x21, 0xb4, 0x09, 0x39, + 0xdf, 0xf6, 0x34, 0xcb, 0xd9, 0xa7, 0xab, 0x19, 0xae, 0x60, 0x3d, 0x36, 0xc8, 0x50, 0x93, 0xf4, + 0xb7, 0x7b, 0x2d, 0x67, 0x9f, 0xd6, 0x0a, 0xe7, 0x67, 0xeb, 0xf3, 0xb2, 0x81, 0xe7, 0x7d, 0xdb, + 0x63, 0x3f, 0xd4, 0xdf, 0x4e, 0x40, 0x21, 0x82, 0x42, 0x77, 0x00, 0x7c, 0x77, 0xec, 0xf9, 0x9a, + 0x4b, 0xa9, 0xcf, 0x9d, 0x55, 0xc4, 0x79, 0x4e, 0xc1, 0x94, 0xfa, 0xa8, 0x02, 0x37, 0x0c, 0xe2, + 0xfa, 0x9a, 0xe5, 0x79, 0x63, 0xe2, 0x6a, 0xde, 0x78, 0xef, 0x53, 0x62, 0xf8, 0xdc, 0x71, 0x45, + 0xbc, 0xcc, 0x58, 0x2d, 0xce, 0xe9, 0x09, 0x06, 0x7a, 0x02, 0x37, 0xa3, 0xf8, 0xd1, 0x78, 0xcf, + 0xb6, 0x0c, 0x8d, 0x4d, 0x66, 0x8a, 0x8b, 0xdc, 0x98, 0x8a, 0x74, 0x39, 0xef, 0x19, 0x99, 0xa8, + 0x3f, 0x4e, 0x80, 0x82, 0xf5, 0x7d, 0x7f, 0x87, 0x0c, 0xf7, 0x88, 0xdb, 0xf3, 0x75, 0x7f, 0xec, + 0xa1, 0x9b, 0x90, 0xb5, 0x89, 0x6e, 0x12, 0x97, 0x1b, 0x95, 0xc3, 0xb2, 0x85, 0x76, 0xd9, 0x0e, + 0xd6, 0x8d, 0x03, 0x7d, 0xcf, 0xb2, 0x2d, 0x7f, 0xc2, 0x4d, 0x59, 0x8c, 0x5f, 0xc2, 0xb3, 0x3a, + 0x2b, 0x38, 0x22, 0x88, 0x2f, 0xa8, 0x41, 0xab, 0x30, 0x3f, 0x24, 0x9e, 0xa7, 0x0f, 0x08, 0xb7, + 0x34, 0x8f, 0x83, 0xa6, 0xfa, 0x11, 0x14, 0xa3, 0x72, 0xa8, 0x00, 0xf3, 0xbb, 0xed, 0x67, 0xed, + 0xce, 0x8b, 0xb6, 0x32, 0x87, 0x96, 0xa0, 0xb0, 0xdb, 0xc6, 0xcd, 0x6a, 0x7d, 0xab, 0x5a, 0xdb, + 0x6e, 0x2a, 0x09, 0xb4, 0x00, 0xf9, 0x69, 0x33, 0xa9, 0xfe, 0x59, 0x02, 0x80, 0xb9, 0x5b, 0x0e, + 0xea, 0x43, 0xc8, 0x78, 0xbe, 0xee, 0x8b, 0x55, 0xb9, 0xf8, 0xf8, 0xdd, 0xcb, 0xe6, 0x50, 0xda, + 0xcb, 0xfe, 0x11, 0x2c, 0x44, 0xa2, 0x16, 0x26, 0x2f, 0x58, 0xc8, 0x02, 0x84, 0x6e, 0x9a, 0xae, + 0x34, 0x9c, 0xff, 0x56, 0x3f, 0x82, 0x0c, 0x97, 0xbe, 0x68, 0x6e, 0x0e, 0xd2, 0x0d, 0xf6, 0x2b, + 0x81, 0xf2, 0x90, 0xc1, 0xcd, 0x6a, 0xe3, 0x7b, 0x4a, 0x12, 0x29, 0x50, 0x6c, 0xb4, 0x7a, 0xf5, + 0x4e, 0xbb, 0xdd, 0xac, 0xf7, 0x9b, 0x0d, 0x25, 0xa5, 0xde, 0x85, 0x4c, 0x6b, 0xc8, 0x34, 0xdf, + 0x66, 0x4b, 0x7e, 0x9f, 0xb8, 0xc4, 0x31, 0x82, 0x9d, 0x34, 0x25, 0xa8, 0x3f, 0x2d, 0x40, 0x66, + 0x87, 0x8e, 0x1d, 0x1f, 0x3d, 0x8e, 0x84, 0xad, 0xc5, 0xf8, 0x0c, 0x81, 0x03, 0x2b, 0xfd, 0xc9, + 0x88, 0xc8, 0xb0, 0x76, 0x13, 0xb2, 0x62, 0x73, 0xc8, 0xe1, 0xc8, 0x16, 0xa3, 0xfb, 0xba, 0x3b, + 0x20, 0xbe, 0x1c, 0x8f, 0x6c, 0xa1, 0xf7, 0xd9, 0x89, 0xa5, 0x9b, 0xd4, 0xb1, 0x27, 0x7c, 0x0f, + 0xe5, 0xc4, 0xb1, 0x84, 0x89, 0x6e, 0x76, 0x1c, 0x7b, 0x82, 0x43, 0x2e, 0xda, 0x82, 0xe2, 0x9e, + 0xe5, 0x98, 0x1a, 0x1d, 0x89, 0x20, 0x9f, 0xb9, 0x7c, 0xc7, 0x09, 0xab, 0x6a, 0x96, 0x63, 0x76, + 0x04, 0x18, 0x17, 0xf6, 0xa6, 0x0d, 0xd4, 0x86, 0xc5, 0x23, 0x6a, 0x8f, 0x87, 0x24, 0xd4, 0x95, + 0xe5, 0xba, 0xde, 0xbb, 0x5c, 0xd7, 0x73, 0x8e, 0x0f, 0xb4, 0x2d, 0x1c, 0x45, 0x9b, 0xe8, 0x19, + 0x2c, 0xf8, 0xc3, 0xd1, 0xbe, 0x17, 0xaa, 0x9b, 0xe7, 0xea, 0xbe, 0x72, 0x85, 0xc3, 0x18, 0x3c, + 0xd0, 0x56, 0xf4, 0x23, 0x2d, 0xb4, 0x09, 0x05, 0x83, 0x3a, 0x9e, 0xe5, 0xf9, 0xc4, 0x31, 0x26, + 0xab, 0x39, 0xee, 0xfb, 0x2b, 0x46, 0x59, 0x9f, 0x82, 0x71, 0x54, 0xb2, 0xf4, 0x6b, 0x29, 0x28, + 0x44, 0x5c, 0x80, 0x7a, 0x50, 0x18, 0xb9, 0x74, 0xa4, 0x0f, 0xf8, 0x89, 0x27, 0x27, 0xf5, 0xd1, + 0x1b, 0xb9, 0xaf, 0xd2, 0x9d, 0x0a, 0xe2, 0xa8, 0x16, 0xf5, 0x34, 0x09, 0x85, 0x08, 0x13, 0xdd, + 0x83, 0x1c, 0xee, 0xe2, 0xd6, 0xf3, 0x6a, 0xbf, 0xa9, 0xcc, 0x95, 0x6e, 0x9f, 0x9c, 0x96, 0x57, + 0xb9, 0xb6, 0xa8, 0x82, 0xae, 0x6b, 0x1d, 0xb1, 0x35, 0xfc, 0x3e, 0xcc, 0x07, 0xd0, 0x44, 0xe9, + 0xcb, 0x27, 0xa7, 0xe5, 0xb7, 0x66, 0xa1, 0x11, 0x24, 0xee, 0x6d, 0x55, 0x71, 0xb3, 0xa1, 0x24, + 0xe3, 0x91, 0xb8, 0x77, 0xa0, 0xbb, 0xc4, 0x44, 0x5f, 0x81, 0xac, 0x04, 0xa6, 0x4a, 0xa5, 0x93, + 0xd3, 0xf2, 0xcd, 0x59, 0xe0, 0x14, 0x87, 0x7b, 0xdb, 0xd5, 0xe7, 0x4d, 0x25, 0x1d, 0x8f, 0xc3, + 0x3d, 0x5b, 0x3f, 0x22, 0xe8, 0x5d, 0xc8, 0x08, 0x58, 0xa6, 0x74, 0xeb, 0xe4, 0xb4, 0xfc, 0xa5, + 0xd7, 0xd4, 0x31, 0x54, 0x69, 0xf5, 0x37, 0xff, 0x70, 0x6d, 0xee, 0xaf, 0xfe, 0x68, 0x4d, 0x99, + 0x65, 0x97, 0xfe, 0x3b, 0x01, 0x0b, 0x17, 0xd6, 0x0e, 0x52, 0x21, 0xeb, 0x50, 0x83, 0x8e, 0xc4, + 0x41, 0x98, 0xab, 0xc1, 0xf9, 0xd9, 0x7a, 0xb6, 0x4d, 0xeb, 0x74, 0x34, 0xc1, 0x92, 0x83, 0x9e, + 0xcd, 0x1c, 0xe5, 0x4f, 0xde, 0x70, 0x61, 0xc6, 0x1e, 0xe6, 0x1f, 0xc3, 0x82, 0xe9, 0x5a, 0x47, + 0xc4, 0xd5, 0x0c, 0xea, 0xec, 0x5b, 0x03, 0x79, 0xc8, 0x95, 0x62, 0xf3, 0x4d, 0x0e, 0xc4, 0x45, + 0x21, 0x50, 0xe7, 0xf8, 0x2f, 0x70, 0x8c, 0x97, 0x9e, 0x43, 0x31, 0xba, 0xd4, 0xd9, 0xb9, 0xe4, + 0x59, 0xbf, 0x44, 0x64, 0x62, 0xc9, 0xd3, 0x50, 0x9c, 0x67, 0x14, 0x91, 0x56, 0xbe, 0x07, 0xe9, + 0x21, 0x35, 0x85, 0x9e, 0x85, 0xda, 0x0d, 0x96, 0x4d, 0xfc, 0xd3, 0xd9, 0x7a, 0x81, 0x7a, 0x95, + 0x0d, 0xcb, 0x26, 0x3b, 0xd4, 0x24, 0x98, 0x03, 0xd4, 0x23, 0x48, 0xb3, 0x98, 0x83, 0xbe, 0x0c, + 0xe9, 0x5a, 0xab, 0xdd, 0x50, 0xe6, 0x4a, 0xcb, 0x27, 0xa7, 0xe5, 0x05, 0xee, 0x12, 0xc6, 0x60, + 0x6b, 0x17, 0xad, 0x43, 0xf6, 0x79, 0x67, 0x7b, 0x77, 0x87, 0x2d, 0xaf, 0x1b, 0x27, 0xa7, 0xe5, + 0xa5, 0x90, 0x2d, 0x9c, 0x86, 0xee, 0x40, 0xa6, 0xbf, 0xd3, 0xdd, 0xe8, 0x29, 0xc9, 0x12, 0x3a, + 0x39, 0x2d, 0x2f, 0x86, 0x7c, 0x6e, 0x73, 0x69, 0x59, 0xce, 0x6a, 0x3e, 0xa4, 0xab, 0x3f, 0x4a, + 0x40, 0x21, 0xb2, 0xe1, 0xd8, 0xc2, 0x6c, 0x34, 0x37, 0xaa, 0xbb, 0xdb, 0x7d, 0x65, 0x2e, 0xb2, + 0x30, 0x23, 0x90, 0x06, 0xd9, 0xd7, 0xc7, 0x36, 0x8b, 0x73, 0x50, 0xef, 0xb4, 0x7b, 0xad, 0x5e, + 0xbf, 0xd9, 0xee, 0x2b, 0x89, 0xd2, 0xea, 0xc9, 0x69, 0x79, 0x65, 0x16, 0xbc, 0x31, 0xb6, 0x6d, + 0xb6, 0x34, 0xeb, 0xd5, 0xfa, 0x16, 0x5f, 0xeb, 0xd3, 0xa5, 0x19, 0x41, 0xd5, 0x75, 0xe3, 0x80, + 0x98, 0xe8, 0x3e, 0xe4, 0x1b, 0xcd, 0xed, 0xe6, 0x66, 0x95, 0x47, 0xf7, 0xd2, 0x9d, 0x93, 0xd3, + 0xf2, 0xad, 0xd7, 0x7b, 0xb7, 0xc9, 0x40, 0xf7, 0x89, 0x39, 0xb3, 0x44, 0x23, 0x10, 0xf5, 0x3f, + 0x93, 0xb0, 0x80, 0x59, 0x39, 0xec, 0xfa, 0x5d, 0x6a, 0x5b, 0xc6, 0x04, 0x75, 0x21, 0x6f, 0x50, + 0xc7, 0xb4, 0x22, 0x71, 0xe2, 0xf1, 0x25, 0x29, 0xd1, 0x54, 0x2a, 0x68, 0xd5, 0x03, 0x49, 0x3c, + 0x55, 0x82, 0x1e, 0x40, 0xc6, 0x24, 0xb6, 0x3e, 0x91, 0xb9, 0xd9, 0xad, 0x8a, 0x28, 0xb8, 0x2b, + 0x41, 0xc1, 0x5d, 0x69, 0xc8, 0x82, 0x1b, 0x0b, 0x1c, 0xaf, 0x41, 0xf4, 0x97, 0x9a, 0xee, 0xfb, + 0x64, 0x38, 0xf2, 0x45, 0x62, 0x96, 0xc6, 0x85, 0xa1, 0xfe, 0xb2, 0x2a, 0x49, 0xe8, 0x11, 0x64, + 0x8f, 0x2d, 0xc7, 0xa4, 0xc7, 0x32, 0xf7, 0xba, 0x42, 0xa9, 0x04, 0xaa, 0x27, 0x2c, 0x25, 0x99, + 0x31, 0x93, 0xad, 0xa1, 0x76, 0xa7, 0xdd, 0x0c, 0xd6, 0x90, 0xe4, 0x77, 0x9c, 0x36, 0x75, 0xd8, + 0xfe, 0x87, 0x4e, 0x5b, 0xdb, 0xa8, 0xb6, 0xb6, 0x77, 0x31, 0x5b, 0x47, 0x2b, 0x27, 0xa7, 0x65, + 0x25, 0x84, 0x6c, 0xe8, 0x96, 0xcd, 0x8a, 0x81, 0x5b, 0x90, 0xaa, 0xb6, 0xbf, 0xa7, 0x24, 0x4b, + 0xca, 0xc9, 0x69, 0xb9, 0x18, 0xb2, 0xab, 0xce, 0x64, 0xea, 0xf7, 0xd9, 0x7e, 0xd5, 0xbf, 0x4d, + 0x41, 0x71, 0x77, 0x64, 0xea, 0x3e, 0x11, 0xfb, 0x0c, 0x95, 0xa1, 0x30, 0xd2, 0x5d, 0xdd, 0xb6, + 0x89, 0x6d, 0x79, 0x43, 0x79, 0x95, 0x10, 0x25, 0xa1, 0x0f, 0xde, 0xd4, 0x8d, 0xb5, 0x1c, 0xdb, + 0x3b, 0x3f, 0xf8, 0x97, 0xf5, 0x44, 0xe0, 0xd0, 0x5d, 0x58, 0xdc, 0x17, 0xd6, 0x6a, 0xba, 0xc1, + 0x27, 0x36, 0xc5, 0x27, 0xb6, 0x12, 0x37, 0xb1, 0x51, 0xb3, 0x2a, 0x72, 0x90, 0x55, 0x2e, 0x85, + 0x17, 0xf6, 0xa3, 0x4d, 0xf4, 0x04, 0xe6, 0x87, 0xd4, 0xb1, 0x7c, 0xea, 0x5e, 0x3f, 0x0b, 0x01, + 0x12, 0xdd, 0x83, 0x65, 0x36, 0xb9, 0x81, 0x3d, 0x9c, 0xcd, 0x8f, 0xf3, 0x24, 0x5e, 0x1a, 0xea, + 0x2f, 0x65, 0x87, 0x98, 0x91, 0x51, 0x0d, 0x32, 0xd4, 0x65, 0xf9, 0x62, 0x96, 0x9b, 0x7b, 0xff, + 0x5a, 0x73, 0x45, 0xa3, 0xc3, 0x64, 0xb0, 0x10, 0x55, 0xbf, 0x09, 0x0b, 0x17, 0x06, 0xc1, 0xd2, + 0xa4, 0x6e, 0x75, 0xb7, 0xd7, 0x54, 0xe6, 0x50, 0x11, 0x72, 0xf5, 0x4e, 0xbb, 0xdf, 0x6a, 0xef, + 0xb2, 0x3c, 0xaf, 0x08, 0x39, 0xdc, 0xd9, 0xde, 0xae, 0x55, 0xeb, 0xcf, 0x94, 0xa4, 0x5a, 0x81, + 0x42, 0x44, 0x1b, 0x5a, 0x04, 0xe8, 0xf5, 0x3b, 0x5d, 0x6d, 0xa3, 0x85, 0x7b, 0x7d, 0x91, 0x25, + 0xf6, 0xfa, 0x55, 0xdc, 0x97, 0x84, 0x84, 0xfa, 0xef, 0xc9, 0x60, 0x46, 0x65, 0x62, 0x58, 0xbb, + 0x98, 0x18, 0x5e, 0x61, 0xbc, 0x4c, 0x0d, 0xa7, 0x8d, 0x30, 0x41, 0xfc, 0x00, 0x80, 0x2f, 0x1c, + 0x62, 0x6a, 0xba, 0x2f, 0x27, 0xbe, 0xf4, 0x9a, 0x93, 0xfb, 0xc1, 0x8d, 0x16, 0xce, 0x4b, 0x74, + 0xd5, 0x47, 0xdf, 0x82, 0xa2, 0x41, 0x87, 0x23, 0x9b, 0x48, 0xe1, 0xd4, 0xb5, 0xc2, 0x85, 0x10, + 0x5f, 0xf5, 0xa3, 0xa9, 0x69, 0xfa, 0x62, 0xf2, 0xfc, 0xeb, 0x89, 0xc0, 0x33, 0x31, 0xd9, 0x68, + 0x11, 0x72, 0xbb, 0xdd, 0x46, 0xb5, 0xdf, 0x6a, 0x6f, 0x2a, 0x09, 0x04, 0x90, 0xe5, 0xae, 0x6e, + 0x28, 0x49, 0x96, 0x45, 0xd7, 0x3b, 0x3b, 0xdd, 0xed, 0x26, 0x8f, 0x58, 0x68, 0x05, 0x94, 0xc0, + 0xd9, 0x1a, 0x77, 0x64, 0xb3, 0xa1, 0xa4, 0xd1, 0x0d, 0x58, 0x0a, 0xa9, 0x52, 0x32, 0x83, 0x6e, + 0x02, 0x0a, 0x89, 0x53, 0x15, 0x59, 0xf5, 0x57, 0x60, 0xa9, 0x4e, 0x1d, 0x5f, 0xb7, 0x9c, 0xb0, + 0xc2, 0x78, 0xcc, 0x06, 0x2d, 0x49, 0x9a, 0x25, 0x6f, 0x82, 0x6a, 0x4b, 0xe7, 0x67, 0xeb, 0x85, + 0x10, 0xda, 0x6a, 0xf0, 0x54, 0x49, 0x36, 0x4c, 0xb6, 0x7f, 0x47, 0x96, 0xc9, 0x9d, 0x9b, 0xa9, + 0xcd, 0x9f, 0x9f, 0xad, 0xa7, 0xba, 0xad, 0x06, 0x66, 0x34, 0xf4, 0x65, 0xc8, 0x93, 0x97, 0x96, + 0xaf, 0x19, 0xec, 0x5c, 0x62, 0x0e, 0xcc, 0xe0, 0x1c, 0x23, 0xd4, 0xd9, 0x31, 0x54, 0x03, 0xe8, + 0x52, 0xd7, 0x97, 0x3d, 0x7f, 0x1d, 0x32, 0x23, 0xea, 0xf2, 0xbb, 0x8b, 0x4b, 0x6f, 0xd4, 0x18, + 0x5c, 0x2c, 0x54, 0x2c, 0xc0, 0xea, 0xef, 0xa6, 0x00, 0xfa, 0xba, 0x77, 0x28, 0x95, 0x3c, 0x85, + 0x7c, 0x78, 0x3b, 0x29, 0x2f, 0x41, 0xae, 0x9c, 0xed, 0x10, 0x8c, 0x9e, 0x04, 0x8b, 0x4d, 0xd4, + 0x4e, 0xb1, 0x45, 0x6c, 0xd0, 0x51, 0x5c, 0xf9, 0x71, 0xb1, 0x40, 0x62, 0xc7, 0x3c, 0x71, 0x5d, + 0x39, 0xf3, 0xec, 0x27, 0xaa, 0xf3, 0x63, 0x41, 0x38, 0x4d, 0x66, 0xdf, 0xb1, 0xd7, 0x3e, 0x33, + 0x33, 0xb2, 0x35, 0x87, 0xa7, 0x72, 0xe8, 0x63, 0x28, 0xb0, 0x71, 0x6b, 0x1e, 0xe7, 0xc9, 0xc4, + 0xfb, 0x52, 0x57, 0x09, 0x0d, 0x18, 0x46, 0x53, 0x2f, 0xdf, 0x01, 0xd0, 0x47, 0x23, 0xdb, 0x22, + 0xa6, 0xb6, 0x37, 0xe1, 0x99, 0x76, 0x1e, 0xe7, 0x25, 0xa5, 0x36, 0x61, 0xdb, 0x25, 0x60, 0xeb, + 0x3e, 0xcf, 0x9e, 0xaf, 0x71, 0xa0, 0x44, 0x57, 0xfd, 0x9a, 0x02, 0x8b, 0xee, 0xd8, 0x61, 0x0e, + 0x95, 0xd6, 0xa9, 0x7f, 0x9a, 0x84, 0xb7, 0xda, 0xc4, 0x3f, 0xa6, 0xee, 0x61, 0xd5, 0xf7, 0x75, + 0xe3, 0x60, 0x48, 0x1c, 0x39, 0x7d, 0x91, 0x82, 0x26, 0x71, 0xa1, 0xa0, 0x59, 0x85, 0x79, 0xdd, + 0xb6, 0x74, 0x8f, 0x88, 0xe4, 0x2d, 0x8f, 0x83, 0x26, 0x2b, 0xbb, 0x58, 0x11, 0x47, 0x3c, 0x8f, + 0x88, 0x7b, 0x15, 0x66, 0x78, 0x40, 0x40, 0xdf, 0x87, 0x9b, 0x32, 0x4d, 0xd3, 0xc3, 0xae, 0x58, + 0x41, 0x11, 0x5c, 0xd0, 0x36, 0x63, 0xab, 0xca, 0x78, 0xe3, 0x64, 0x1e, 0x37, 0x25, 0x77, 0x46, + 0xbe, 0xcc, 0x0a, 0x57, 0xcc, 0x18, 0x56, 0x69, 0x13, 0x6e, 0x5d, 0x2a, 0xf2, 0xb9, 0xee, 0x6d, + 0xfe, 0x21, 0x09, 0xd0, 0xea, 0x56, 0x77, 0xa4, 0x93, 0x1a, 0x90, 0xdd, 0xd7, 0x87, 0x96, 0x3d, + 0xb9, 0x2a, 0x02, 0x4e, 0xf1, 0x95, 0xaa, 0x70, 0xc7, 0x06, 0x97, 0xc1, 0x52, 0x96, 0xd7, 0x94, + 0xe3, 0x3d, 0x87, 0xf8, 0x61, 0x4d, 0xc9, 0x5b, 0xcc, 0x0c, 0x57, 0x77, 0xc2, 0xa5, 0x2b, 0x1a, + 0x6c, 0x02, 0x58, 0xca, 0x73, 0xac, 0x4f, 0x82, 0xb0, 0x25, 0x9b, 0x68, 0x8b, 0xdf, 0x8e, 0x12, + 0xf7, 0x88, 0x98, 0xab, 0x19, 0xee, 0xd4, 0xeb, 0xec, 0xc1, 0x12, 0x2e, 0x7c, 0x17, 0x4a, 0x97, + 0x3e, 0xe2, 0x29, 0xd3, 0x94, 0xf5, 0xb9, 0x7c, 0xf4, 0x10, 0x16, 0x2e, 0x8c, 0xf3, 0xb5, 0x62, + 0xbe, 0xd5, 0x7d, 0xfe, 0x75, 0x25, 0x2d, 0x7f, 0x7d, 0x53, 0xc9, 0xaa, 0x7f, 0x9c, 0x12, 0x81, + 0x46, 0x7a, 0x35, 0xfe, 0x55, 0x20, 0xc7, 0x57, 0xb7, 0x41, 0x6d, 0x19, 0x00, 0xde, 0xbb, 0x3a, + 0xfe, 0xb0, 0x9a, 0x8e, 0xc3, 0x71, 0x28, 0x88, 0xd6, 0xa1, 0x20, 0x56, 0xb1, 0xc6, 0x36, 0x1c, + 0x77, 0xeb, 0x02, 0x06, 0x41, 0x62, 0x92, 0xe8, 0x2e, 0x2c, 0xf2, 0xcb, 0x1f, 0xef, 0x80, 0x98, + 0x02, 0x93, 0xe6, 0x98, 0x85, 0x90, 0xca, 0x61, 0x3b, 0x50, 0x94, 0x04, 0x8d, 0xe7, 0xf3, 0x19, + 0x6e, 0xd0, 0xbd, 0xeb, 0x0c, 0x12, 0x22, 0x3c, 0xcd, 0x2f, 0x8c, 0xa6, 0x0d, 0xb5, 0x01, 0xb9, + 0xc0, 0x58, 0xb4, 0x0a, 0xa9, 0x7e, 0xbd, 0xab, 0xcc, 0x95, 0x96, 0x4e, 0x4e, 0xcb, 0x85, 0x80, + 0xdc, 0xaf, 0x77, 0x19, 0x67, 0xb7, 0xd1, 0x55, 0x12, 0x17, 0x39, 0xbb, 0x8d, 0x6e, 0x29, 0xcd, + 0x72, 0x30, 0x75, 0x1f, 0x0a, 0x91, 0x1e, 0xd0, 0x3b, 0x30, 0xdf, 0x6a, 0x6f, 0xe2, 0x66, 0xaf, + 0xa7, 0xcc, 0x95, 0x6e, 0x9e, 0x9c, 0x96, 0x51, 0x84, 0xdb, 0x72, 0x06, 0x6c, 0x7e, 0xd0, 0x1d, + 0x48, 0x6f, 0x75, 0xd8, 0xd9, 0x2e, 0x0a, 0x88, 0x08, 0x62, 0x8b, 0x7a, 0x7e, 0xe9, 0x86, 0x4c, + 0xee, 0xa2, 0x8a, 0xd5, 0xdf, 0x4b, 0x40, 0x56, 0x6c, 0xa6, 0xd8, 0x89, 0xaa, 0xc2, 0x7c, 0x70, + 0x4d, 0x20, 0x8a, 0xbb, 0xf7, 0x2e, 0x2f, 0xc4, 0x2a, 0xb2, 0x6e, 0x12, 0xcb, 0x2f, 0x90, 0x2b, + 0x7d, 0x08, 0xc5, 0x28, 0xe3, 0x73, 0x2d, 0xbe, 0xef, 0x43, 0x81, 0xad, 0xef, 0xa0, 0x20, 0x7b, + 0x0c, 0x59, 0x11, 0x10, 0xc2, 0xb3, 0xe6, 0xf2, 0xaa, 0x50, 0x22, 0xd1, 0x53, 0x98, 0x17, 0x95, + 0x64, 0x70, 0x3b, 0xbc, 0x76, 0xf5, 0x2e, 0xc2, 0x01, 0x5c, 0xfd, 0x18, 0xd2, 0x5d, 0x42, 0x5c, + 0xe6, 0x7b, 0x87, 0x9a, 0x64, 0x7a, 0x3c, 0xcb, 0x22, 0xd8, 0x24, 0xad, 0x06, 0x2b, 0x82, 0x4d, + 0xd2, 0x32, 0xc3, 0xfb, 0xaf, 0x64, 0xe4, 0xfe, 0xab, 0x0f, 0xc5, 0x17, 0xc4, 0x1a, 0x1c, 0xf8, + 0xc4, 0xe4, 0x8a, 0xee, 0x43, 0x7a, 0x44, 0x42, 0xe3, 0x57, 0x63, 0x17, 0x18, 0x21, 0x2e, 0xe6, + 0x28, 0x16, 0x47, 0x8e, 0xb9, 0xb4, 0x7c, 0xd2, 0x90, 0x2d, 0xf5, 0xef, 0x93, 0xb0, 0xd8, 0xf2, + 0xbc, 0xb1, 0xee, 0x18, 0x41, 0xe6, 0xf6, 0xed, 0x8b, 0x99, 0x5b, 0xec, 0xdb, 0xcf, 0x45, 0x91, + 0x8b, 0xd7, 0x7a, 0xf2, 0xf4, 0x4c, 0x86, 0xa7, 0xa7, 0xfa, 0xd3, 0x44, 0x70, 0x77, 0x77, 0x37, + 0xb2, 0xdd, 0x45, 0x1d, 0x18, 0xd5, 0x44, 0x76, 0x9d, 0x43, 0x87, 0x1e, 0x3b, 0xe8, 0x6d, 0xc8, + 0xe0, 0x66, 0xbb, 0xf9, 0x42, 0x49, 0x88, 0xe5, 0x79, 0x01, 0x84, 0x89, 0x43, 0x8e, 0x99, 0xa6, + 0x6e, 0xb3, 0xdd, 0x60, 0x99, 0x56, 0x32, 0x46, 0x53, 0x97, 0x38, 0xa6, 0xe5, 0x0c, 0xd0, 0x3b, + 0x90, 0x6d, 0xf5, 0x7a, 0xbb, 0xbc, 0x4c, 0x7c, 0xeb, 0xe4, 0xb4, 0x7c, 0xe3, 0x02, 0x8a, 0xdf, + 0xdb, 0x9a, 0x0c, 0xc4, 0xca, 0x1c, 0x96, 0x83, 0xc5, 0x80, 0x58, 0xfe, 0x2c, 0x40, 0xb8, 0xd3, + 0xaf, 0xf6, 0x9b, 0x4a, 0x26, 0x06, 0x84, 0x29, 0xfb, 0x2b, 0xb7, 0xdb, 0x3f, 0x27, 0x41, 0xa9, + 0x1a, 0x06, 0x19, 0xf9, 0x8c, 0x2f, 0x2b, 0xcb, 0x3e, 0xe4, 0x46, 0xec, 0x97, 0x45, 0x82, 0x2c, + 0xe9, 0x69, 0xec, 0xeb, 0xe5, 0x8c, 0x5c, 0x05, 0x53, 0x9b, 0x54, 0xcd, 0xa1, 0xe5, 0x79, 0x16, + 0x75, 0x04, 0x0d, 0x87, 0x9a, 0x4a, 0xff, 0x91, 0x80, 0x1b, 0x31, 0x08, 0xf4, 0x10, 0xd2, 0x2e, + 0xb5, 0x83, 0x39, 0xbc, 0x7d, 0xd9, 0xb5, 0x2c, 0x13, 0xc5, 0x1c, 0x89, 0xd6, 0x00, 0xf4, 0xb1, + 0x4f, 0x75, 0xde, 0x3f, 0x9f, 0xbd, 0x1c, 0x8e, 0x50, 0xd0, 0x0b, 0xc8, 0x7a, 0xc4, 0x70, 0x49, + 0x90, 0x4b, 0x7f, 0xfc, 0x7f, 0xb5, 0xbe, 0xd2, 0xe3, 0x6a, 0xb0, 0x54, 0x57, 0xaa, 0x40, 0x56, + 0x50, 0xd8, 0xb2, 0x37, 0x75, 0x5f, 0x97, 0x97, 0xf6, 0xfc, 0x37, 0x5b, 0x4d, 0xba, 0x3d, 0x08, + 0x56, 0x93, 0x6e, 0x0f, 0xd4, 0xbf, 0x49, 0x02, 0x34, 0x5f, 0xfa, 0xc4, 0x75, 0x74, 0xbb, 0x5e, + 0x45, 0xcd, 0x48, 0xf4, 0x17, 0xa3, 0xfd, 0x6a, 0xec, 0x4b, 0x44, 0x28, 0x51, 0xa9, 0x57, 0x63, + 0xe2, 0xff, 0x2d, 0x48, 0x8d, 0x5d, 0xf9, 0x20, 0x2d, 0xf2, 0xe0, 0x5d, 0xbc, 0x8d, 0x19, 0x0d, + 0x35, 0xa7, 0x61, 0x2b, 0x75, 0xf9, 0xb3, 0x73, 0xa4, 0x83, 0xd8, 0xd0, 0xc5, 0x76, 0xbe, 0xa1, + 0x6b, 0x06, 0x91, 0x27, 0x47, 0x51, 0xec, 0xfc, 0x7a, 0xb5, 0x4e, 0x5c, 0x1f, 0x67, 0x0d, 0x9d, + 0xfd, 0xff, 0x42, 0xf1, 0xed, 0x3e, 0xc0, 0x74, 0x68, 0x68, 0x0d, 0x32, 0xf5, 0x8d, 0x5e, 0x6f, + 0x5b, 0x99, 0x13, 0x01, 0x7c, 0xca, 0xe2, 0x64, 0xf5, 0x2f, 0x93, 0x90, 0xab, 0x57, 0xe5, 0xb1, + 0x5a, 0x07, 0x85, 0x47, 0x25, 0xfe, 0xd4, 0x41, 0x5e, 0x8e, 0x2c, 0x77, 0x22, 0x03, 0xcb, 0x15, + 0x45, 0xed, 0x22, 0x13, 0x61, 0x56, 0x37, 0xb9, 0x00, 0xc2, 0x50, 0x24, 0xd2, 0x09, 0x9a, 0xa1, + 0x07, 0x31, 0x7e, 0xed, 0x6a, 0x67, 0x89, 0xf2, 0x64, 0xda, 0xf6, 0x70, 0x21, 0x50, 0x52, 0xd7, + 0x3d, 0xf4, 0x01, 0x2c, 0x79, 0xd6, 0xc0, 0xb1, 0x9c, 0x81, 0x16, 0x38, 0x8f, 0xbf, 0xbb, 0xd4, + 0x96, 0xcf, 0xcf, 0xd6, 0x17, 0x7a, 0x82, 0x25, 0x7d, 0xb8, 0x20, 0x91, 0x75, 0xee, 0x4a, 0xf4, + 0x4d, 0x58, 0x8c, 0x88, 0x32, 0x2f, 0x0a, 0xb7, 0x2b, 0xe7, 0x67, 0xeb, 0xc5, 0x50, 0xf2, 0x19, + 0x99, 0xe0, 0x62, 0x28, 0xf8, 0x8c, 0xf0, 0xfb, 0x97, 0x7d, 0xea, 0x1a, 0x44, 0x73, 0xf9, 0x9e, + 0xe6, 0x27, 0x78, 0x1a, 0x17, 0x38, 0x4d, 0x6c, 0x73, 0xf5, 0x39, 0xdc, 0xe8, 0xb8, 0xc6, 0x01, + 0xf1, 0x7c, 0xe1, 0x0a, 0xe9, 0xc5, 0x8f, 0xe1, 0xb6, 0xaf, 0x7b, 0x87, 0xda, 0x81, 0xe5, 0xf9, + 0xd4, 0x9d, 0x68, 0x2e, 0xf1, 0x89, 0xc3, 0xf8, 0x1a, 0x7f, 0xac, 0x95, 0x97, 0x7e, 0xb7, 0x18, + 0x66, 0x4b, 0x40, 0x70, 0x80, 0xd8, 0x66, 0x00, 0xb5, 0x05, 0x45, 0x56, 0xa6, 0xc8, 0x8b, 0x33, + 0x36, 0x7a, 0xb0, 0xe9, 0x40, 0x7b, 0xe3, 0x63, 0x2a, 0x6f, 0xd3, 0x81, 0xf8, 0xa9, 0x7e, 0x17, + 0x94, 0x86, 0xe5, 0x8d, 0x74, 0xdf, 0x38, 0x08, 0x6e, 0x33, 0x51, 0x03, 0x94, 0x03, 0xa2, 0xbb, + 0xfe, 0x1e, 0xd1, 0x7d, 0x6d, 0x44, 0x5c, 0x8b, 0x9a, 0xd7, 0xcf, 0xf2, 0x52, 0x28, 0xd2, 0xe5, + 0x12, 0xea, 0x7f, 0x25, 0x00, 0xb0, 0xbe, 0x1f, 0x64, 0x64, 0x5f, 0x83, 0x65, 0xcf, 0xd1, 0x47, + 0xde, 0x01, 0xf5, 0x35, 0xcb, 0xf1, 0x89, 0x7b, 0xa4, 0xdb, 0xf2, 0x02, 0x47, 0x09, 0x18, 0x2d, + 0x49, 0x47, 0xf7, 0x01, 0x1d, 0x12, 0x32, 0xd2, 0xa8, 0x6d, 0x6a, 0x01, 0x53, 0x3c, 0x25, 0xa7, + 0xb1, 0xc2, 0x38, 0x1d, 0xdb, 0xec, 0x05, 0x74, 0x54, 0x83, 0x35, 0x36, 0x7c, 0xe2, 0xf8, 0xae, + 0x45, 0x3c, 0x6d, 0x9f, 0xba, 0x9a, 0x67, 0xd3, 0x63, 0x6d, 0x9f, 0xda, 0x36, 0x3d, 0x26, 0x6e, + 0x70, 0x37, 0x56, 0xb2, 0xe9, 0xa0, 0x29, 0x40, 0x1b, 0xd4, 0xed, 0xd9, 0xf4, 0x78, 0x23, 0x40, + 0xb0, 0xb4, 0x6d, 0x3a, 0x66, 0xdf, 0x32, 0x0e, 0x83, 0xb4, 0x2d, 0xa4, 0xf6, 0x2d, 0xe3, 0x10, + 0xbd, 0x03, 0x0b, 0xc4, 0x26, 0xfc, 0x8a, 0x44, 0xa0, 0x32, 0x1c, 0x55, 0x0c, 0x88, 0x0c, 0xa4, + 0x7e, 0x02, 0x4a, 0xd3, 0x31, 0xdc, 0xc9, 0x28, 0x32, 0xe7, 0xf7, 0x01, 0xb1, 0x20, 0xa9, 0xd9, + 0xd4, 0x38, 0xd4, 0x86, 0xba, 0xa3, 0x0f, 0x98, 0x5d, 0xe2, 0x85, 0x4f, 0x61, 0x9c, 0x6d, 0x6a, + 0x1c, 0xee, 0x48, 0xba, 0xfa, 0x01, 0x40, 0x6f, 0xe4, 0x12, 0xdd, 0xec, 0xb0, 0x6c, 0x82, 0xb9, + 0x8e, 0xb7, 0x34, 0x53, 0xbe, 0x90, 0x52, 0x57, 0x6e, 0x75, 0x45, 0x30, 0x1a, 0x21, 0x5d, 0xfd, + 0x79, 0xb8, 0xd1, 0xb5, 0x75, 0x83, 0x7f, 0x2d, 0xd0, 0x0d, 0x9f, 0xac, 0xd0, 0x53, 0xc8, 0x0a, + 0xa8, 0x9c, 0xc9, 0xd8, 0xed, 0x36, 0xed, 0x73, 0x6b, 0x0e, 0x4b, 0x7c, 0xad, 0x08, 0x30, 0xd5, + 0xa3, 0xfe, 0x79, 0x02, 0xf2, 0xa1, 0x7e, 0x54, 0x16, 0x2f, 0x31, 0xbe, 0xab, 0x5b, 0x8e, 0xac, + 0xea, 0xf3, 0x38, 0x4a, 0x42, 0x2d, 0x28, 0x8c, 0x42, 0xe9, 0x2b, 0xf3, 0xb9, 0x18, 0xab, 0x71, + 0x54, 0x16, 0x7d, 0x08, 0xf9, 0xe0, 0x49, 0x3a, 0x88, 0xb0, 0x57, 0xbf, 0x60, 0x4f, 0xe1, 0xea, + 0xb7, 0x01, 0xbe, 0x43, 0x2d, 0xa7, 0x4f, 0x0f, 0x89, 0xc3, 0x9f, 0x58, 0x59, 0x4d, 0x48, 0x02, + 0x2f, 0xca, 0x16, 0x2f, 0xf5, 0xc5, 0x14, 0x84, 0x2f, 0x8d, 0xa2, 0xa9, 0xfe, 0x75, 0x12, 0xb2, + 0x98, 0x52, 0xbf, 0x5e, 0x45, 0x65, 0xc8, 0xca, 0x38, 0xc1, 0xcf, 0x9f, 0x5a, 0xfe, 0xfc, 0x6c, + 0x3d, 0x23, 0x02, 0x44, 0xc6, 0xe0, 0x91, 0x21, 0x12, 0xc1, 0x93, 0x97, 0x45, 0x70, 0xf4, 0x10, + 0x8a, 0x12, 0xa4, 0x1d, 0xe8, 0xde, 0x81, 0x28, 0xd0, 0x6a, 0x8b, 0xe7, 0x67, 0xeb, 0x20, 0x90, + 0x5b, 0xba, 0x77, 0x80, 0x41, 0xa0, 0xd9, 0x6f, 0xd4, 0x84, 0xc2, 0xa7, 0xd4, 0x72, 0x34, 0x9f, + 0x0f, 0x42, 0x5e, 0x26, 0xc6, 0xce, 0xe3, 0x74, 0xa8, 0xf2, 0x7b, 0x03, 0xf8, 0x74, 0x3a, 0xf8, + 0x26, 0x2c, 0xb8, 0x94, 0xfa, 0x22, 0x6c, 0x59, 0xd4, 0x91, 0xf7, 0x14, 0xe5, 0xd8, 0xeb, 0x6b, + 0x4a, 0x7d, 0x2c, 0x71, 0xb8, 0xe8, 0x46, 0x5a, 0xe8, 0x21, 0xac, 0xd8, 0xba, 0xe7, 0x6b, 0x3c, + 0xde, 0x99, 0x53, 0x6d, 0x59, 0xbe, 0xd5, 0x10, 0xe3, 0x6d, 0x70, 0x56, 0x20, 0xa1, 0xfe, 0x63, + 0x02, 0x0a, 0x6c, 0x30, 0xd6, 0xbe, 0x65, 0xb0, 0x24, 0xef, 0xf3, 0xe7, 0x1e, 0xb7, 0x20, 0x65, + 0x78, 0xae, 0x74, 0x2a, 0x3f, 0x7c, 0xeb, 0x3d, 0x8c, 0x19, 0x0d, 0x7d, 0x02, 0x59, 0x79, 0x5f, + 0x22, 0xd2, 0x0e, 0xf5, 0xfa, 0x74, 0x54, 0xfa, 0x46, 0xca, 0xf1, 0xb5, 0x3c, 0xb5, 0x4e, 0x1c, + 0x02, 0x38, 0x4a, 0x42, 0x37, 0x21, 0x69, 0x08, 0x77, 0xc9, 0x0f, 0x5a, 0xea, 0x6d, 0x9c, 0x34, + 0x1c, 0xf5, 0x47, 0x09, 0x58, 0x98, 0x6e, 0x78, 0xb6, 0x02, 0x6e, 0x43, 0xde, 0x1b, 0xef, 0x79, + 0x13, 0xcf, 0x27, 0xc3, 0xe0, 0xf9, 0x38, 0x24, 0xa0, 0x16, 0xe4, 0x75, 0x7b, 0x40, 0x5d, 0xcb, + 0x3f, 0x18, 0xca, 0x4a, 0x34, 0x3e, 0x55, 0x88, 0xea, 0xac, 0x54, 0x03, 0x11, 0x3c, 0x95, 0x0e, + 0xce, 0x7d, 0xf1, 0x8d, 0x01, 0x3f, 0xf7, 0xdf, 0x86, 0xa2, 0xad, 0x0f, 0xf9, 0x05, 0x92, 0x6f, + 0x0d, 0xc5, 0x38, 0xd2, 0xb8, 0x20, 0x69, 0x7d, 0x6b, 0x48, 0x54, 0x15, 0xf2, 0xa1, 0x32, 0xb4, + 0x04, 0x85, 0x6a, 0xb3, 0xa7, 0x3d, 0x7a, 0xfc, 0x54, 0xdb, 0xac, 0xef, 0x28, 0x73, 0x32, 0x37, + 0xfd, 0x8b, 0x04, 0x2c, 0xc8, 0x70, 0x24, 0xf3, 0xfd, 0x77, 0x60, 0xde, 0xd5, 0xf7, 0xfd, 0xa0, + 0x22, 0x49, 0x8b, 0x55, 0xcd, 0x22, 0x3c, 0xab, 0x48, 0x18, 0x2b, 0xbe, 0x22, 0x89, 0x7c, 0xd0, + 0x90, 0xba, 0xf2, 0x83, 0x86, 0xf4, 0xcf, 0xe4, 0x83, 0x06, 0xf5, 0x57, 0x01, 0x36, 0x2c, 0x9b, + 0xf4, 0xc5, 0x5d, 0x53, 0x5c, 0x7d, 0xc9, 0x72, 0x38, 0x79, 0x97, 0x19, 0xe4, 0x70, 0xad, 0x06, + 0x66, 0x34, 0xc6, 0x1a, 0x58, 0xa6, 0xdc, 0x8c, 0x9c, 0xb5, 0xc9, 0x58, 0x03, 0xcb, 0x0c, 0x5f, + 0xde, 0xd2, 0xd7, 0xbd, 0xbc, 0x9d, 0x26, 0x60, 0x49, 0xe6, 0xae, 0x61, 0xf8, 0xfd, 0x2a, 0xe4, + 0x45, 0x1a, 0x3b, 0x2d, 0xe8, 0xf8, 0x23, 0xbe, 0xc0, 0xb5, 0x1a, 0x38, 0x27, 0xd8, 0x2d, 0x13, + 0xad, 0x43, 0x41, 0x42, 0x23, 0x1f, 0x3f, 0x81, 0x20, 0xb5, 0x99, 0xf9, 0x5f, 0x87, 0xf4, 0xbe, + 0x65, 0x13, 0xb9, 0xd0, 0x63, 0x03, 0xc0, 0xd4, 0x01, 0x5b, 0x73, 0x98, 0xa3, 0x6b, 0xb9, 0xe0, + 0x32, 0x8e, 0xdb, 0x27, 0xcb, 0xce, 0xa8, 0x7d, 0xa2, 0x02, 0x9d, 0xb1, 0x4f, 0xe0, 0x98, 0x7d, + 0x82, 0x2d, 0xec, 0x93, 0xd0, 0xa8, 0x7d, 0x82, 0xf4, 0x33, 0xb1, 0x6f, 0x1b, 0x6e, 0xd6, 0x6c, + 0xdd, 0x38, 0xb4, 0x2d, 0xcf, 0x27, 0x66, 0x34, 0x62, 0x3c, 0x86, 0xec, 0x85, 0xa4, 0xf3, 0xaa, + 0x5b, 0x4b, 0x89, 0x54, 0xff, 0x2d, 0x01, 0xc5, 0x2d, 0xa2, 0xdb, 0xfe, 0xc1, 0xf4, 0x6a, 0xc8, + 0x27, 0x9e, 0x2f, 0x0f, 0x2b, 0xfe, 0x1b, 0x7d, 0x03, 0x72, 0x61, 0x4e, 0x72, 0xed, 0xfb, 0x5b, + 0x08, 0x45, 0x4f, 0x60, 0x9e, 0xed, 0x31, 0x3a, 0x0e, 0x8a, 0x9d, 0xab, 0x9e, 0x76, 0x24, 0x92, + 0x1d, 0x32, 0x2e, 0xe1, 0x49, 0x08, 0x5f, 0x4a, 0x19, 0x1c, 0x34, 0xd1, 0xff, 0x87, 0x22, 0x7f, + 0x99, 0x08, 0x72, 0xae, 0xcc, 0x75, 0x3a, 0x0b, 0xe2, 0x71, 0x51, 0xe4, 0x5b, 0xff, 0x93, 0x80, + 0x95, 0x1d, 0x7d, 0xb2, 0x47, 0x64, 0xd8, 0x20, 0x26, 0x26, 0x06, 0x75, 0x4d, 0xd4, 0x8d, 0x86, + 0x9b, 0x2b, 0xde, 0x2a, 0xe3, 0x84, 0xe3, 0xa3, 0x4e, 0x50, 0x80, 0x25, 0x23, 0x05, 0xd8, 0x0a, + 0x64, 0x1c, 0xea, 0x18, 0x44, 0xc6, 0x22, 0xd1, 0x50, 0xad, 0x68, 0xa8, 0x29, 0x85, 0xcf, 0x88, + 0xfc, 0x11, 0xb0, 0x4d, 0xfd, 0xb0, 0x37, 0xf4, 0x09, 0x94, 0x7a, 0xcd, 0x3a, 0x6e, 0xf6, 0x6b, + 0x9d, 0xef, 0x6a, 0xbd, 0xea, 0x76, 0xaf, 0xfa, 0xf8, 0xa1, 0xd6, 0xed, 0x6c, 0x7f, 0xef, 0xd1, + 0x93, 0x87, 0xdf, 0x50, 0x12, 0xa5, 0xf2, 0xc9, 0x69, 0xf9, 0x76, 0xbb, 0x5a, 0xdf, 0x16, 0x3b, + 0x66, 0x8f, 0xbe, 0xec, 0xe9, 0xb6, 0xa7, 0x3f, 0x7e, 0xd8, 0xa5, 0xf6, 0x84, 0x61, 0xd8, 0xb2, + 0x2e, 0x46, 0xcf, 0xab, 0xe8, 0x31, 0x9c, 0xb8, 0xf4, 0x18, 0x9e, 0x9e, 0xe6, 0xc9, 0x4b, 0x4e, + 0xf3, 0x0d, 0x58, 0x31, 0x5c, 0xea, 0x79, 0x1a, 0xcb, 0xfe, 0x89, 0x39, 0x53, 0x5f, 0x7c, 0xe9, + 0xfc, 0x6c, 0x7d, 0xb9, 0xce, 0xf8, 0x3d, 0xce, 0x96, 0xea, 0x97, 0x8d, 0x08, 0x89, 0xf7, 0xa4, + 0xfe, 0x7e, 0x8a, 0x25, 0x52, 0xd6, 0x91, 0x65, 0x93, 0x01, 0xf1, 0xd0, 0x73, 0x58, 0x32, 0x5c, + 0x62, 0xb2, 0xb4, 0x5e, 0xb7, 0xa3, 0x1f, 0xd1, 0xfe, 0xbf, 0xd8, 0x9c, 0x26, 0x14, 0xac, 0xd4, + 0x43, 0xa9, 0xde, 0x88, 0x18, 0x78, 0xd1, 0xb8, 0xd0, 0x46, 0x9f, 0xc2, 0x92, 0x47, 0x6c, 0xcb, + 0x19, 0xbf, 0xd4, 0x0c, 0xea, 0xf8, 0xe4, 0x65, 0xf0, 0x22, 0x76, 0x9d, 0xde, 0x5e, 0x73, 0x9b, + 0x49, 0xd5, 0x85, 0x50, 0x0d, 0x9d, 0x9f, 0xad, 0x2f, 0x5e, 0xa4, 0xe1, 0x45, 0xa9, 0x59, 0xb6, + 0x4b, 0x6d, 0x58, 0xbc, 0x68, 0x0d, 0x5a, 0x91, 0x7b, 0x9f, 0x87, 0x90, 0x60, 0x6f, 0xa3, 0xdb, + 0x90, 0x73, 0xc9, 0xc0, 0xf2, 0x7c, 0x57, 0xb8, 0x99, 0x71, 0x42, 0x0a, 0xdb, 0xf9, 0xe2, 0x0b, + 0xa8, 0xd2, 0x2f, 0xc3, 0x4c, 0x8f, 0x6c, 0xb3, 0x98, 0x96, 0xa7, 0xef, 0x49, 0x95, 0x39, 0x1c, + 0x34, 0xd9, 0x1a, 0x1c, 0x7b, 0x61, 0xa2, 0xc6, 0x7f, 0x33, 0x1a, 0xcf, 0x28, 0xe4, 0xf7, 0x60, + 0x3c, 0x67, 0x08, 0x3e, 0x2c, 0x4d, 0x47, 0x3e, 0x2c, 0x5d, 0x81, 0x8c, 0x4d, 0x8e, 0x88, 0x2d, + 0xce, 0x72, 0x2c, 0x1a, 0xf7, 0x1e, 0x42, 0x31, 0xf8, 0x82, 0x91, 0x7f, 0x39, 0x91, 0x83, 0x74, + 0xbf, 0xda, 0x7b, 0xa6, 0xcc, 0x21, 0x80, 0xac, 0x58, 0x9c, 0xe2, 0xb5, 0xae, 0xde, 0x69, 0x6f, + 0xb4, 0x36, 0x95, 0xe4, 0xbd, 0xdf, 0x49, 0x43, 0x3e, 0x7c, 0x2f, 0x62, 0x67, 0x47, 0xbb, 0xf9, + 0x22, 0x58, 0xdd, 0x21, 0xbd, 0x4d, 0x8e, 0xd1, 0xdb, 0xd3, 0x5b, 0xa8, 0x4f, 0xc4, 0x03, 0x79, + 0xc8, 0x0e, 0x6e, 0xa0, 0xde, 0x85, 0x5c, 0xb5, 0xd7, 0x6b, 0x6d, 0xb6, 0x9b, 0x0d, 0xe5, 0xb3, + 0x44, 0xe9, 0x4b, 0x27, 0xa7, 0xe5, 0xe5, 0x10, 0x54, 0xf5, 0xc4, 0xe2, 0xe3, 0xa8, 0x7a, 0xbd, + 0xd9, 0xed, 0x37, 0x1b, 0xca, 0xab, 0xe4, 0x2c, 0x8a, 0xdf, 0xaa, 0xf0, 0x4f, 0x77, 0xf2, 0x5d, + 0xdc, 0xec, 0x56, 0x31, 0xeb, 0xf0, 0xb3, 0xa4, 0xb8, 0x1c, 0x9b, 0xf6, 0xe8, 0x92, 0x91, 0xee, + 0xb2, 0x3e, 0xd7, 0x82, 0x6f, 0xe1, 0x5e, 0xa5, 0xc4, 0xe7, 0x1d, 0xd3, 0xc7, 0x2f, 0xa2, 0x9b, + 0x13, 0xd6, 0x1b, 0x7f, 0x75, 0xe4, 0x6a, 0x52, 0x33, 0xbd, 0xf5, 0x58, 0xec, 0x61, 0x5a, 0x54, + 0x98, 0xc7, 0xbb, 0xed, 0x36, 0x03, 0xbd, 0x4a, 0xcf, 0x8c, 0x0e, 0x8f, 0x1d, 0x56, 0x31, 0xa3, + 0xbb, 0x90, 0x0b, 0x1e, 0x25, 0x95, 0xcf, 0xd2, 0x33, 0x06, 0xd5, 0x83, 0x17, 0x55, 0xde, 0xe1, + 0xd6, 0x6e, 0x9f, 0x7f, 0xaa, 0xf7, 0x2a, 0x33, 0xdb, 0xe1, 0xc1, 0xd8, 0x37, 0xe9, 0xb1, 0xc3, + 0xf6, 0xac, 0xbc, 0x87, 0xfb, 0x2c, 0x23, 0x2e, 0x2d, 0x42, 0x8c, 0xbc, 0x84, 0x7b, 0x17, 0x72, + 0xb8, 0xf9, 0x1d, 0xf1, 0x55, 0xdf, 0xab, 0xec, 0x8c, 0x1e, 0x4c, 0x3e, 0x25, 0x06, 0xeb, 0xad, + 0x0c, 0x59, 0xdc, 0xdc, 0xe9, 0x3c, 0x6f, 0x2a, 0x7f, 0x90, 0x9d, 0xd1, 0x83, 0xc9, 0x90, 0xf2, + 0x6f, 0x9b, 0x72, 0x1d, 0xdc, 0xdd, 0xaa, 0xf2, 0x49, 0x99, 0xd5, 0xd3, 0x71, 0x47, 0x07, 0xba, + 0x43, 0xcc, 0xe9, 0x57, 0x30, 0x21, 0xeb, 0xde, 0x2f, 0x40, 0x2e, 0xc8, 0x5d, 0xd1, 0x1a, 0x64, + 0x5f, 0x74, 0xf0, 0xb3, 0x26, 0x56, 0xe6, 0x84, 0x97, 0x03, 0xce, 0x0b, 0x51, 0x75, 0x94, 0x61, + 0x7e, 0xa7, 0xda, 0xae, 0x6e, 0x36, 0x71, 0x70, 0x89, 0x1e, 0x00, 0x64, 0x02, 0x56, 0x52, 0x64, + 0x07, 0xa1, 0xce, 0xda, 0xea, 0x0f, 0x7f, 0xb2, 0x36, 0xf7, 0xe3, 0x9f, 0xac, 0xcd, 0xbd, 0x3a, + 0x5f, 0x4b, 0xfc, 0xf0, 0x7c, 0x2d, 0xf1, 0x77, 0xe7, 0x6b, 0x89, 0x7f, 0x3d, 0x5f, 0x4b, 0xec, + 0x65, 0xf9, 0x31, 0xf1, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x08, 0x6c, 0xc9, 0x88, 0x12, + 0x32, 0x00, 0x00, } diff --git a/vendor/github.com/docker/swarmkit/api/types.proto b/vendor/github.com/docker/swarmkit/api/types.proto index 635d12b200..3b60c544c3 100644 --- a/vendor/github.com/docker/swarmkit/api/types.proto +++ b/vendor/github.com/docker/swarmkit/api/types.proto @@ -458,7 +458,16 @@ enum TaskState { SHUTDOWN = 640 [(gogoproto.enumvalue_customname)="TaskStateShutdown"]; // orchestrator requested shutdown FAILED = 704 [(gogoproto.enumvalue_customname)="TaskStateFailed"]; // task execution failed with error REJECTED = 768 [(gogoproto.enumvalue_customname)="TaskStateRejected"]; // task could not be executed here. - ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // The node on which this task is scheduled is Down for too long + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + REMOVE = 800 [(gogoproto.enumvalue_customname)="TaskStateRemove"]; + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // NOTE(stevvooe): The state of a task is actually a lamport clock, in that // given two observations, the greater of the two can be considered diff --git a/vendor/github.com/docker/swarmkit/api/watch.pb.go b/vendor/github.com/docker/swarmkit/api/watch.pb.go index 58fa8917e5..78f0014c7a 100644 --- a/vendor/github.com/docker/swarmkit/api/watch.pb.go +++ b/vendor/github.com/docker/swarmkit/api/watch.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -2043,12 +2044,12 @@ func NewRaftProxyWatchServer(local WatchServer, connSelector raftselector.ConnPr redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/vendor/github.com/docker/swarmkit/ca/auth.go b/vendor/github.com/docker/swarmkit/ca/auth.go index 488d34dd33..df4547fb13 100644 --- a/vendor/github.com/docker/swarmkit/ca/auth.go +++ b/vendor/github.com/docker/swarmkit/ca/auth.go @@ -10,10 +10,10 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) type localRequestKeyType struct{} @@ -52,13 +52,13 @@ func LogTLSState(ctx context.Context, tlsState *tls.ConnectionState) { // getCertificateSubject extracts the subject from a verified client certificate func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { if tlsState == nil { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "request is not using TLS") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "request is not using TLS") } if len(tlsState.PeerCertificates) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no client certificates in request") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no client certificates in request") } if len(tlsState.VerifiedChains) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") } return tlsState.VerifiedChains[0][0].Subject, nil @@ -67,11 +67,11 @@ func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { func tlsConnStateFromContext(ctx context.Context) (*tls.ConnectionState, error) { peer, ok := peer.FromContext(ctx) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } tlsInfo, ok := peer.AuthInfo.(credentials.TLSInfo) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") } return &tlsInfo.State, nil } @@ -98,21 +98,21 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, blacklistedCerts map[s return authorizeOrg(certSubj, org, blacklistedCerts) } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) } // authorizeOrg takes in a certificate subject and an organization, and returns // the Node ID of the node. func authorizeOrg(certSubj pkix.Name, org string, blacklistedCerts map[string]*api.BlacklistedCertificate) (string, error) { if _, ok := blacklistedCerts[certSubj.CommonName]; ok { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) } if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org { return certSubj.CommonName, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) } // AuthorizeForwardedRoleAndOrg checks for proper roles and organization of caller. The RPC may have @@ -123,7 +123,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde if isForwardedRequest(ctx) { _, err := AuthorizeOrgAndRole(ctx, org, blacklistedCerts, forwarderRoles...) if err != nil { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) } // This was a forwarded request. Authorize the forwarder, and @@ -132,15 +132,15 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde _, forwardedID, forwardedOrg, forwardedOUs := forwardedTLSInfoFromContext(ctx) if len(forwardedOUs) == 0 || forwardedID == "" || forwardedOrg == "" { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return "", status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } if !intersectArrays(forwardedOUs, authorizedRoles) { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) } if forwardedOrg != org { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) } return forwardedID, nil @@ -152,7 +152,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde return nodeID, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) } // intersectArrays returns true when there is at least one element in common @@ -219,7 +219,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { peer, ok := peer.FromContext(ctx) if !ok { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } directInfo := RemoteNodeInfo{ @@ -232,7 +232,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { if isForwardedRequest(ctx) { remoteAddr, cn, org, ous := forwardedTLSInfoFromContext(ctx) if len(ous) == 0 || cn == "" || org == "" { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } return RemoteNodeInfo{ Roles: ous, diff --git a/vendor/github.com/docker/swarmkit/ca/server.go b/vendor/github.com/docker/swarmkit/ca/server.go index 16dbedc8da..a456df7900 100644 --- a/vendor/github.com/docker/swarmkit/ca/server.go +++ b/vendor/github.com/docker/swarmkit/ca/server.go @@ -16,8 +16,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -149,7 +149,7 @@ func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequ // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } serverCtx, err := s.isRunningLocked() @@ -180,7 +180,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer // This node ID doesn't exist if node == nil { - return nil, grpc.Errorf(codes.NotFound, codes.NotFound.String()) + return nil, status.Errorf(codes.NotFound, codes.NotFound.String()) } log.G(ctx).WithFields(logrus.Fields{ @@ -236,7 +236,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNodeCertificateRequest) (*api.IssueNodeCertificateResponse, error) { // First, let's see if the remote node is presenting a non-empty CSR if len(request.CSR) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } if err := s.isReadyLocked(); err != nil { @@ -295,7 +295,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod s.mu.Unlock() if role < 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") + return nil, status.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") } // Max number of collisions of ID or CN to tolerate before giving up @@ -369,7 +369,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [ "method": "issueRenewCertificate", }).Warnf("node does not exist") // If this node doesn't exist, we shouldn't be renewing a certificate for it - return grpc.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) + return status.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) } // Create a new Certificate entry for this node with the new CSR and a RENEW state @@ -594,7 +594,7 @@ func (s *Server) isRunningLocked() (context.Context, error) { s.mu.Lock() if !s.isRunning() { s.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "CA signer is stopped") + return nil, status.Errorf(codes.Aborted, "CA signer is stopped") } ctx := s.ctx s.mu.Unlock() @@ -605,10 +605,10 @@ func (s *Server) isReadyLocked() error { s.mu.Lock() defer s.mu.Unlock() if !s.isRunning() { - return grpc.Errorf(codes.Aborted, "CA signer is stopped") + return status.Errorf(codes.Aborted, "CA signer is stopped") } if s.joinTokens == nil { - return grpc.Errorf(codes.Aborted, "CA signer is still starting") + return status.Errorf(codes.Aborted, "CA signer is still starting") } return nil } diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go b/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go index 5e8fa43ce3..d39c7d2b69 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go @@ -10,13 +10,12 @@ import ( "net/url" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/cloudflare/cfssl/helpers" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var minRootExpiration = 1 * helpers.OneYear @@ -60,7 +59,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi crossSignedCert, err = oldRootCA.CrossSignCACertificate(rootCert) } case !newRootHasSigner: // the original CA and the new CA both require external CAs - return nil, grpc.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") + return nil, status.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") default: // We need the same credentials but to connect to the original URLs (in case we are in the middle of a root rotation already) var urls []string @@ -70,7 +69,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi } } if len(urls) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, + return nil, status.Errorf(codes.InvalidArgument, "must provide an external CA for the current external root CA to generate a cross-signed certificate") } rootPool := x509.NewCertPool() @@ -83,7 +82,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi if err != nil { log.G(ctx).WithError(err).Error("unable to generate a cross-signed certificate for root rotation") - return nil, grpc.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") + return nil, status.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") } copied := apiRootCA.Copy() @@ -146,7 +145,7 @@ func validateHasAtLeastOneExternalCA(ctx context.Context, externalCAs map[string } } } - return nil, grpc.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) + return nil, status.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) } // validates that the list of external CAs have valid certs associated with them, and produce a mapping of subject/pubkey:external @@ -193,7 +192,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl newConfig.SigningCACert = ca.NormalizePEMs(newConfig.SigningCACert) // ensure this is normalized before we use it if len(newConfig.SigningCAKey) > 0 && len(newConfig.SigningCACert) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") + return nil, status.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") } normalizedRootCA := ca.NormalizePEMs(cluster.RootCA.CACert) @@ -216,7 +215,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl if cluster.RootCA.LastForcedRotation != newConfig.ForceRotate { newRootCA, err := ca.CreateRootCA(ca.DefaultRootCN) if err != nil { - return nil, grpc.Errorf(codes.Internal, err.Error()) + return nil, status.Errorf(codes.Internal, err.Error()) } return newRootRotationObject(ctx, securityConfig, &cluster.RootCA, newRootCA, oldCertExtCAs, newConfig.ForceRotate) } @@ -240,21 +239,21 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl } newRootCA, err := ca.NewRootCA(newConfig.SigningCACert, signingCert, newConfig.SigningCAKey, ca.DefaultNodeCertExpiration, nil) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, err.Error()) + return nil, status.Errorf(codes.InvalidArgument, err.Error()) } if len(newRootCA.Pool.Subjects()) != 1 { - return nil, grpc.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") + return nil, status.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") } parsedCert, err := helpers.ParseCertificatePEM(newConfig.SigningCACert) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") + return nil, status.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") } // The new certificate's expiry must be at least one year away if parsedCert.NotAfter.Before(time.Now().Add(minRootExpiration)) { - return nil, grpc.Errorf(codes.InvalidArgument, "CA certificate expires too soon") + return nil, status.Errorf(codes.InvalidArgument, "CA certificate expires too soon") } if !hasSigningKey(newConfig) { diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go b/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go index 329313a950..0876113d70 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -23,17 +23,17 @@ const ( func validateClusterSpec(spec *api.ClusterSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } // Validate that expiry time being provided is valid, and over our minimum if spec.CAConfig.NodeCertExpiry != nil { expiry, err := gogotypes.DurationFromProto(spec.CAConfig.NodeCertExpiry) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if expiry < ca.MinNodeCertExpiration { - return grpc.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) + return status.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) } } @@ -42,7 +42,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if len(spec.AcceptancePolicy.Policies) > 0 { for _, policy := range spec.AcceptancePolicy.Policies { if policy.Secret != nil && strings.ToLower(policy.Secret.Alg) != "bcrypt" { - return grpc.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) + return status.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) } } } @@ -51,13 +51,17 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if spec.Dispatcher.HeartbeatPeriod != nil { heartbeatPeriod, err := gogotypes.DurationFromProto(spec.Dispatcher.HeartbeatPeriod) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if heartbeatPeriod < 0 { - return grpc.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") + return status.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") } } + if spec.Annotations.Name != store.DefaultClusterName { + return status.Errorf(codes.InvalidArgument, "modification of cluster name is not allowed") + } + return nil } @@ -66,7 +70,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { // - Returns `NotFound` if the Cluster is not found. func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) (*api.GetClusterResponse, error) { if request.ClusterID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var cluster *api.Cluster @@ -74,7 +78,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) cluster = store.GetCluster(tx, request.ClusterID) }) if cluster == nil { - return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return nil, status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } redactedClusters := redactClusters([]*api.Cluster{cluster}) @@ -92,7 +96,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) // - Returns an error if the update fails. func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) { if request.ClusterID == "" || request.ClusterVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateClusterSpec(request.Spec); err != nil { return nil, err @@ -102,7 +106,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe err := s.store.Update(func(tx store.Tx) error { cluster = store.GetCluster(tx, request.ClusterID) if cluster == nil { - return grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } // This ensures that we have the current rootCA with which to generate tokens (expiration doesn't matter // for generating the tokens) @@ -110,7 +114,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe if err != nil { log.G(ctx).WithField( "method", "(*controlapi.Server).UpdateCluster").WithError(err).Error("invalid cluster root CA") - return grpc.Errorf(codes.Internal, "error loading cluster rootCA for update") + return status.Errorf(codes.Internal, "error loading cluster rootCA for update") } cluster.Meta.Version = *request.ClusterVersion diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/common.go b/vendor/github.com/docker/swarmkit/manager/controlapi/common.go index c01662379e..9e52179464 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/common.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/common.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/state/store" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var isValidDNSName = regexp.MustCompile(`^[a-zA-Z0-9](?:[-_]*[A-Za-z0-9]+)*$`) @@ -70,25 +70,25 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo func validateAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided") + return status.Errorf(codes.InvalidArgument, "meta: name must be provided") } if !isValidDNSName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") + return status.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") } if len(m.Name) > 63 { // DNS labels are limited to 63 characters - return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") + return status.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") } return nil } func validateConfigOrSecretAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "name must be provided") + return status.Errorf(codes.InvalidArgument, "name must be provided") } else if len(m.Name) > 64 || !isValidConfigOrSecretName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "invalid name, only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]") } return nil @@ -102,7 +102,7 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") + return status.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") } // First check against the known drivers @@ -119,16 +119,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if pg == nil { - return grpc.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) + return status.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) } p, err := pg.Get(driver.Name, pluginType, plugingetter.Lookup) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) + return status.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) } if p.IsV1() { - return grpc.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) + return status.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) } return nil diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/config.go b/vendor/github.com/docker/swarmkit/manager/controlapi/config.go index d0fe8a56c1..ae08885b00 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/config.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/config.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // MaxConfigSize is the maximum byte length of the `Config.Spec.Data` field. @@ -32,7 +32,7 @@ func configFromConfigSpec(spec *api.ConfigSpec) *api.Config { // - Returns an error if getting fails. func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (*api.GetConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } var config *api.Config @@ -41,7 +41,7 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( }) if config == nil { - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } return &api.GetConfigResponse{Config: config}, nil @@ -53,21 +53,21 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateConfig(ctx context.Context, request *api.UpdateConfigRequest) (*api.UpdateConfigResponse, error) { if request.ConfigID == "" || request.ConfigVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var config *api.Config err := s.store.Update(func(tx store.Tx) error { config = store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } // Check if the Name is different than the current name, or the config is non-nil and different // than the current config if config.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && !bytes.Equal(request.Spec.Data, config.Spec.Data)) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -164,7 +164,7 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.Name": request.Spec.Annotations.Name, @@ -184,20 +184,20 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ // - Returns an error if the deletion fails. func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequest) (*api.RemoveConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the config exists config := store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) + return status.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) } // Check if any services currently reference this config, return error if so services, err := store.FindServices(tx, store.ByReferencedConfigID(request.ConfigID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) + return status.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) } if len(services) != 0 { @@ -213,14 +213,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) } return store.DeleteConfig(tx, request.ConfigID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.ID": request.ConfigID, @@ -235,14 +235,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ func validateConfigSpec(spec *api.ConfigSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err } if len(spec.Data) >= MaxConfigSize || len(spec.Data) < 1 { - return grpc.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) + return status.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) } return nil } diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/network.go b/vendor/github.com/docker/swarmkit/manager/controlapi/network.go index b150de0e78..481b0cfe4b 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/network.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/network.go @@ -12,39 +12,39 @@ import ( "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateIPAMConfiguration(ipamConf *api.IPAMConfig) error { if ipamConf == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") + return status.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") } _, subnet, err := net.ParseCIDR(ipamConf.Subnet) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) } if ipamConf.Range != "" { ip, _, err := net.ParseCIDR(ipamConf.Range) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) } } if ipamConf.Gateway != "" { ip := net.ParseIP(ipamConf.Gateway) if ip == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) } } @@ -73,15 +73,15 @@ func validateIPAM(ipam *api.IPAMOptions, pg plugingetter.PluginGetter) error { func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" { - return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") + return status.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") } if spec.Attachable && spec.Ingress { - return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") + return status.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") } if err := validateAnnotations(spec.Annotations); err != nil { @@ -89,7 +89,7 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er } if _, ok := spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok { - return grpc.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", + return status.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", networkallocator.PredefinedLabel) } if err := validateDriver(spec.DriverConfig, pg, driverapi.NetworkPluginEndpointType); err != nil { @@ -117,9 +117,9 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe err := s.store.Update(func(tx store.Tx) error { if request.Spec.Ingress { if n, err := allocator.GetIngressNetwork(s.store); err == nil { - return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) + return status.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) } else if err != allocator.ErrNoIngress { - return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err) + return status.Errorf(codes.Internal, "failed ingress network presence check: %v", err) } } return store.CreateNetwork(tx, n) @@ -138,7 +138,7 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe // - Returns `NotFound` if the Network is not found. func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) (*api.GetNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var n *api.Network @@ -146,7 +146,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return &api.GetNetworkResponse{ Network: n, @@ -159,7 +159,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) // - Returns an error if the deletion fails. func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var ( @@ -171,7 +171,7 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } if allocator.IsIngressNetwork(n) { @@ -179,13 +179,13 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe } if v, ok := n.Spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok && v == "true" { - return nil, grpc.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", + return nil, status.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", request.NetworkID, n.Spec.Annotations.Name) } if err := rm(n.ID); err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return nil, err } @@ -196,21 +196,21 @@ func (s *Server) removeNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } if len(services) != 0 { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) } tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) } for _, t := range tasks { if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) } } @@ -222,11 +222,11 @@ func (s *Server) removeIngressNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.All) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } for _, srv := range services { if allocator.IsIngressNetworkNeeded(srv) { - return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) + return status.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) } } return store.DeleteNetwork(tx, id) diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/node.go b/vendor/github.com/docker/swarmkit/manager/controlapi/node.go index bac6b8073d..e1fe3dec1d 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/node.go @@ -9,13 +9,13 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateNodeSpec(spec *api.NodeSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } return nil } @@ -25,7 +25,7 @@ func validateNodeSpec(spec *api.NodeSpec) error { // - Returns `NotFound` if the Node is not found. func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var node *api.Node @@ -33,7 +33,7 @@ func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api node = store.GetNode(tx, request.NodeID) }) if node == nil { - return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return nil, status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if s.raft != nil { @@ -196,7 +196,7 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) { if request.NodeID == "" || request.NodeVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateNodeSpec(request.Spec); err != nil { return nil, err @@ -210,7 +210,7 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) err := s.store.Update(func(tx store.Tx) error { node = store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } // Demotion sanity checks. @@ -218,20 +218,20 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) // Check for manager entries in Store. managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager)) if err != nil { - return grpc.Errorf(codes.Internal, "internal store error: %v", err) + return status.Errorf(codes.Internal, "internal store error: %v", err) } if len(managers) == 1 && managers[0].ID == node.ID { - return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") + return status.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") } // Check for node in memberlist if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil { - return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist") + return status.Errorf(codes.NotFound, "can't find manager in raft memberlist") } // Quorum safeguard if !s.raft.CanRemoveMember(member.RaftID) { - return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") + return status.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") } } @@ -278,33 +278,33 @@ func removeNodeAttachments(tx store.Tx, nodeID string) error { // - Returns an error if the delete fails. func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { node := store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if node.Spec.DesiredRole == api.NodeRoleManager { if s.raft == nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) } if member := s.raft.GetMemberByNodeID(request.NodeID); member != nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) } } if !request.Force && node.Status.State == api.NodeStatus_READY { - return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) } // lookup the cluster - clusters, err := store.FindClusters(tx, store.ByName("default")) + clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName)) if err != nil { return err } if len(clusters) != 1 { - return grpc.Errorf(codes.Internal, "could not fetch cluster object") + return status.Errorf(codes.Internal, "could not fetch cluster object") } cluster := clusters[0] diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go b/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go index bac4c102ef..fdcd2c412c 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // assumes spec is not nil @@ -30,7 +30,7 @@ func secretFromSecretSpec(spec *api.SecretSpec) *api.Secret { // - Returns an error if getting fails. func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (*api.GetSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } var secret *api.Secret @@ -39,7 +39,7 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( }) if secret == nil { - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } secret.Spec.Data = nil // clean the actual secret data so it's never returned @@ -52,20 +52,20 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateSecret(ctx context.Context, request *api.UpdateSecretRequest) (*api.UpdateSecretResponse, error) { if request.SecretID == "" || request.SecretVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var secret *api.Secret err := s.store.Update(func(tx store.Tx) error { secret = store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } // Check if the Name is different than the current name, or the secret is non-nil and different // than the current secret if secret.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && subtle.ConstantTimeCompare(request.Spec.Data, secret.Spec.Data) == 0) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -171,7 +171,7 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) case nil: secret.Spec.Data = nil // clean the actual secret data so it's never returned log.G(ctx).WithFields(logrus.Fields{ @@ -192,20 +192,20 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ // - Returns an error if the deletion fails. func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequest) (*api.RemoveSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the secret exists secret := store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) + return status.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) } // Check if any services currently reference this secret, return error if so services, err := store.FindServices(tx, store.ByReferencedSecretID(request.SecretID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) + return status.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) } if len(services) != 0 { @@ -221,14 +221,14 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) } return store.DeleteSecret(tx, request.SecretID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) case nil: log.G(ctx).WithFields(logrus.Fields{ "secret.ID": request.SecretID, @@ -243,7 +243,7 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ func validateSecretSpec(spec *api.SecretSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err @@ -252,12 +252,12 @@ func validateSecretSpec(spec *api.SecretSpec) error { if spec.Driver != nil { // Ensure secret driver has a name if spec.Driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name") + return status.Errorf(codes.InvalidArgument, "secret driver must have a name") } return nil } if err := validation.ValidateSecretPayload(spec.Data); err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } return nil } diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/service.go b/vendor/github.com/docker/swarmkit/manager/controlapi/service.go index 951c27b01e..e4c27df977 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/service.go @@ -19,8 +19,8 @@ import ( "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -37,11 +37,11 @@ func validateResources(r *api.Resources) error { } if r.NanoCPUs != 0 && r.NanoCPUs < 1e6 { - return grpc.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) + return status.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) } if r.MemoryBytes != 0 && r.MemoryBytes < 4*1024*1024 { - return grpc.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) + return status.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) } if err := genericresource.ValidateTask(r); err != nil { return nil @@ -70,7 +70,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") } } @@ -80,7 +80,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if win < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") } } @@ -101,7 +101,7 @@ func validateUpdate(uc *api.UpdateConfig) error { } if uc.Delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") } if uc.Monitor != nil { @@ -110,12 +110,12 @@ func validateUpdate(uc *api.UpdateConfig) error { return err } if monitor < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") } } if uc.MaxFailureRatio < 0 || uc.MaxFailureRatio > 1 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") } return nil @@ -147,7 +147,7 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { LogDriver: taskSpec.LogDriver, }) if err != nil { - return grpc.Errorf(codes.InvalidArgument, err.Error()) + return status.Errorf(codes.InvalidArgument, err.Error()) } if err := validateImage(container.Image); err != nil { @@ -164,11 +164,11 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { // validateImage validates image name in containerSpec func validateImage(image string) error { if image == "" { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") } if _, err := reference.ParseNormalizedNamed(image); err != nil { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) } return nil } @@ -178,7 +178,7 @@ func validateMounts(mounts []api.Mount) error { mountMap := make(map[string]bool) for _, mount := range mounts { if _, exists := mountMap[mount.Target]; exists { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) } mountMap[mount.Target] = true } @@ -198,7 +198,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if interval != 0 && interval < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) } } @@ -208,7 +208,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if timeout != 0 && timeout < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) } } @@ -218,12 +218,12 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if sp != 0 && sp < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) } } if hc.Retries < 0 { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") } return nil @@ -233,28 +233,28 @@ func validateGenericRuntimeSpec(taskSpec api.TaskSpec) error { generic := taskSpec.GetGeneric() if len(generic.Kind) < 3 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) } reservedNames := []string{"container", "attachment"} for _, n := range reservedNames { if strings.ToLower(generic.Kind) == n { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) } } payload := generic.Payload if payload == nil { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") } if payload.TypeUrl == "" { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") } if len(payload.Value) == 0 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") } return nil @@ -284,7 +284,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { } if taskSpec.GetRuntime() == nil { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") + return status.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") } switch taskSpec.GetRuntime().(type) { @@ -297,7 +297,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { return err } default: - return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") + return status.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") } return nil @@ -324,7 +324,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { // for the backend network and hence we accept that configuration. if epSpec.Mode == api.ResolutionModeDNSRoundRobin && port.PublishMode == api.PublishModeIngress { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") } // If published port is not specified, it does not conflict @@ -335,7 +335,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol} if _, ok := portSet[portSpec]; ok { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") } portSet[portSpec] = struct{}{} @@ -358,23 +358,23 @@ func validateSecretRefsSpec(spec api.TaskSpec) error { for _, secretRef := range container.Secrets { // SecretID and SecretName are mandatory, we have invalid references without them if secretRef.SecretID == "" || secretRef.SecretName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference") + return status.Errorf(codes.InvalidArgument, "malformed secret reference") } // Every secret reference requires a Target if secretRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") } // If this is a file target, we will ensure filename uniqueness if secretRef.GetFile() != nil { fileName := secretRef.GetFile().Name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevSecretName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) + return status.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) } existingTargets[fileName] = secretRef.SecretName @@ -398,12 +398,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { for _, configRef := range container.Configs { // ConfigID and ConfigName are mandatory, we have invalid references without them if configRef.ConfigID == "" || configRef.ConfigName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference") + return status.Errorf(codes.InvalidArgument, "malformed config reference") } // Every config reference requires a Target if configRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") } // If this is a file target, we will ensure filename uniqueness @@ -411,12 +411,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { fileName := configRef.GetFile().Name // Validate the file name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevConfigName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) + return status.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) } existingTargets[fileName] = configRef.ConfigName @@ -436,7 +436,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error continue } if allocator.IsIngressNetwork(network) { - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "Service cannot be explicitly attached to the ingress network %q", network.Spec.Annotations.Name) } } @@ -448,11 +448,11 @@ func validateMode(s *api.ServiceSpec) error { switch m.(type) { case *api.ServiceSpec_Replicated: if int64(m.(*api.ServiceSpec_Replicated).Replicated.Replicas) < 0 { - return grpc.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") + return status.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") } case *api.ServiceSpec_Global: default: - return grpc.Errorf(codes.InvalidArgument, "Unrecognized service mode") + return status.Errorf(codes.InvalidArgument, "Unrecognized service mode") } return nil @@ -460,7 +460,7 @@ func validateMode(s *api.ServiceSpec) error { func validateServiceSpec(spec *api.ServiceSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateAnnotations(spec.Annotations); err != nil { return err @@ -536,7 +536,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err switch pc.PublishMode { case api.PublishModeHost: if _, ok := ingressPorts[pcToStruct(pc)]; ok { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } // Multiple services with same port in host publish mode can @@ -546,7 +546,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err _, ingressConflict := ingressPorts[pcToStruct(pc)] _, hostModeConflict := hostModePorts[pcToStruct(pc)] if ingressConflict || hostModeConflict { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } } @@ -598,7 +598,7 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error secretStr = "secret" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) } @@ -627,7 +627,7 @@ func (s *Server) checkConfigExistence(tx store.Tx, spec *api.ServiceSpec) error configStr = "config" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) } @@ -662,7 +662,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return nil, status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -694,7 +694,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe // - Returns `NotFound` if the Service is not found. func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) (*api.GetServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var service *api.Service @@ -702,7 +702,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.InsertDefaults { @@ -721,7 +721,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) // - Returns an error if the update fails. func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) { if request.ServiceID == "" || request.ServiceVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateServiceSpec(request.Spec); err != nil { return nil, err @@ -732,7 +732,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) { @@ -744,7 +744,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe err := s.store.Update(func(tx store.Tx) error { service = store.GetService(tx, request.ServiceID) if service == nil { - return grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } // It's not okay to update Service.Spec.Networks on its own. @@ -754,7 +754,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if (len(request.Spec.Networks) != 0 || len(service.Spec.Networks) != 0) && !reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) && reflect.DeepEqual(request.Spec.Task.Networks, service.Spec.Task.Networks) { - return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) } // Check to see if all the secrets being added exist as objects @@ -773,18 +773,18 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // with service mode change (comparing current config with previous config). // proper way to change service mode is to delete and re-add. if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) { - return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) + return status.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) } if service.Spec.Annotations.Name != request.Spec.Annotations.Name { - return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) } service.Meta.Version = *request.ServiceVersion if request.Rollback == api.UpdateServiceRequest_PREVIOUS { if service.PreviousSpec == nil { - return grpc.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) + return status.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) } curSpec := service.Spec.Copy() @@ -815,7 +815,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -836,7 +836,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // - Returns an error if the deletion fails. func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRequest) (*api.RemoveServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -844,7 +844,7 @@ func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRe }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } return nil, err } diff --git a/vendor/github.com/docker/swarmkit/manager/controlapi/task.go b/vendor/github.com/docker/swarmkit/manager/controlapi/task.go index d2ae21522f..51b7bf82d8 100644 --- a/vendor/github.com/docker/swarmkit/manager/controlapi/task.go +++ b/vendor/github.com/docker/swarmkit/manager/controlapi/task.go @@ -6,8 +6,8 @@ import ( "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // GetTask returns a Task given a TaskID. @@ -15,7 +15,7 @@ import ( // - Returns `NotFound` if the Task is not found. func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api.GetTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var task *api.Task @@ -23,7 +23,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api task = store.GetTask(tx, request.TaskID) }) if task == nil { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return &api.GetTaskResponse{ Task: task, @@ -36,7 +36,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api // - Returns an error if the deletion fails. func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) (*api.RemoveTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -44,7 +44,7 @@ func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return nil, err } diff --git a/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 4de6a30c4d..13d68293ae 100644 --- a/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -7,11 +7,8 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/transport" - "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/ca" @@ -25,6 +22,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" ) const ( @@ -66,8 +66,18 @@ var ( ErrSessionInvalid = errors.New("session invalid") // ErrNodeNotFound returned when the Node doesn't exist in raft. ErrNodeNotFound = errors.New("node not found") + + // Scheduling delay timer. + schedulingDelayTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "dispatcher", nil) + schedulingDelayTimer = ns.NewTimer("scheduling_delay", + "Scheduling delay is the time a task takes to go from NEW to RUNNING state.") + metrics.Register(ns) +} + // Config is configuration for Dispatcher. For default you should use // DefaultConfig. type Config struct { @@ -322,7 +332,7 @@ func (d *Dispatcher) isRunningLocked() (context.Context, error) { d.mu.Lock() if !d.isRunning() { d.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "dispatcher is stopped") + return nil, status.Errorf(codes.Aborted, "dispatcher is stopped") } ctx := d.ctx d.mu.Unlock() @@ -556,7 +566,7 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat } if t.NodeID != nodeID { - err := grpc.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") + err := status.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") log.WithField("task.id", u.TaskID).Error(err) return nil, err } @@ -632,6 +642,17 @@ func (d *Dispatcher) processUpdates(ctx context.Context) { return nil } + // Update scheduling delay metric for running tasks. + // We use the status update time on the leader to calculate the scheduling delay. + // Because of this, the recorded scheduling delay will be an overestimate and include + // the network delay between the worker and the leader. + // This is not ideal, but its a known overestimation, rather than using the status update time + // from the worker node, which may cause unknown incorrect results due to possible clock skew. + if status.State == api.TaskStateRunning { + start := time.Unix(status.AppliedAt.GetSeconds(), int64(status.AppliedAt.GetNanos())) + schedulingDelayTimer.UpdateSince(start) + } + task.Status = *status task.Status.AppliedBy = d.securityConfig.ClientTLSCreds.NodeID() task.Status.AppliedAt = ptypes.MustTimestampProto(time.Now()) @@ -1189,7 +1210,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio log.WithError(err).Error("failed to remove node") } // still return an abort if the transport closure was ineffective. - return grpc.Errorf(codes.Aborted, "node must disconnect") + return status.Errorf(codes.Aborted, "node must disconnect") } for { diff --git a/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go b/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go index 8a0de558d6..cf35bb869a 100644 --- a/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go +++ b/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go @@ -4,12 +4,11 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager/dispatcher/heartbeat" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const rateLimitCount = 3 @@ -36,7 +35,7 @@ func (rn *registeredNode) checkSessionID(sessionID string) error { // changed. If it has, we will the stream and make the node // re-register. if sessionID == "" || rn.SessionID != sessionID { - return grpc.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) + return status.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) } return nil @@ -97,7 +96,7 @@ func (s *nodeStore) CheckRateLimit(id string) error { } existRn.Attempts++ if existRn.Attempts > rateLimitCount { - return grpc.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) + return status.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) } existRn.Registered = time.Now() } @@ -136,7 +135,7 @@ func (s *nodeStore) Get(id string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, nil } @@ -146,7 +145,7 @@ func (s *nodeStore) GetWithSession(id, sid string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, rn.checkSessionID(sid) } diff --git a/vendor/github.com/docker/swarmkit/manager/health/health.go b/vendor/github.com/docker/swarmkit/manager/health/health.go index bf220bdc18..ef6658b09d 100644 --- a/vendor/github.com/docker/swarmkit/manager/health/health.go +++ b/vendor/github.com/docker/swarmkit/manager/health/health.go @@ -12,8 +12,8 @@ import ( "github.com/docker/swarmkit/api" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Server represents a Health Check server to check @@ -46,7 +46,7 @@ func (s *Server) Check(ctx context.Context, in *api.HealthCheckRequest) (*api.He Status: status, }, nil } - return nil, grpc.Errorf(codes.NotFound, "unknown service") + return nil, status.Errorf(codes.NotFound, "unknown service") } // SetServingStatus is called when need to reset the serving status of a service diff --git a/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go b/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go index 860b55c30e..de2b93633f 100644 --- a/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go +++ b/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go @@ -6,9 +6,6 @@ import ( "io" "sync" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" @@ -18,6 +15,8 @@ import ( "github.com/docker/swarmkit/watch" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -93,11 +92,11 @@ func (lb *LogBroker) Stop() error { func validateSelector(selector *api.LogSelector) error { if selector == nil { - return grpc.Errorf(codes.InvalidArgument, "log selector must be provided") + return status.Errorf(codes.InvalidArgument, "log selector must be provided") } if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 { - return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty") + return status.Errorf(codes.InvalidArgument, "log selector must not be empty") } return nil @@ -401,17 +400,17 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er } if logMsg.SubscriptionID == "" { - return grpc.Errorf(codes.InvalidArgument, "missing subscription ID") + return status.Errorf(codes.InvalidArgument, "missing subscription ID") } if currentSubscription == nil { currentSubscription = lb.getSubscription(logMsg.SubscriptionID) if currentSubscription == nil { - return grpc.Errorf(codes.NotFound, "unknown subscription ID") + return status.Errorf(codes.NotFound, "unknown subscription ID") } } else { if logMsg.SubscriptionID != currentSubscription.message.ID { - return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") + return status.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") } } @@ -427,7 +426,7 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er // Make sure logs are emitted using the right Node ID to avoid impersonation. for _, msg := range logMsg.Messages { if msg.Context.NodeID != remote.NodeID { - return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) + return status.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) } } diff --git a/vendor/github.com/docker/swarmkit/manager/manager.go b/vendor/github.com/docker/swarmkit/manager/manager.go index 39db22b09a..1ce727dcac 100644 --- a/vendor/github.com/docker/swarmkit/manager/manager.go +++ b/vendor/github.com/docker/swarmkit/manager/manager.go @@ -38,6 +38,7 @@ import ( "github.com/docker/swarmkit/manager/resourceapi" "github.com/docker/swarmkit/manager/scheduler" "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/raft/transport" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/manager/watchapi" "github.com/docker/swarmkit/remotes" @@ -54,9 +55,6 @@ import ( const ( // defaultTaskHistoryRetentionLimit is the number of tasks to keep. defaultTaskHistoryRetentionLimit = 5 - - // Default value for grpc max message size. - grpcMaxMessageSize = 128 << 20 ) // RemoteAddrs provides a listening address and an optional advertise address @@ -234,7 +232,7 @@ func New(config *Config) (*Manager, error) { grpc.Creds(config.SecurityConfig.ServerTLSCreds), grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), - grpc.MaxMsgSize(grpcMaxMessageSize), + grpc.MaxMsgSize(transport.GRPCMaxMsgSize), } m := &Manager{ @@ -404,7 +402,7 @@ func (m *Manager) Run(parent context.Context) error { ) m.raftNode.MemoryStore().View(func(readTx store.ReadTx) { - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) }) @@ -954,13 +952,18 @@ func (m *Manager) becomeLeader(ctx context.Context) { // store. Don't check the error because // we expect this to fail unless this // is a brand new cluster. - store.CreateCluster(tx, defaultClusterObject( + err := store.CreateCluster(tx, defaultClusterObject( clusterID, initialCAConfig, raftCfg, api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, unlockKeys, rootCA)) + + if err != nil && err != store.ErrExist { + log.G(ctx).WithError(err).Errorf("error creating cluster object") + } + // Add Node entry for ourself, if one // doesn't exist already. freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) diff --git a/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go b/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go index a1d2873e80..f6217ff7b4 100644 --- a/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go +++ b/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go @@ -73,7 +73,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { var err error g.store.View(func(readTx store.ReadTx) { var clusters []*api.Cluster - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if len(clusters) != 1 { return // just pick up the cluster when it is created. @@ -147,7 +147,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { if !orchestrator.IsGlobalService(v.Service) { continue } - orchestrator.DeleteServiceTasks(ctx, g.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, g.store, v.Service) // delete the service from service map delete(g.globalServices, v.Service.ID) g.restarts.ClearServiceHistory(v.Service.ID) diff --git a/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go b/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go index f4d0511f80..04aea8795a 100644 --- a/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go +++ b/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go @@ -17,7 +17,7 @@ import ( // responds to changes in individual tasks (or nodes which run them). func (r *Orchestrator) initCluster(readTx store.ReadTx) error { - clusters, err := store.FindClusters(readTx, store.ByName("default")) + clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if err != nil { return err } @@ -50,7 +50,7 @@ func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Even if !orchestrator.IsReplicatedService(v.Service) { return } - orchestrator.DeleteServiceTasks(ctx, r.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, r.store, v.Service) r.restarts.ClearServiceHistory(v.Service.ID) delete(r.reconcileServices, v.Service.ID) case api.EventCreateService: @@ -86,6 +86,12 @@ func (r *Orchestrator) resolveService(ctx context.Context, task *api.Task) *api. return service } +// reconcile decides what actions must be taken depending on the number of +// specificed slots and actual running slots. If the actual running slots are +// fewer than what is requested, it creates new tasks. If the actual running +// slots are more than requested, then it decides which slots must be removed +// and sets desired state of those tasks to REMOVE (the actual removal is handled +// by the task reaper, after the agent shuts the tasks down). func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { runningSlots, deadSlots, err := r.updatableAndDeadSlots(ctx, service) if err != nil { @@ -157,7 +163,11 @@ func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { r.updater.Update(ctx, r.cluster, service, sortedSlots[:specifiedSlots]) err = r.store.Batch(func(batch *store.Batch) error { r.deleteTasksMap(ctx, batch, deadSlots) - r.deleteTasks(ctx, batch, sortedSlots[specifiedSlots:]) + // for all slots that we are removing, we set the desired state of those tasks + // to REMOVE. Then, the agent is responsible for shutting them down, and the + // task reaper is responsible for actually removing them from the store after + // shutdown. + r.setTasksDesiredState(ctx, batch, sortedSlots[specifiedSlots:], api.TaskStateRemove) return nil }) if err != nil { @@ -198,10 +208,34 @@ func (r *Orchestrator) addTasks(ctx context.Context, batch *store.Batch, service } } -func (r *Orchestrator) deleteTasks(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot) { +// setTasksDesiredState sets the desired state for all tasks for the given slots to the +// requested state +func (r *Orchestrator) setTasksDesiredState(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot, newDesiredState api.TaskState) { for _, slot := range slots { for _, t := range slot { - r.deleteTask(ctx, batch, t) + err := batch.Update(func(tx store.Tx) error { + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > newDesiredState { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, newDesiredState, + ) + return nil + } + // update desired state + t.DesiredState = newDesiredState + + return store.UpdateTask(tx, t) + }) + + // log an error if we get one + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to update task to %v", newDesiredState.String()) + } } } } diff --git a/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go b/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go index 4e52c83abf..7356c38cd5 100644 --- a/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go +++ b/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go @@ -27,8 +27,10 @@ func IsGlobalService(service *api.Service) bool { return ok } -// DeleteServiceTasks deletes the tasks associated with a service. -func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api.Service) { +// SetServiceTasksRemove sets the desired state of tasks associated with a service +// to REMOVE, so that they can be properly shut down by the agent and later removed +// by the task reaper. +func SetServiceTasksRemove(ctx context.Context, s *store.MemoryStore, service *api.Service) { var ( tasks []*api.Task err error @@ -44,8 +46,23 @@ func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api. err = s.Batch(func(batch *store.Batch) error { for _, t := range tasks { err := batch.Update(func(tx store.Tx) error { - if err := store.DeleteTask(tx, t.ID); err != nil { - log.G(ctx).WithError(err).Errorf("failed to delete task") + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > api.TaskStateRemove { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, api.TaskStateRemove, + ) + return nil + } + // update desired state to REMOVE + t.DesiredState = api.TaskStateRemove + + if err := store.UpdateTask(tx, t); err != nil { + log.G(ctx).WithError(err).Errorf("failed transaction: update task desired state to REMOVE") } return nil }) diff --git a/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go b/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go index 577319c8e8..bcef801f63 100644 --- a/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go +++ b/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go @@ -23,12 +23,19 @@ const ( // exist for the same service/instance or service/nodeid combination. type TaskReaper struct { store *store.MemoryStore + // taskHistory is the number of tasks to keep taskHistory int64 - dirty map[orchestrator.SlotTuple]struct{} - orphaned []string - stopChan chan struct{} - doneChan chan struct{} + + // List of slot tubles to be inspected for task history cleanup. + dirty map[orchestrator.SlotTuple]struct{} + + // List of tasks collected for cleanup, which includes two kinds of tasks + // - serviceless orphaned tasks + // - tasks with desired state REMOVE that have already been shut down + cleanup []string + stopChan chan struct{} + doneChan chan struct{} } // New creates a new TaskReaper. @@ -41,7 +48,13 @@ func New(store *store.MemoryStore) *TaskReaper { } } -// Run is the TaskReaper's main loop. +// Run is the TaskReaper's watch loop which collects candidates for cleanup. +// Task history is mainly used in task restarts but is also available for administrative purposes. +// Note that the task history is stored per-slot-per-service for replicated services +// and per-node-per-service for global services. History does not apply to serviceless +// since they are not attached to a service. In addition, the TaskReaper watch loop is also +// responsible for cleaning up tasks associated with slots that were removed as part of +// service scale down or service removal. func (tr *TaskReaper) Run(ctx context.Context) { watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{}) @@ -50,7 +63,8 @@ func (tr *TaskReaper) Run(ctx context.Context) { watchCancel() }() - var tasks []*api.Task + var orphanedTasks []*api.Task + var removeTasks []*api.Task tr.store.View(func(readTx store.ReadTx) { var err error @@ -59,29 +73,54 @@ func (tr *TaskReaper) Run(ctx context.Context) { tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit } - tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) + // On startup, scan the entire store and inspect orphaned tasks from previous life. + orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) if err != nil { log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init") } + removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove)) + if err != nil { + log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init") + } }) - if len(tasks) > 0 { - for _, t := range tasks { - // Do not reap service tasks immediately + if len(orphanedTasks)+len(removeTasks) > 0 { + for _, t := range orphanedTasks { + // Do not reap service tasks immediately. + // Let them go through the regular history cleanup process + // of checking TaskHistoryRetentionLimit. if t.ServiceID != "" { continue } - tr.orphaned = append(tr.orphaned, t.ID) + // Serviceless tasks can be cleaned up right away since they are not attached to a service. + tr.cleanup = append(tr.cleanup, t.ID) } - - if len(tr.orphaned) > 0 { + // tasks with desired state REMOVE that have progressed beyond SHUTDOWN can be cleaned up + // right away + for _, t := range removeTasks { + if t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) + } + } + // Clean up tasks in 'cleanup' right away + if len(tr.cleanup) > 0 { tr.tick() } } + // Clean up when we hit TaskHistoryRetentionLimit or when the timer expires, + // whichever happens first. timer := time.NewTimer(reaperBatchingInterval) + // Watch for: + // 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot. + // 2. EventUpdateTask for cleaning + // - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED) + // - tasks which have desired state REMOVE and have been shut down by the agent + // (these are tasks which are associated with slots removed as part of service + // remove or scale down) + // 3. EventUpdateCluster for TaskHistoryRetentionLimit update. for { select { case event := <-watcher: @@ -95,14 +134,21 @@ func (tr *TaskReaper) Run(ctx context.Context) { }] = struct{}{} case api.EventUpdateTask: t := v.Task + // add serviceless orphaned tasks if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" { - tr.orphaned = append(tr.orphaned, t.ID) + tr.cleanup = append(tr.cleanup, t.ID) + } + // add tasks that have progressed beyond SHUTDOWN and have desired state REMOVE. These + // tasks are associated with slots that were removed as part of a service scale down + // or service removal. + if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) } case api.EventUpdateCluster: tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit } - if len(tr.dirty)+len(tr.orphaned) > maxDirty { + if len(tr.dirty)+len(tr.cleanup) > maxDirty { timer.Stop() tr.tick() } else { @@ -118,19 +164,22 @@ func (tr *TaskReaper) Run(ctx context.Context) { } } +// tick performs task history cleanup. func (tr *TaskReaper) tick() { - if len(tr.dirty) == 0 && len(tr.orphaned) == 0 { + if len(tr.dirty) == 0 && len(tr.cleanup) == 0 { return } defer func() { - tr.orphaned = nil + tr.cleanup = nil }() deleteTasks := make(map[string]struct{}) - for _, tID := range tr.orphaned { + for _, tID := range tr.cleanup { deleteTasks[tID] = struct{}{} } + + // Check history of dirty tasks for cleanup. tr.store.View(func(tx store.ReadTx) { for dirty := range tr.dirty { service := store.GetService(tx, dirty.ServiceID) @@ -141,8 +190,8 @@ func (tr *TaskReaper) tick() { taskHistory := tr.taskHistory // If MaxAttempts is set, keep at least one more than - // that number of tasks. This is necessary reconstruct - // restart history when the orchestrator starts up. + // that number of tasks (this overrides TaskHistoryRetentionLimit). + // This is necessary to reconstruct restart history when the orchestrator starts up. // TODO(aaronl): Consider hiding tasks beyond the normal // retention limit in the UI. // TODO(aaronl): There are some ways to cut down the @@ -156,6 +205,7 @@ func (tr *TaskReaper) tick() { taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1 } + // Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history. if taskHistory < 0 { continue } @@ -164,6 +214,7 @@ func (tr *TaskReaper) tick() { switch service.Spec.GetMode().(type) { case *api.ServiceSpec_Replicated: + // Clean out the slot for which we received EventCreateTask. var err error historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot)) if err != nil { @@ -171,6 +222,7 @@ func (tr *TaskReaper) tick() { } case *api.ServiceSpec_Global: + // Clean out the node history in case of global services. tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID)) if err != nil { continue @@ -215,6 +267,7 @@ func (tr *TaskReaper) tick() { } }) + // Perform cleanup. if len(deleteTasks) > 0 { tr.store.Batch(func(batch *store.Batch) error { for taskID := range deleteTasks { diff --git a/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go b/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go index 87b01ebd3b..ec19fba850 100644 --- a/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go +++ b/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -50,11 +50,11 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att } }) if network == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.Config.Target) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.Config.Target) } if !network.Spec.Attachable { - return nil, grpc.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) + return nil, status.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) } t := &api.Task{ @@ -98,7 +98,7 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att // - Returns an error if the deletion fails. func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.DetachNetworkRequest) (*api.DetachNetworkResponse, error) { if request.AttachmentID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } nodeInfo, err := ca.RemoteNode(ctx) @@ -109,10 +109,10 @@ func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.Det if err := ra.store.Update(func(tx store.Tx) error { t := store.GetTask(tx, request.AttachmentID) if t == nil { - return grpc.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) + return status.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) } if t.NodeID != nodeInfo.NodeID { - return grpc.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) + return status.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) } return store.DeleteTask(tx, request.AttachmentID) diff --git a/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go b/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go index 3b1c73fe2d..dab3c6695d 100644 --- a/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go +++ b/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go @@ -169,7 +169,10 @@ func (f *PluginFilter) Check(n *NodeInfo) bool { } } - if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" { + // It's possible that the LogDriver object does not carry a name, just some + // configuration options. In that case, the plugin filter shouldn't fail to + // schedule the task + if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" && f.t.Spec.LogDriver.Name != "" { // If there are no log driver types in the list at all, most likely this is // an older daemon that did not report this information. In this case don't filter if typeFound, exists := f.pluginExistsOnNode("Log", f.t.Spec.LogDriver.Name, nodePlugins); !exists && typeFound { @@ -294,6 +297,14 @@ func (f *PlatformFilter) platformEqual(imgPlatform, nodePlatform api.Platform) b nodePlatform.Architecture = "amd64" } + // normalize "aarch64" architectures to "arm64" + if imgPlatform.Architecture == "aarch64" { + imgPlatform.Architecture = "arm64" + } + if nodePlatform.Architecture == "aarch64" { + nodePlatform.Architecture = "arm64" + } + if (imgPlatform.Architecture == "" || imgPlatform.Architecture == nodePlatform.Architecture) && (imgPlatform.OS == "" || imgPlatform.OS == nodePlatform.OS) { return true } diff --git a/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go b/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go index 28c7cfa47e..56b7c7c966 100644 --- a/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -2,6 +2,7 @@ package raft import ( "fmt" + "io" "math" "math/rand" "net" @@ -14,6 +15,7 @@ import ( "github.com/coreos/etcd/raft/raftpb" "github.com/docker/docker/pkg/signal" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" @@ -34,6 +36,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) var ( @@ -62,6 +65,9 @@ var ( // work around lint lostQuorumMessage = "The swarm does not have a leader. It's possible that too few managers are online. Make sure more than half of the managers are online." errLostQuorum = errors.New(lostQuorumMessage) + + // Timer to capture ProposeValue() latency. + proposeLatencyTimer metrics.Timer ) // LeadershipState indicates whether the node is a leader or follower. @@ -180,12 +186,9 @@ type NodeOptions struct { ClockSource clock.Clock // SendTimeout is the timeout on the sending messages to other raft // nodes. Leave this as 0 to get the default value. - SendTimeout time.Duration - // LargeSendTimeout is the timeout on the sending snapshots to other raft - // nodes. Leave this as 0 to get the default value. - LargeSendTimeout time.Duration - TLSCredentials credentials.TransportCredentials - KeyRotator EncryptionKeyRotator + SendTimeout time.Duration + TLSCredentials credentials.TransportCredentials + KeyRotator EncryptionKeyRotator // DisableStackDump prevents Run from dumping goroutine stacks when the // store becomes stuck. DisableStackDump bool @@ -193,6 +196,9 @@ type NodeOptions struct { func init() { rand.Seed(time.Now().UnixNano()) + ns := metrics.NewNamespace("swarm", "raft", nil) + proposeLatencyTimer = ns.NewTimer("transaction_latency", "Raft transaction latency.") + metrics.Register(ns) } // NewNode generates a new Raft node @@ -207,11 +213,6 @@ func NewNode(opts NodeOptions) *Node { if opts.SendTimeout == 0 { opts.SendTimeout = 2 * time.Second } - if opts.LargeSendTimeout == 0 { - // a "slow" 100Mbps connection can send over 240MB data in 20 seconds - // which is well over the gRPC message limit of 128MB allowed by SwarmKit - opts.LargeSendTimeout = 20 * time.Second - } raftStore := raft.NewMemoryStorage() @@ -357,7 +358,6 @@ func (n *Node) initTransport() { transportConfig := &transport.Config{ HeartbeatInterval: time.Duration(n.Config.ElectionTick) * n.opts.TickInterval, SendTimeout: n.opts.SendTimeout, - LargeSendTimeout: n.opts.LargeSendTimeout, Credentials: n.opts.TLSCredentials, Raft: n, } @@ -664,7 +664,7 @@ func (n *Node) Run(ctx context.Context) error { if n.snapshotInProgress == nil && (n.needsSnapshot(ctx) || raftConfig.SnapshotInterval > 0 && n.appliedIndex-n.snapshotMeta.Index >= raftConfig.SnapshotInterval) { - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 { @@ -706,7 +706,7 @@ func (n *Node) Run(ctx context.Context) error { // there was a key rotation that took place before while the snapshot // was in progress - we have to take another snapshot and encrypt with the new key n.rotationQueued = false - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } case <-n.keyRotator.RotationNotify(): // There are 2 separate checks: rotationQueued, and n.needsSnapshot(). @@ -719,7 +719,7 @@ func (n *Node) Run(ctx context.Context) error { case n.snapshotInProgress != nil: n.rotationQueued = true case n.needsSnapshot(ctx): - n.doSnapshot(ctx, n.getCurrentRaftConfig()) + n.triggerSnapshot(ctx, n.getCurrentRaftConfig()) } case <-ctx.Done(): return nil @@ -929,11 +929,11 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons defer n.membershipLock.Unlock() if !n.IsMember() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) } if !n.isLeader() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) } remoteAddr := req.Addr @@ -944,7 +944,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons requestHost, requestPort, err := net.SplitHostPort(remoteAddr) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) + return nil, status.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) } requestIP := net.ParseIP(requestHost) @@ -1118,7 +1118,7 @@ func (n *Node) UpdateNode(id uint64, addr string) { // membership to an active member of the raft func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) { if req.Node == nil { - return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided") + return nil, status.Errorf(codes.InvalidArgument, "no node information provided") } nodeInfo, err := ca.RemoteNode(ctx) @@ -1302,6 +1302,82 @@ func (n *Node) reportNewAddress(ctx context.Context, id uint64) error { return n.transport.UpdatePeerAddr(id, newAddr) } +// StreamRaftMessage is the server endpoint for streaming Raft messages. +// It accepts a stream of raft messages to be processed on this raft member, +// returning a StreamRaftMessageResponse when processing of the streamed +// messages is complete. +// It is called from the Raft leader, which uses it to stream messages +// to this raft member. +// A single stream corresponds to a single raft message, +// which may be disassembled and streamed by the sender +// as individual messages. Therefore, each of the messages +// received by the stream will have the same raft message type and index. +// Currently, only messages of type raftpb.MsgSnap can be disassembled, sent +// and received on the stream. +func (n *Node) StreamRaftMessage(stream api.Raft_StreamRaftMessageServer) error { + // recvdMsg is the current messasge received from the stream. + // assembledMessage is where the data from recvdMsg is appended to. + var recvdMsg, assembledMessage *api.StreamRaftMessageRequest + var err error + + // First message index. + var raftMsgIndex uint64 + + for { + recvdMsg, err = stream.Recv() + if err == io.EOF { + break + } else if err != nil { + log.G(stream.Context()).WithError(err).Error("error while reading from stream") + return err + } + + // Initialized the message to be used for assembling + // the raft message. + if assembledMessage == nil { + // For all message types except raftpb.MsgSnap, + // we don't expect more than a single message + // on the stream so we'll get an EOF on the next Recv() + // and go on to process the received message. + assembledMessage = recvdMsg + raftMsgIndex = recvdMsg.Message.Index + continue + } + + // Verify raft message index. + if recvdMsg.Message.Index != raftMsgIndex { + errMsg := fmt.Sprintf("Raft message chunk with index %d is different from the previously received raft message index %d", + recvdMsg.Message.Index, raftMsgIndex) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Verify that multiple message received on a stream + // can only be of type raftpb.MsgSnap. + if recvdMsg.Message.Type != raftpb.MsgSnap { + errMsg := fmt.Sprintf("Raft message chunk is not of type %d", + raftpb.MsgSnap) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Append the received snapshot data. + assembledMessage.Message.Snapshot.Data = append(assembledMessage.Message.Snapshot.Data, recvdMsg.Message.Snapshot.Data...) + } + + // We should have the complete snapshot. Verify and process. + if err == io.EOF { + _, err = n.ProcessRaftMessage(stream.Context(), &api.ProcessRaftMessageRequest{Message: assembledMessage.Message}) + if err == nil { + // Translate the response of ProcessRaftMessage() from + // ProcessRaftMessageResponse to StreamRaftMessageResponse if needed. + return stream.SendAndClose(&api.StreamRaftMessageResponse{}) + } + } + + return err +} + // ProcessRaftMessage calls 'Step' which advances the // raft state machine with the provided message on the // receiving node @@ -1315,7 +1391,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa // a node in the remove set if n.cluster.IsIDRemoved(msg.Message.From) { n.processRaftMessageLogger(ctx, msg).Debug("received message from removed member") - return nil, grpc.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) + return nil, status.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) } ctx, cancel := n.WithContext(ctx) @@ -1393,7 +1469,7 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques member := n.cluster.GetMember(msg.RaftID) if member == nil { - return nil, grpc.Errorf(codes.NotFound, "member %x not found", msg.RaftID) + return nil, status.Errorf(codes.NotFound, "member %x not found", msg.RaftID) } return &api.ResolveAddressResponse{Addr: member.Addr}, nil } @@ -1497,9 +1573,11 @@ func (n *Node) registerNode(node *api.RaftMember) error { // ProposeValue calls Propose on the underlying raft library(etcd/raft) and waits // on the commit log action before returning a result func (n *Node) ProposeValue(ctx context.Context, storeAction []api.StoreAction, cb func()) error { + defer metrics.StartTimer(proposeLatencyTimer)() ctx, cancel := n.WithContext(ctx) defer cancel() _, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb) + if err != nil { return err } @@ -1808,12 +1886,13 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { } if !n.wait.trigger(r.ID, r) { - log.G(ctx).Errorf("wait not found for raft request id %x", r.ID) - // There was no wait on this ID, meaning we don't have a // transaction in progress that would be committed to the - // memory store by the "trigger" call. Either a different node - // wrote this to raft, or we wrote it before losing the leader + // memory store by the "trigger" call. This could mean that: + // 1. Startup is in progress, and the raft WAL is being parsed, + // processed and applied to the store, or + // 2. Either a different node wrote this to raft, + // or we wrote it before losing the leader // position and cancelling the transaction. This entry still needs // to be committed since other nodes have already committed it. // Create a new transaction to commit this entry. @@ -1827,7 +1906,6 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { err := n.memoryStore.ApplyStoreActions(r.Action) if err != nil { log.G(ctx).WithError(err).Error("failed to apply actions from raft") - // TODO(anshul) return err here ? } } return nil diff --git a/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go b/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go index d12c43af2c..f538317c06 100644 --- a/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -5,6 +5,7 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/encryption" @@ -15,6 +16,18 @@ import ( "golang.org/x/net/context" ) +var ( + // Snapshot create latency timer. + snapshotLatencyTimer metrics.Timer +) + +func init() { + ns := metrics.NewNamespace("swarm", "raft", nil) + snapshotLatencyTimer = ns.NewTimer("snapshot_latency", + "Raft snapshot create latency.") + metrics.Register(ns) +} + func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) { keys := n.keyRotator.GetKeys() @@ -169,7 +182,7 @@ func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) { return raft.Peer{ID: n.Config.ID, Context: metadata}, nil } -func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { +func (n *Node) triggerSnapshot(ctx context.Context, raftConfig api.RaftConfig) { snapshot := api.Snapshot{Version: api.Snapshot_V0} for _, member := range n.cluster.Members() { snapshot.Membership.Members = append(snapshot.Membership.Members, @@ -185,6 +198,9 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { n.asyncTasks.Add(1) n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) { + // Deferred latency capture. + defer metrics.StartTimer(snapshotLatencyTimer)() + defer func() { n.asyncTasks.Done() n.snapshotInProgress <- snapshotMeta diff --git a/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go b/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go index 8c7ca75458..eb849c0803 100644 --- a/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go +++ b/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go @@ -18,6 +18,11 @@ import ( "github.com/pkg/errors" ) +const ( + // GRPCMaxMsgSize is the max allowed gRPC message size for raft messages. + GRPCMaxMsgSize = 4 << 20 +) + type peer struct { id uint64 @@ -132,17 +137,112 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) { return resp.Addr, nil } -func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { - timeout := p.tr.config.SendTimeout - // if a snapshot is being sent, set timeout to LargeSendTimeout because - // sending snapshots can take more time than other messages sent between peers. - // The same applies to AppendEntries as well, where messages can get large. - if m.Type == raftpb.MsgSnap || m.Type == raftpb.MsgApp { - timeout = p.tr.config.LargeSendTimeout +// Returns the raft message struct size (not including the payload size) for the given raftpb.Message. +// The payload is typically the snapshot or append entries. +func raftMessageStructSize(m *raftpb.Message) int { + return (&api.ProcessRaftMessageRequest{Message: m}).Size() - len(m.Snapshot.Data) +} + +// Returns the max allowable payload based on MaxRaftMsgSize and +// the struct size for the given raftpb.Message. +func raftMessagePayloadSize(m *raftpb.Message) int { + return GRPCMaxMsgSize - raftMessageStructSize(m) +} + +// Split a large raft message into smaller messages. +// Currently this means splitting the []Snapshot.Data into chunks whose size +// is dictacted by MaxRaftMsgSize. +func splitSnapshotData(ctx context.Context, m *raftpb.Message) []api.StreamRaftMessageRequest { + var messages []api.StreamRaftMessageRequest + if m.Type != raftpb.MsgSnap { + return messages } - ctx, cancel := context.WithTimeout(ctx, timeout) + + // get the size of the data to be split. + size := len(m.Snapshot.Data) + + // Get the max payload size. + payloadSize := raftMessagePayloadSize(m) + + // split the snapshot into smaller messages. + for snapDataIndex := 0; snapDataIndex < size; { + chunkSize := size - snapDataIndex + if chunkSize > payloadSize { + chunkSize = payloadSize + } + + raftMsg := *m + + // sub-slice for this snapshot chunk. + raftMsg.Snapshot.Data = m.Snapshot.Data[snapDataIndex : snapDataIndex+chunkSize] + + snapDataIndex += chunkSize + + // add message to the list of messages to be sent. + msg := api.StreamRaftMessageRequest{Message: &raftMsg} + messages = append(messages, msg) + } + + return messages +} + +// Function to check if this message needs to be split to be streamed +// (because it is larger than GRPCMaxMsgSize). +// Returns true if the message type is MsgSnap +// and size larger than MaxRaftMsgSize. +func needsSplitting(m *raftpb.Message) bool { + raftMsg := api.ProcessRaftMessageRequest{Message: m} + return m.Type == raftpb.MsgSnap && raftMsg.Size() > GRPCMaxMsgSize +} + +func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { + ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout) defer cancel() - _, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + + var err error + var stream api.Raft_StreamRaftMessageClient + stream, err = api.NewRaftClient(p.conn()).StreamRaftMessage(ctx) + + if err == nil { + // Split the message if needed. + // Currently only supported for MsgSnap. + var msgs []api.StreamRaftMessageRequest + if needsSplitting(&m) { + msgs = splitSnapshotData(ctx, &m) + } else { + raftMsg := api.StreamRaftMessageRequest{Message: &m} + msgs = append(msgs, raftMsg) + } + + // Stream + for _, msg := range msgs { + err = stream.Send(&msg) + if err != nil { + log.G(ctx).WithError(err).Error("error streaming message to peer") + stream.CloseAndRecv() + break + } + } + + // Finished sending all the messages. + // Close and receive response. + if err == nil { + _, err = stream.CloseAndRecv() + + if err != nil { + log.G(ctx).WithError(err).Error("error receiving response") + } + } + } else { + log.G(ctx).WithError(err).Error("error sending message to peer") + } + + // Try doing a regular rpc if the receiver doesn't support streaming. + if grpc.Code(err) == codes.Unimplemented { + _, err = api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + } + + // Handle errors. if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() { p.tr.config.NodeRemoved() } diff --git a/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go b/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go index 6f096ef9b2..b741c4aa67 100644 --- a/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go +++ b/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go @@ -36,7 +36,6 @@ type Raft interface { type Config struct { HeartbeatInterval time.Duration SendTimeout time.Duration - LargeSendTimeout time.Duration Credentials credentials.TransportCredentials RaftID string diff --git a/vendor/github.com/docker/swarmkit/manager/state/store/memory.go b/vendor/github.com/docker/swarmkit/manager/state/store/memory.go index 01245a6966..e64565fae8 100644 --- a/vendor/github.com/docker/swarmkit/manager/state/store/memory.go +++ b/vendor/github.com/docker/swarmkit/manager/state/store/memory.go @@ -11,6 +11,7 @@ import ( "time" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" pb "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" @@ -76,8 +77,38 @@ var ( // WedgeTimeout is the maximum amount of time the store lock may be // held before declaring a suspected deadlock. WedgeTimeout = 30 * time.Second + + // update()/write tx latency timer. + updateLatencyTimer metrics.Timer + + // view()/read tx latency timer. + viewLatencyTimer metrics.Timer + + // lookup() latency timer. + lookupLatencyTimer metrics.Timer + + // Batch() latency timer. + batchLatencyTimer metrics.Timer + + // timer to capture the duration for which the memory store mutex is locked. + storeLockDurationTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "store", nil) + updateLatencyTimer = ns.NewTimer("write_tx_latency", + "Raft store write tx latency.") + viewLatencyTimer = ns.NewTimer("read_tx_latency", + "Raft store read tx latency.") + lookupLatencyTimer = ns.NewTimer("lookup_latency", + "Raft store read latency.") + batchLatencyTimer = ns.NewTimer("batch_latency", + "Raft store batch latency.") + storeLockDurationTimer = ns.NewTimer("memory_store_lock_duration", + "Duration for which the raft memory store lock was held.") + metrics.Register(ns) +} + func register(os ObjectStoreConfig) { objectStorers = append(objectStorers, os) schema.Tables[os.Table.Name] = os.Table @@ -94,8 +125,13 @@ func (m *timedMutex) Lock() { m.lockedAt.Store(time.Now()) } +// Unlocks the timedMutex and captures the duration +// for which it was locked in a metric. func (m *timedMutex) Unlock() { + unlockedTimestamp := m.lockedAt.Load() m.Mutex.Unlock() + lockedFor := time.Since(unlockedTimestamp.(time.Time)) + storeLockDurationTimer.Update(lockedFor) m.lockedAt.Store(time.Time{}) } @@ -184,6 +220,7 @@ type readTx struct { // View executes a read transaction. func (s *MemoryStore) View(cb func(ReadTx)) { + defer metrics.StartTimer(viewLatencyTimer)() memDBTx := s.memDB.Txn(false) readTx := readTx{ @@ -280,6 +317,7 @@ func applyStoreAction(tx Tx, sa api.StoreAction) error { } func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { + defer metrics.StartTimer(updateLatencyTimer)() s.updateLock.Lock() memDBTx := s.memDB.Txn(true) @@ -329,7 +367,6 @@ func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { } s.updateLock.Unlock() return err - } func (s *MemoryStore) updateLocal(cb func(Tx) error) error { @@ -458,6 +495,7 @@ func (batch *Batch) commit() error { // If Batch returns an error, no guarantees are made about how many updates // were committed successfully. func (s *MemoryStore) Batch(cb func(*Batch) error) error { + defer metrics.StartTimer(batchLatencyTimer)() s.updateLock.Lock() batch := Batch{ @@ -498,6 +536,7 @@ func (tx tx) changelistStoreActions() ([]api.StoreAction, error) { // lookup is an internal typed wrapper around memdb. func (tx readTx) lookup(table, index, id string) api.StoreObject { + defer metrics.StartTimer(lookupLatencyTimer)() j, err := tx.memDBTx.First(table, index, id) if err != nil { return nil diff --git a/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go b/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go index 53bed49f1c..223dcb55d6 100644 --- a/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go +++ b/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go @@ -1,12 +1,11 @@ package watchapi import ( - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Watch starts a stream that returns any changes to objects that match @@ -26,7 +25,7 @@ func (s *Server) Watch(request *api.WatchRequest, stream api.Watch_WatchServer) watchArgs, err := api.ConvertWatchArgs(request.Entries) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } watchArgs = append(watchArgs, state.EventCommit{})