From 0682468431867e3382a759402eb92df5877e310b Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Fri, 3 Jun 2016 17:28:14 +0100 Subject: [PATCH 1/4] Make the docker proxy a standalone binary not a re-exec This reduces memory usage with a lot of docker proxy processes. On Docker for Mac we are currently carrying a patch to replace the binary as we modify it to forward ports to the Mac rather than the Linux VM, this allows us to simply replace this binary in our packaging with one that has a compatible interface. This patch does not provide an easy way to substitute a binary as the interface is complex and there are few use cases, but where needed this can be done. Signed-off-by: Justin Cormack --- hack/.vendor-helpers.sh | 3 + hack/make/.binary-setup | 1 + hack/make/.build-deb/rules | 1 + hack/make/.build-rpm/docker-engine.spec | 1 + hack/make/binary-daemon | 3 + hack/make/dynbinary-daemon | 3 + hack/make/gccgo | 18 ++ hack/make/install-binary-daemon | 1 + hack/make/tgz | 5 + pkg/proxy/network_proxy_test.go | 216 ------------------------ pkg/proxy/proxy.go | 37 ---- pkg/proxy/stub_proxy.go | 31 ---- pkg/proxy/tcp_proxy.go | 96 ----------- pkg/proxy/udp_proxy.go | 169 ------------------ 14 files changed, 36 insertions(+), 549 deletions(-) delete mode 100644 pkg/proxy/network_proxy_test.go delete mode 100644 pkg/proxy/proxy.go delete mode 100644 pkg/proxy/stub_proxy.go delete mode 100644 pkg/proxy/tcp_proxy.go delete mode 100644 pkg/proxy/udp_proxy.go diff --git a/hack/.vendor-helpers.sh b/hack/.vendor-helpers.sh index 94ed5cc5bb..82fe7b3b38 100755 --- a/hack/.vendor-helpers.sh +++ b/hack/.vendor-helpers.sh @@ -131,6 +131,9 @@ clean() { findArgs+=( -path "vendor/src/$import" ) done + # The docker proxy command is built from libnetwork + findArgs+=( -or -path vendor/src/github.com/docker/libnetwork/cmd/proxy ) + local IFS=$'\n' local prune=( $($find vendor -depth -type d -not '(' "${findArgs[@]}" ')') ) unset IFS diff --git a/hack/make/.binary-setup b/hack/make/.binary-setup index 90c8a336ea..e388c64485 100644 --- a/hack/make/.binary-setup +++ b/hack/make/.binary-setup @@ -2,3 +2,4 @@ DOCKER_CLIENT_BINARY_NAME='docker' DOCKER_DAEMON_BINARY_NAME='dockerd' +DOCKER_PROXY_BINARY_NAME='docker-proxy' diff --git a/hack/make/.build-deb/rules b/hack/make/.build-deb/rules index 2cd86691d3..2a4d6176ea 100755 --- a/hack/make/.build-deb/rules +++ b/hack/make/.build-deb/rules @@ -22,6 +22,7 @@ override_dh_auto_install: mkdir -p debian/docker-engine/usr/bin cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-client/docker)" debian/docker-engine/usr/bin/docker cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-daemon/dockerd)" debian/docker-engine/usr/bin/dockerd + cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-daemon/docker-proxy)" debian/docker-engine/usr/bin/docker-proxy cp -aT /usr/local/bin/containerd debian/docker-engine/usr/bin/docker-containerd cp -aT /usr/local/bin/containerd-shim debian/docker-engine/usr/bin/docker-containerd-shim cp -aT /usr/local/bin/ctr debian/docker-engine/usr/bin/docker-containerd-ctr diff --git a/hack/make/.build-rpm/docker-engine.spec b/hack/make/.build-rpm/docker-engine.spec index a433d54bc7..d3ae77f404 100644 --- a/hack/make/.build-rpm/docker-engine.spec +++ b/hack/make/.build-rpm/docker-engine.spec @@ -126,6 +126,7 @@ export DOCKER_GITCOMMIT=%{_gitcommit} install -d $RPM_BUILD_ROOT/%{_bindir} install -p -m 755 bundles/%{_origversion}/dynbinary-client/docker-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/docker install -p -m 755 bundles/%{_origversion}/dynbinary-daemon/dockerd-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/dockerd +install -p -m 755 bundles/%{_origversion}/dynbinary-daemon/docker-proxy-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/docker-proxy # install containerd install -p -m 755 /usr/local/bin/containerd $RPM_BUILD_ROOT/%{_bindir}/docker-containerd diff --git a/hack/make/binary-daemon b/hack/make/binary-daemon index bf2fb7b2ca..e75c44c587 100644 --- a/hack/make/binary-daemon +++ b/hack/make/binary-daemon @@ -9,5 +9,8 @@ set -e export BINARY_SHORT_NAME="$DOCKER_DAEMON_BINARY_NAME" export SOURCE_PATH='./cmd/dockerd' source "${MAKEDIR}/.binary" + export BINARY_SHORT_NAME="$DOCKER_PROXY_BINARY_NAME" + export SOURCE_PATH='./vendor/src/github.com/docker/libnetwork/cmd/proxy' + source "${MAKEDIR}/.binary" copy_containerd "$DEST" 'hash' ) diff --git a/hack/make/dynbinary-daemon b/hack/make/dynbinary-daemon index 398b951d85..cd7937859c 100644 --- a/hack/make/dynbinary-daemon +++ b/hack/make/dynbinary-daemon @@ -9,4 +9,7 @@ set -e export BUILDFLAGS=( "${BUILDFLAGS[@]/netgo /}" ) # disable netgo, since we don't need it for a dynamic binary export BUILDFLAGS=( "${BUILDFLAGS[@]/static_build /}" ) # we're not building a "static" binary here source "${MAKEDIR}/.binary" + export BINARY_SHORT_NAME='docker-proxy' + export SOURCE_PATH='./vendor/src/github.com/docker/libnetwork/cmd/proxy' + source "${MAKEDIR}/.binary" ) diff --git a/hack/make/gccgo b/hack/make/gccgo index 93e064a879..54c983eb2e 100644 --- a/hack/make/gccgo +++ b/hack/make/gccgo @@ -5,6 +5,9 @@ BINARY_NAME="dockerd-$VERSION" BINARY_EXTENSION="$(binary_extension)" BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION" +PROXY_NAME="docker-proxy-$VERSION" +PROXY_FULLNAME="$PROXY_NAME$BINARY_EXTENSION" + CLIENTBIN_NAME="docker-$VERSION" CLIENTBIN_FULLNAME="$CLIENTBIN_NAME$BINARY_EXTENSION" @@ -29,6 +32,21 @@ go build -compiler=gccgo \ echo "Created binary: $DEST/$BINARY_FULLNAME" ln -sf "$BINARY_FULLNAME" "$DEST/dockerd$BINARY_EXTENSION" +go build -compiler=gccgo \ + -o "$DEST/$PROXY_FULLNAME" \ + "${BUILDFLAGS[@]}" \ + -gccgoflags " + -g + $EXTLDFLAGS_STATIC + -Wl,--no-export-dynamic + -ldl + -pthread + " \ + ./vendor/src/github.com/docker/libnetwork/cmd/proxy + +echo "Created binary: $DEST/$PROXY_FULLNAME" +ln -sf "$PROXY_FULLNAME" "$DEST/docker-proxy$BINARY_EXTENSION" + copy_containerd "$DEST" "hash" hash_files "$DEST/$BINARY_FULLNAME" diff --git a/hack/make/install-binary-daemon b/hack/make/install-binary-daemon index 36b6c5e95a..e80d8431fd 100644 --- a/hack/make/install-binary-daemon +++ b/hack/make/install-binary-daemon @@ -7,4 +7,5 @@ rm -rf "$DEST" DEST="$(dirname $DEST)/binary-daemon" source "${MAKEDIR}/.binary-setup" install_binary "${DEST}/${DOCKER_DAEMON_BINARY_NAME}" + install_binary "${DEST}/${DOCKER_PROXY_BINARY_NAME}" ) diff --git a/hack/make/tgz b/hack/make/tgz index 5b1663256b..b1abfef92f 100644 --- a/hack/make/tgz +++ b/hack/make/tgz @@ -18,6 +18,7 @@ for d in "$CROSS/"*/*; do BINARY_NAME="${DOCKER_CLIENT_BINARY_NAME}-$VERSION" DAEMON_BINARY_NAME="${DOCKER_DAEMON_BINARY_NAME}-$VERSION" + PROXY_BINARY_NAME="${DOCKER_PROXY_BINARY_NAME}-$VERSION" BINARY_EXTENSION="$(export GOOS && binary_extension)" if [ "$GOOS" = 'windows' ]; then # if windows use a zip, not tgz @@ -29,6 +30,7 @@ for d in "$CROSS/"*/*; do fi BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION" DAEMON_BINARY_FULLNAME="$DAEMON_BINARY_NAME$BINARY_EXTENSION" + PROXY_BINARY_FULLNAME="$PROXY_BINARY_NAME$BINARY_EXTENSION" mkdir -p "$DEST/$GOOS/$GOARCH" TGZ="$DEST/$GOOS/$GOARCH/$BINARY_NAME$BUNDLE_EXTENSION" @@ -47,6 +49,9 @@ for d in "$CROSS/"*/*; do if [ -f "$d/$DAEMON_BINARY_FULLNAME" ]; then cp -L "$d/$DAEMON_BINARY_FULLNAME" "$TAR_PATH/${DOCKER_DAEMON_BINARY_NAME}${BINARY_EXTENSION}" fi + if [ -f "$d/$PROXY_BINARY_FULLNAME" ]; then + cp -L "$d/$PROXY_BINARY_FULLNAME" "$TAR_PATH/${DOCKER_PROXY_BINARY_NAME}${BINARY_EXTENSION}" + fi # copy over all the containerd binaries copy_containerd $TAR_PATH diff --git a/pkg/proxy/network_proxy_test.go b/pkg/proxy/network_proxy_test.go deleted file mode 100644 index 9a73548019..0000000000 --- a/pkg/proxy/network_proxy_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package proxy - -import ( - "bytes" - "fmt" - "io" - "net" - "strings" - "testing" - "time" -) - -var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo") -var testBufSize = len(testBuf) - -type EchoServer interface { - Run() - Close() - LocalAddr() net.Addr -} - -type TCPEchoServer struct { - listener net.Listener - testCtx *testing.T -} - -type UDPEchoServer struct { - conn net.PacketConn - testCtx *testing.T -} - -func NewEchoServer(t *testing.T, proto, address string) EchoServer { - var server EchoServer - if strings.HasPrefix(proto, "tcp") { - listener, err := net.Listen(proto, address) - if err != nil { - t.Fatal(err) - } - server = &TCPEchoServer{listener: listener, testCtx: t} - } else { - socket, err := net.ListenPacket(proto, address) - if err != nil { - t.Fatal(err) - } - server = &UDPEchoServer{conn: socket, testCtx: t} - } - return server -} - -func (server *TCPEchoServer) Run() { - go func() { - for { - client, err := server.listener.Accept() - if err != nil { - return - } - go func(client net.Conn) { - if _, err := io.Copy(client, client); err != nil { - server.testCtx.Logf("can't echo to the client: %v\n", err.Error()) - } - client.Close() - }(client) - } - }() -} - -func (server *TCPEchoServer) LocalAddr() net.Addr { return server.listener.Addr() } -func (server *TCPEchoServer) Close() { server.listener.Close() } - -func (server *UDPEchoServer) Run() { - go func() { - readBuf := make([]byte, 1024) - for { - read, from, err := server.conn.ReadFrom(readBuf) - if err != nil { - return - } - for i := 0; i != read; { - written, err := server.conn.WriteTo(readBuf[i:read], from) - if err != nil { - break - } - i += written - } - } - }() -} - -func (server *UDPEchoServer) LocalAddr() net.Addr { return server.conn.LocalAddr() } -func (server *UDPEchoServer) Close() { server.conn.Close() } - -func testProxyAt(t *testing.T, proto string, proxy Proxy, addr string) { - defer proxy.Close() - go proxy.Run() - client, err := net.Dial(proto, addr) - if err != nil { - t.Fatalf("Can't connect to the proxy: %v", err) - } - defer client.Close() - client.SetDeadline(time.Now().Add(10 * time.Second)) - if _, err = client.Write(testBuf); err != nil { - t.Fatal(err) - } - recvBuf := make([]byte, testBufSize) - if _, err = client.Read(recvBuf); err != nil { - t.Fatal(err) - } - if !bytes.Equal(testBuf, recvBuf) { - t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) - } -} - -func testProxy(t *testing.T, proto string, proxy Proxy) { - testProxyAt(t, proto, proxy, proxy.FrontendAddr().String()) -} - -func TestTCP4Proxy(t *testing.T) { - backend := NewEchoServer(t, "tcp", "127.0.0.1:0") - defer backend.Close() - backend.Run() - frontendAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} - proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) - if err != nil { - t.Fatal(err) - } - testProxy(t, "tcp", proxy) -} - -func TestTCP6Proxy(t *testing.T) { - backend := NewEchoServer(t, "tcp", "[::1]:0") - defer backend.Close() - backend.Run() - frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} - proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) - if err != nil { - t.Fatal(err) - } - testProxy(t, "tcp", proxy) -} - -func TestTCPDualStackProxy(t *testing.T) { - // If I understand `godoc -src net favoriteAddrFamily` (used by the - // net.Listen* functions) correctly this should work, but it doesn't. - t.Skip("No support for dual stack yet") - backend := NewEchoServer(t, "tcp", "[::1]:0") - defer backend.Close() - backend.Run() - frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} - proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) - if err != nil { - t.Fatal(err) - } - ipv4ProxyAddr := &net.TCPAddr{ - IP: net.IPv4(127, 0, 0, 1), - Port: proxy.FrontendAddr().(*net.TCPAddr).Port, - } - testProxyAt(t, "tcp", proxy, ipv4ProxyAddr.String()) -} - -func TestUDP4Proxy(t *testing.T) { - backend := NewEchoServer(t, "udp", "127.0.0.1:0") - defer backend.Close() - backend.Run() - frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} - proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) - if err != nil { - t.Fatal(err) - } - testProxy(t, "udp", proxy) -} - -func TestUDP6Proxy(t *testing.T) { - backend := NewEchoServer(t, "udp", "[::1]:0") - defer backend.Close() - backend.Run() - frontendAddr := &net.UDPAddr{IP: net.IPv6loopback, Port: 0} - proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) - if err != nil { - t.Fatal(err) - } - testProxy(t, "udp", proxy) -} - -func TestUDPWriteError(t *testing.T) { - frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} - // Hopefully, this port will be free: */ - backendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 25587} - proxy, err := NewProxy(frontendAddr, backendAddr) - if err != nil { - t.Fatal(err) - } - defer proxy.Close() - go proxy.Run() - client, err := net.Dial("udp", "127.0.0.1:25587") - if err != nil { - t.Fatalf("Can't connect to the proxy: %v", err) - } - defer client.Close() - // Make sure the proxy doesn't stop when there is no actual backend: - client.Write(testBuf) - client.Write(testBuf) - backend := NewEchoServer(t, "udp", "127.0.0.1:25587") - defer backend.Close() - backend.Run() - client.SetDeadline(time.Now().Add(10 * time.Second)) - if _, err = client.Write(testBuf); err != nil { - t.Fatal(err) - } - recvBuf := make([]byte, testBufSize) - if _, err = client.Read(recvBuf); err != nil { - t.Fatal(err) - } - if !bytes.Equal(testBuf, recvBuf) { - t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) - } -} diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go deleted file mode 100644 index 4e24e5f6a8..0000000000 --- a/pkg/proxy/proxy.go +++ /dev/null @@ -1,37 +0,0 @@ -// Package proxy provides a network Proxy interface and implementations for TCP -// and UDP. -package proxy - -import ( - "fmt" - "net" -) - -// Proxy defines the behavior of a proxy. It forwards traffic back and forth -// between two endpoints : the frontend and the backend. -// It can be used to do software port-mapping between two addresses. -// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000 -// to the backend (container) at 172.17.42.108:4000. -type Proxy interface { - // Run starts forwarding traffic back and forth between the front - // and back-end addresses. - Run() - // Close stops forwarding traffic and close both ends of the Proxy. - Close() - // FrontendAddr returns the address on which the proxy is listening. - FrontendAddr() net.Addr - // BackendAddr returns the proxied address. - BackendAddr() net.Addr -} - -// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr. -func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { - switch frontendAddr.(type) { - case *net.UDPAddr: - return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr)) - case *net.TCPAddr: - return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr)) - default: - panic(fmt.Errorf("Unsupported protocol")) - } -} diff --git a/pkg/proxy/stub_proxy.go b/pkg/proxy/stub_proxy.go deleted file mode 100644 index 571749e467..0000000000 --- a/pkg/proxy/stub_proxy.go +++ /dev/null @@ -1,31 +0,0 @@ -package proxy - -import ( - "net" -) - -// StubProxy is a proxy that is a stub (does nothing). -type StubProxy struct { - frontendAddr net.Addr - backendAddr net.Addr -} - -// Run does nothing. -func (p *StubProxy) Run() {} - -// Close does nothing. -func (p *StubProxy) Close() {} - -// FrontendAddr returns the frontend address. -func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr } - -// BackendAddr returns the backend address. -func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr } - -// NewStubProxy creates a new StubProxy -func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { - return &StubProxy{ - frontendAddr: frontendAddr, - backendAddr: backendAddr, - }, nil -} diff --git a/pkg/proxy/tcp_proxy.go b/pkg/proxy/tcp_proxy.go deleted file mode 100644 index 8f42580c1b..0000000000 --- a/pkg/proxy/tcp_proxy.go +++ /dev/null @@ -1,96 +0,0 @@ -package proxy - -import ( - "io" - "net" - "sync" - "syscall" - - "github.com/Sirupsen/logrus" -) - -// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to -// handle TCP traffic forwarding between the frontend and backend addresses. -type TCPProxy struct { - listener *net.TCPListener - frontendAddr *net.TCPAddr - backendAddr *net.TCPAddr -} - -// NewTCPProxy creates a new TCPProxy. -func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { - listener, err := net.ListenTCP("tcp", frontendAddr) - if err != nil { - return nil, err - } - // If the port in frontendAddr was 0 then ListenTCP will have a picked - // a port to listen on, hence the call to Addr to get that actual port: - return &TCPProxy{ - listener: listener, - frontendAddr: listener.Addr().(*net.TCPAddr), - backendAddr: backendAddr, - }, nil -} - -func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { - backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) - if err != nil { - logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) - client.Close() - return - } - - var wg sync.WaitGroup - var broker = func(to, from *net.TCPConn) { - if _, err := io.Copy(to, from); err != nil { - // If the socket we are writing to is shutdown with - // SHUT_WR, forward it to the other end of the pipe: - if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE { - from.CloseWrite() - } - } - to.CloseRead() - wg.Done() - } - - wg.Add(2) - go broker(client, backend) - go broker(backend, client) - - finish := make(chan struct{}) - go func() { - wg.Wait() - close(finish) - }() - - select { - case <-quit: - case <-finish: - } - client.Close() - backend.Close() - <-finish -} - -// Run starts forwarding the traffic using TCP. -func (proxy *TCPProxy) Run() { - quit := make(chan bool) - defer close(quit) - for { - client, err := proxy.listener.Accept() - if err != nil { - logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) - return - } - go proxy.clientLoop(client.(*net.TCPConn), quit) - } -} - -// Close stops forwarding the traffic. -func (proxy *TCPProxy) Close() { proxy.listener.Close() } - -// FrontendAddr returns the TCP address on which the proxy is listening. -func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } - -// BackendAddr returns the TCP proxied address. -func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr } diff --git a/pkg/proxy/udp_proxy.go b/pkg/proxy/udp_proxy.go deleted file mode 100644 index b8375c374f..0000000000 --- a/pkg/proxy/udp_proxy.go +++ /dev/null @@ -1,169 +0,0 @@ -package proxy - -import ( - "encoding/binary" - "net" - "strings" - "sync" - "syscall" - "time" - - "github.com/Sirupsen/logrus" -) - -const ( - // UDPConnTrackTimeout is the timeout used for UDP connection tracking - UDPConnTrackTimeout = 90 * time.Second - // UDPBufSize is the buffer size for the UDP proxy - UDPBufSize = 65507 -) - -// A net.Addr where the IP is split into two fields so you can use it as a key -// in a map: -type connTrackKey struct { - IPHigh uint64 - IPLow uint64 - Port int -} - -func newConnTrackKey(addr *net.UDPAddr) *connTrackKey { - if len(addr.IP) == net.IPv4len { - return &connTrackKey{ - IPHigh: 0, - IPLow: uint64(binary.BigEndian.Uint32(addr.IP)), - Port: addr.Port, - } - } - return &connTrackKey{ - IPHigh: binary.BigEndian.Uint64(addr.IP[:8]), - IPLow: binary.BigEndian.Uint64(addr.IP[8:]), - Port: addr.Port, - } -} - -type connTrackMap map[connTrackKey]*net.UDPConn - -// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy -// interface to handle UDP traffic forwarding between the frontend and backend -// addresses. -type UDPProxy struct { - listener *net.UDPConn - frontendAddr *net.UDPAddr - backendAddr *net.UDPAddr - connTrackTable connTrackMap - connTrackLock sync.Mutex -} - -// NewUDPProxy creates a new UDPProxy. -func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) { - listener, err := net.ListenUDP("udp", frontendAddr) - if err != nil { - return nil, err - } - return &UDPProxy{ - listener: listener, - frontendAddr: listener.LocalAddr().(*net.UDPAddr), - backendAddr: backendAddr, - connTrackTable: make(connTrackMap), - }, nil -} - -func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) { - defer func() { - proxy.connTrackLock.Lock() - delete(proxy.connTrackTable, *clientKey) - proxy.connTrackLock.Unlock() - proxyConn.Close() - }() - - readBuf := make([]byte, UDPBufSize) - for { - proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout)) - again: - read, err := proxyConn.Read(readBuf) - if err != nil { - if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED { - // This will happen if the last write failed - // (e.g: nothing is actually listening on the - // proxied port on the container), ignore it - // and continue until UDPConnTrackTimeout - // expires: - goto again - } - return - } - for i := 0; i != read; { - written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr) - if err != nil { - return - } - i += written - } - } -} - -// Run starts forwarding the traffic using UDP. -func (proxy *UDPProxy) Run() { - readBuf := make([]byte, UDPBufSize) - for { - read, from, err := proxy.listener.ReadFromUDP(readBuf) - if err != nil { - // NOTE: Apparently ReadFrom doesn't return - // ECONNREFUSED like Read do (see comment in - // UDPProxy.replyLoop) - if !isClosedError(err) { - logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) - } - break - } - - fromKey := newConnTrackKey(from) - proxy.connTrackLock.Lock() - proxyConn, hit := proxy.connTrackTable[*fromKey] - if !hit { - proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) - if err != nil { - logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) - proxy.connTrackLock.Unlock() - continue - } - proxy.connTrackTable[*fromKey] = proxyConn - go proxy.replyLoop(proxyConn, from, fromKey) - } - proxy.connTrackLock.Unlock() - for i := 0; i != read; { - written, err := proxyConn.Write(readBuf[i:read]) - if err != nil { - logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) - break - } - i += written - } - } -} - -// Close stops forwarding the traffic. -func (proxy *UDPProxy) Close() { - proxy.listener.Close() - proxy.connTrackLock.Lock() - defer proxy.connTrackLock.Unlock() - for _, conn := range proxy.connTrackTable { - conn.Close() - } -} - -// FrontendAddr returns the UDP address on which the proxy is listening. -func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } - -// BackendAddr returns the proxied UDP address. -func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr } - -func isClosedError(err error) bool { - /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. - * See: - * http://golang.org/src/pkg/net/net.go - * https://code.google.com/p/go/issues/detail?id=4337 - * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ - */ - return strings.HasSuffix(err.Error(), "use of closed network connection") -} From 3ae0c664507e0d8da5e3151f28b8d00316569e47 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Thu, 7 Jul 2016 12:23:10 +0100 Subject: [PATCH 2/4] Update libnetwork vendoring Update now that the changes in https://github.com/docker/libnetwork/pull/1230 have been merged Signed-off-by: Justin Cormack --- hack/vendor.sh | 2 +- .../src/github.com/docker/libnetwork/Makefile | 7 +- .../docker/libnetwork/cmd/proxy/main.go | 67 +++++++ .../docker/libnetwork/cmd/proxy/proxy.go | 37 ++++ .../docker/libnetwork/cmd/proxy/stub_proxy.go | 31 ++++ .../docker/libnetwork/cmd/proxy/tcp_proxy.go | 96 ++++++++++ .../docker/libnetwork/cmd/proxy/udp_proxy.go | 169 ++++++++++++++++++ .../docker/libnetwork/controller.go | 5 + .../libnetwork/drivers/bridge/bridge.go | 2 +- .../docker/libnetwork/drivers_ipam.go | 23 +++ .../libnetwork/drvregistry/drvregistry.go | 22 --- .../docker/libnetwork/etchosts/etchosts.go | 21 ++- .../docker/libnetwork/networkdb/cluster.go | 2 +- .../docker/libnetwork/networkdb/delegate.go | 5 + .../docker/libnetwork/portmapper/mapper.go | 10 +- .../libnetwork/portmapper/mock_proxy.go | 4 +- .../docker/libnetwork/portmapper/proxy.go | 73 +------- .../docker/libnetwork/sandbox_dns_unix.go | 10 +- .../libnetwork/sandbox_externalkey_unix.go | 2 + 19 files changed, 483 insertions(+), 105 deletions(-) create mode 100644 vendor/src/github.com/docker/libnetwork/cmd/proxy/main.go create mode 100644 vendor/src/github.com/docker/libnetwork/cmd/proxy/proxy.go create mode 100644 vendor/src/github.com/docker/libnetwork/cmd/proxy/stub_proxy.go create mode 100644 vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go create mode 100644 vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go create mode 100644 vendor/src/github.com/docker/libnetwork/drivers_ipam.go diff --git a/hack/vendor.sh b/hack/vendor.sh index 330b2491ad..2bcc3a6664 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837 clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages -clone git github.com/docker/libnetwork 377a7337f2387cce3be1df7a4503446147b68ff1 +clone git github.com/docker/libnetwork 6eece7dcc21dcd34d907f3e91dd71cb8640b661c clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/vendor/src/github.com/docker/libnetwork/Makefile b/vendor/src/github.com/docker/libnetwork/Makefile index 899ca4b9be..a130dc656d 100644 --- a/vendor/src/github.com/docker/libnetwork/Makefile +++ b/vendor/src/github.com/docker/libnetwork/Makefile @@ -7,6 +7,7 @@ docker = docker run --rm -it ${dockerargs} $$EXTRA_ARGS ${container_env} ${build ciargs = -e CIRCLECI -e "COVERALLS_TOKEN=$$COVERALLS_TOKEN" -e "INSIDECONTAINER=-incontainer=true" cidocker = docker run ${dockerargs} ${ciargs} $$EXTRA_ARGS ${container_env} ${build_image} CROSS_PLATFORMS = linux/amd64 linux/386 linux/arm windows/amd64 +export PATH := $(CURDIR)/bin:$(PATH) all: ${build_image}.created build check integration-tests clean @@ -24,10 +25,11 @@ build: ${build_image}.created build-local: @mkdir -p "bin" $(shell which godep) go build -tags experimental -o "bin/dnet" ./cmd/dnet + $(shell which godep) go build -o "bin/docker-proxy" ./cmd/proxy clean: @if [ -d bin ]; then \ - echo "Removing dnet binaries"; \ + echo "Removing dnet and proxy binaries"; \ rm -rf bin; \ fi @@ -41,6 +43,7 @@ cross: ${build_image}.created cross-local: $(shell which godep) go build -o "bin/dnet-$$GOOS-$$GOARCH" ./cmd/dnet + $(shell which godep) go build -o "bin/docker-proxy-$$GOOS-$$GOARCH" ./cmd/proxy check: ${build_image}.created @${docker} ./wrapmake.sh check-local @@ -102,4 +105,4 @@ circle-ci-check: ${build_image}.created circle-ci-build: ${build_image}.created @${cidocker} make build-local -circle-ci: circle-ci-check circle-ci-cross circle-ci-build integration-tests +circle-ci: circle-ci-build circle-ci-check circle-ci-cross integration-tests diff --git a/vendor/src/github.com/docker/libnetwork/cmd/proxy/main.go b/vendor/src/github.com/docker/libnetwork/cmd/proxy/main.go new file mode 100644 index 0000000000..fb68796148 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/cmd/proxy/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "os/signal" + "syscall" +) + +func main() { + f := os.NewFile(3, "signal-parent") + host, container := parseHostContainerAddrs() + + p, err := NewProxy(host, container) + if err != nil { + fmt.Fprintf(f, "1\n%s", err) + f.Close() + os.Exit(1) + } + go handleStopSignals(p) + fmt.Fprint(f, "0\n") + f.Close() + + // Run will block until the proxy stops + p.Run() +} + +// parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP +// net.Addrs to map the host and container ports +func parseHostContainerAddrs() (host net.Addr, container net.Addr) { + var ( + proto = flag.String("proto", "tcp", "proxy protocol") + hostIP = flag.String("host-ip", "", "host ip") + hostPort = flag.Int("host-port", -1, "host port") + containerIP = flag.String("container-ip", "", "container ip") + containerPort = flag.Int("container-port", -1, "container port") + ) + + flag.Parse() + + switch *proto { + case "tcp": + host = &net.TCPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort} + container = &net.TCPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort} + case "udp": + host = &net.UDPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort} + container = &net.UDPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort} + default: + log.Fatalf("unsupported protocol %s", *proto) + } + + return host, container +} + +func handleStopSignals(p Proxy) { + s := make(chan os.Signal, 10) + signal.Notify(s, os.Interrupt, syscall.SIGTERM) + + for range s { + p.Close() + + os.Exit(0) + } +} diff --git a/vendor/src/github.com/docker/libnetwork/cmd/proxy/proxy.go b/vendor/src/github.com/docker/libnetwork/cmd/proxy/proxy.go new file mode 100644 index 0000000000..44dd177bb6 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/cmd/proxy/proxy.go @@ -0,0 +1,37 @@ +// docker-proxy provides a network Proxy interface and implementations for TCP +// and UDP. +package main + +import ( + "fmt" + "net" +) + +// Proxy defines the behavior of a proxy. It forwards traffic back and forth +// between two endpoints : the frontend and the backend. +// It can be used to do software port-mapping between two addresses. +// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000 +// to the backend (container) at 172.17.42.108:4000. +type Proxy interface { + // Run starts forwarding traffic back and forth between the front + // and back-end addresses. + Run() + // Close stops forwarding traffic and close both ends of the Proxy. + Close() + // FrontendAddr returns the address on which the proxy is listening. + FrontendAddr() net.Addr + // BackendAddr returns the proxied address. + BackendAddr() net.Addr +} + +// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr. +func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { + switch frontendAddr.(type) { + case *net.UDPAddr: + return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr)) + case *net.TCPAddr: + return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr)) + default: + panic(fmt.Errorf("Unsupported protocol")) + } +} diff --git a/vendor/src/github.com/docker/libnetwork/cmd/proxy/stub_proxy.go b/vendor/src/github.com/docker/libnetwork/cmd/proxy/stub_proxy.go new file mode 100644 index 0000000000..95e0e36352 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/cmd/proxy/stub_proxy.go @@ -0,0 +1,31 @@ +package main + +import ( + "net" +) + +// StubProxy is a proxy that is a stub (does nothing). +type StubProxy struct { + frontendAddr net.Addr + backendAddr net.Addr +} + +// Run does nothing. +func (p *StubProxy) Run() {} + +// Close does nothing. +func (p *StubProxy) Close() {} + +// FrontendAddr returns the frontend address. +func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr } + +// BackendAddr returns the backend address. +func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr } + +// NewStubProxy creates a new StubProxy +func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { + return &StubProxy{ + frontendAddr: frontendAddr, + backendAddr: backendAddr, + }, nil +} diff --git a/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go b/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go new file mode 100644 index 0000000000..514b778773 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go @@ -0,0 +1,96 @@ +package main + +import ( + "io" + "net" + "sync" + "syscall" + + "github.com/Sirupsen/logrus" +) + +// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to +// handle TCP traffic forwarding between the frontend and backend addresses. +type TCPProxy struct { + listener *net.TCPListener + frontendAddr *net.TCPAddr + backendAddr *net.TCPAddr +} + +// NewTCPProxy creates a new TCPProxy. +func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { + listener, err := net.ListenTCP("tcp", frontendAddr) + if err != nil { + return nil, err + } + // If the port in frontendAddr was 0 then ListenTCP will have a picked + // a port to listen on, hence the call to Addr to get that actual port: + return &TCPProxy{ + listener: listener, + frontendAddr: listener.Addr().(*net.TCPAddr), + backendAddr: backendAddr, + }, nil +} + +func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { + backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) + if err != nil { + logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) + client.Close() + return + } + + var wg sync.WaitGroup + var broker = func(to, from *net.TCPConn) { + if _, err := io.Copy(to, from); err != nil { + // If the socket we are writing to is shutdown with + // SHUT_WR, forward it to the other end of the pipe: + if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE { + from.CloseWrite() + } + } + to.CloseRead() + wg.Done() + } + + wg.Add(2) + go broker(client, backend) + go broker(backend, client) + + finish := make(chan struct{}) + go func() { + wg.Wait() + close(finish) + }() + + select { + case <-quit: + case <-finish: + } + client.Close() + backend.Close() + <-finish +} + +// Run starts forwarding the traffic using TCP. +func (proxy *TCPProxy) Run() { + quit := make(chan bool) + defer close(quit) + for { + client, err := proxy.listener.Accept() + if err != nil { + logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + return + } + go proxy.clientLoop(client.(*net.TCPConn), quit) + } +} + +// Close stops forwarding the traffic. +func (proxy *TCPProxy) Close() { proxy.listener.Close() } + +// FrontendAddr returns the TCP address on which the proxy is listening. +func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } + +// BackendAddr returns the TCP proxied address. +func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr } diff --git a/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go b/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go new file mode 100644 index 0000000000..2b895ae9d4 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go @@ -0,0 +1,169 @@ +package main + +import ( + "encoding/binary" + "net" + "strings" + "sync" + "syscall" + "time" + + "github.com/Sirupsen/logrus" +) + +const ( + // UDPConnTrackTimeout is the timeout used for UDP connection tracking + UDPConnTrackTimeout = 90 * time.Second + // UDPBufSize is the buffer size for the UDP proxy + UDPBufSize = 65507 +) + +// A net.Addr where the IP is split into two fields so you can use it as a key +// in a map: +type connTrackKey struct { + IPHigh uint64 + IPLow uint64 + Port int +} + +func newConnTrackKey(addr *net.UDPAddr) *connTrackKey { + if len(addr.IP) == net.IPv4len { + return &connTrackKey{ + IPHigh: 0, + IPLow: uint64(binary.BigEndian.Uint32(addr.IP)), + Port: addr.Port, + } + } + return &connTrackKey{ + IPHigh: binary.BigEndian.Uint64(addr.IP[:8]), + IPLow: binary.BigEndian.Uint64(addr.IP[8:]), + Port: addr.Port, + } +} + +type connTrackMap map[connTrackKey]*net.UDPConn + +// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy +// interface to handle UDP traffic forwarding between the frontend and backend +// addresses. +type UDPProxy struct { + listener *net.UDPConn + frontendAddr *net.UDPAddr + backendAddr *net.UDPAddr + connTrackTable connTrackMap + connTrackLock sync.Mutex +} + +// NewUDPProxy creates a new UDPProxy. +func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) { + listener, err := net.ListenUDP("udp", frontendAddr) + if err != nil { + return nil, err + } + return &UDPProxy{ + listener: listener, + frontendAddr: listener.LocalAddr().(*net.UDPAddr), + backendAddr: backendAddr, + connTrackTable: make(connTrackMap), + }, nil +} + +func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) { + defer func() { + proxy.connTrackLock.Lock() + delete(proxy.connTrackTable, *clientKey) + proxy.connTrackLock.Unlock() + proxyConn.Close() + }() + + readBuf := make([]byte, UDPBufSize) + for { + proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout)) + again: + read, err := proxyConn.Read(readBuf) + if err != nil { + if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED { + // This will happen if the last write failed + // (e.g: nothing is actually listening on the + // proxied port on the container), ignore it + // and continue until UDPConnTrackTimeout + // expires: + goto again + } + return + } + for i := 0; i != read; { + written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr) + if err != nil { + return + } + i += written + } + } +} + +// Run starts forwarding the traffic using UDP. +func (proxy *UDPProxy) Run() { + readBuf := make([]byte, UDPBufSize) + for { + read, from, err := proxy.listener.ReadFromUDP(readBuf) + if err != nil { + // NOTE: Apparently ReadFrom doesn't return + // ECONNREFUSED like Read do (see comment in + // UDPProxy.replyLoop) + if !isClosedError(err) { + logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + } + break + } + + fromKey := newConnTrackKey(from) + proxy.connTrackLock.Lock() + proxyConn, hit := proxy.connTrackTable[*fromKey] + if !hit { + proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) + if err != nil { + logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + proxy.connTrackLock.Unlock() + continue + } + proxy.connTrackTable[*fromKey] = proxyConn + go proxy.replyLoop(proxyConn, from, fromKey) + } + proxy.connTrackLock.Unlock() + for i := 0; i != read; { + written, err := proxyConn.Write(readBuf[i:read]) + if err != nil { + logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + break + } + i += written + } + } +} + +// Close stops forwarding the traffic. +func (proxy *UDPProxy) Close() { + proxy.listener.Close() + proxy.connTrackLock.Lock() + defer proxy.connTrackLock.Unlock() + for _, conn := range proxy.connTrackTable { + conn.Close() + } +} + +// FrontendAddr returns the UDP address on which the proxy is listening. +func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } + +// BackendAddr returns the proxied UDP address. +func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr } + +func isClosedError(err error) bool { + /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. + * See: + * http://golang.org/src/pkg/net/net.go + * https://code.google.com/p/go/issues/detail?id=4337 + * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ + */ + return strings.HasSuffix(err.Error(), "use of closed network connection") +} diff --git a/vendor/src/github.com/docker/libnetwork/controller.go b/vendor/src/github.com/docker/libnetwork/controller.go index 9108d87844..216f7ccd10 100644 --- a/vendor/src/github.com/docker/libnetwork/controller.go +++ b/vendor/src/github.com/docker/libnetwork/controller.go @@ -193,6 +193,11 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { return nil, err } } + + if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope)); err != nil { + return nil, err + } + c.drvRegistry = drvRegistry if c.cfg != nil && c.cfg.Cluster.Watcher != nil { diff --git a/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go b/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go index c75ec9f4f9..d2b1d24829 100644 --- a/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -330,7 +330,7 @@ func (c *networkConfiguration) conflictsWithNetworks(id string, others []*bridge // bridges. This could not be completely caught by the config conflict // check, because networks which config does not specify the AddressIPv4 // get their address and subnet selected by the driver (see electBridgeIPv4()) - if c.AddressIPv4 != nil { + if c.AddressIPv4 != nil && nwBridge.bridgeIPv4 != nil { if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) || c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) { return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName) diff --git a/vendor/src/github.com/docker/libnetwork/drivers_ipam.go b/vendor/src/github.com/docker/libnetwork/drivers_ipam.go new file mode 100644 index 0000000000..be0204ab10 --- /dev/null +++ b/vendor/src/github.com/docker/libnetwork/drivers_ipam.go @@ -0,0 +1,23 @@ +package libnetwork + +import ( + "github.com/docker/libnetwork/drvregistry" + "github.com/docker/libnetwork/ipamapi" + builtinIpam "github.com/docker/libnetwork/ipams/builtin" + nullIpam "github.com/docker/libnetwork/ipams/null" + remoteIpam "github.com/docker/libnetwork/ipams/remote" +) + +func initIPAMDrivers(r *drvregistry.DrvRegistry, lDs, gDs interface{}) error { + for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){ + builtinIpam.Init, + remoteIpam.Init, + nullIpam.Init, + } { + if err := fn(r, lDs, gDs); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/src/github.com/docker/libnetwork/drvregistry/drvregistry.go b/vendor/src/github.com/docker/libnetwork/drvregistry/drvregistry.go index f12da15c0b..d2cf781193 100644 --- a/vendor/src/github.com/docker/libnetwork/drvregistry/drvregistry.go +++ b/vendor/src/github.com/docker/libnetwork/drvregistry/drvregistry.go @@ -8,10 +8,6 @@ import ( "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/types" - - builtinIpam "github.com/docker/libnetwork/ipams/builtin" - nullIpam "github.com/docker/libnetwork/ipams/null" - remoteIpam "github.com/docker/libnetwork/ipams/remote" ) type driverData struct { @@ -64,10 +60,6 @@ func New(lDs, gDs interface{}, dfn DriverNotifyFunc, ifn IPAMNotifyFunc) (*DrvRe ifn: ifn, } - if err := r.initIPAMs(lDs, gDs); err != nil { - return nil, err - } - return r, nil } @@ -157,20 +149,6 @@ func (r *DrvRegistry) IPAMDefaultAddressSpaces(name string) (string, string, err return i.defaultLocalAddressSpace, i.defaultGlobalAddressSpace, nil } -func (r *DrvRegistry) initIPAMs(lDs, gDs interface{}) error { - for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){ - builtinIpam.Init, - remoteIpam.Init, - nullIpam.Init, - } { - if err := fn(r, nil, gDs); err != nil { - return err - } - } - - return nil -} - // RegisterDriver registers the network driver when it gets discovered. func (r *DrvRegistry) RegisterDriver(ntype string, driver driverapi.Driver, capability driverapi.Capability) error { if strings.TrimSpace(ntype) == "" { diff --git a/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go b/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go index 4526532593..d55298af45 100644 --- a/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go +++ b/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go @@ -119,25 +119,34 @@ func Add(path string, recs []Record) error { return nil } - f, err := os.Open(path) + b, err := mergeRecords(path, recs) if err != nil { return err } + return ioutil.WriteFile(path, b, 0644) +} + +func mergeRecords(path string, recs []Record) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + content := bytes.NewBuffer(nil) - _, err = content.ReadFrom(f) - if err != nil { - return err + if _, err := content.ReadFrom(f); err != nil { + return nil, err } for _, r := range recs { if _, err := r.WriteTo(content); err != nil { - return err + return nil, err } } - return ioutil.WriteFile(path, content.Bytes(), 0644) + return content.Bytes(), nil } // Delete deletes an arbitrary number of Records already existing in /etc/hosts file diff --git a/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go b/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go index e8340401e5..a627135fc3 100644 --- a/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go +++ b/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go @@ -14,7 +14,7 @@ import ( "github.com/hashicorp/memberlist" ) -const reapInterval = 2 * time.Second +const reapInterval = 30 * time.Second type logWriter struct{} diff --git a/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go b/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go index afe078b997..216027905f 100644 --- a/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go +++ b/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go @@ -130,6 +130,11 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) { return } + // Ignore messages that this node generated. + if tEvent.NodeName == nDB.config.NodeName { + return + } + // Do not rebroadcast a bulk sync if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync { var err error diff --git a/vendor/src/github.com/docker/libnetwork/portmapper/mapper.go b/vendor/src/github.com/docker/libnetwork/portmapper/mapper.go index d125fa8d4b..0bf7630557 100644 --- a/vendor/src/github.com/docker/libnetwork/portmapper/mapper.go +++ b/vendor/src/github.com/docker/libnetwork/portmapper/mapper.go @@ -90,7 +90,10 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } if useProxy { - m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) + m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) + if err != nil { + return nil, err + } } else { m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort) } @@ -107,7 +110,10 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } if useProxy { - m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port) + m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port) + if err != nil { + return nil, err + } } else { m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort) } diff --git a/vendor/src/github.com/docker/libnetwork/portmapper/mock_proxy.go b/vendor/src/github.com/docker/libnetwork/portmapper/mock_proxy.go index 29b1605889..587026f9ce 100644 --- a/vendor/src/github.com/docker/libnetwork/portmapper/mock_proxy.go +++ b/vendor/src/github.com/docker/libnetwork/portmapper/mock_proxy.go @@ -2,8 +2,8 @@ package portmapper import "net" -func newMockProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy { - return &mockProxyCommand{} +func newMockProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) { + return &mockProxyCommand{}, nil } type mockProxyCommand struct { diff --git a/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go b/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go index ddde2744c2..25a341a98f 100644 --- a/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go +++ b/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go @@ -1,29 +1,19 @@ package portmapper import ( - "flag" "fmt" "io" "io/ioutil" - "log" "net" "os" "os/exec" - "os/signal" "strconv" "syscall" "time" - - "github.com/docker/docker/pkg/proxy" - "github.com/docker/docker/pkg/reexec" ) const userlandProxyCommandName = "docker-proxy" -func init() { - reexec.Register(userlandProxyCommandName, execProxy) -} - type userlandProxy interface { Start() error Stop() error @@ -35,66 +25,15 @@ type proxyCommand struct { cmd *exec.Cmd } -// execProxy is the reexec function that is registered to start the userland proxies -func execProxy() { - f := os.NewFile(3, "signal-parent") - host, container := parseHostContainerAddrs() +func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) { + cmd, err := exec.LookPath(userlandProxyCommandName) - p, err := proxy.NewProxy(host, container) if err != nil { - fmt.Fprintf(f, "1\n%s", err) - f.Close() - os.Exit(1) - } - go handleStopSignals(p) - fmt.Fprint(f, "0\n") - f.Close() - - // Run will block until the proxy stops - p.Run() -} - -// parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP -// net.Addrs to map the host and container ports -func parseHostContainerAddrs() (host net.Addr, container net.Addr) { - var ( - proto = flag.String("proto", "tcp", "proxy protocol") - hostIP = flag.String("host-ip", "", "host ip") - hostPort = flag.Int("host-port", -1, "host port") - containerIP = flag.String("container-ip", "", "container ip") - containerPort = flag.Int("container-port", -1, "container port") - ) - - flag.Parse() - - switch *proto { - case "tcp": - host = &net.TCPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort} - container = &net.TCPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort} - case "udp": - host = &net.UDPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort} - container = &net.UDPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort} - default: - log.Fatalf("unsupported protocol %s", *proto) + return nil, err } - return host, container -} - -func handleStopSignals(p proxy.Proxy) { - s := make(chan os.Signal, 10) - signal.Notify(s, os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP) - - for range s { - p.Close() - - os.Exit(0) - } -} - -func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy { args := []string{ - userlandProxyCommandName, + cmd, "-proto", proto, "-host-ip", hostIP.String(), "-host-port", strconv.Itoa(hostPort), @@ -104,13 +43,13 @@ func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net. return &proxyCommand{ cmd: &exec.Cmd{ - Path: reexec.Self(), + Path: cmd, Args: args, SysProcAttr: &syscall.SysProcAttr{ Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies }, }, - } + }, nil } func (p *proxyCommand) Start() error { diff --git a/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go b/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go index 735708563f..6b70cb7a1b 100644 --- a/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go +++ b/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go @@ -275,7 +275,15 @@ func (sb *sandbox) updateDNS(ipv6Enabled bool) error { if err != nil { return err } - if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newRC.Hash), filePerm); err != nil { + if err = tmpHashFile.Chmod(filePerm); err != nil { + tmpHashFile.Close() + return err + } + _, err = tmpHashFile.Write([]byte(newRC.Hash)) + if err1 := tmpHashFile.Close(); err == nil { + err = err1 + } + if err != nil { return err } return os.Rename(tmpHashFile.Name(), hashFile) diff --git a/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go b/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go index 898cedd2b4..a57bb8e371 100644 --- a/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go +++ b/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go @@ -135,6 +135,8 @@ func (c *controller) acceptClientConnections(sock string, l net.Listener) { continue } go func() { + defer conn.Close() + err := c.processExternalKey(conn) ret := success if err != nil { From d428a7a425f071ee9e5707c7319d3197540adc19 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Thu, 7 Jul 2016 12:58:43 +0100 Subject: [PATCH 3/4] Update swarmkit vendoring Needed for libnetwork vendoring Update Secret API name change correspondingly Signed-off-by: Justin Cormack --- daemon/cluster/convert/swarm.go | 6 +- hack/vendor.sh | 2 +- .../github.com/docker/swarmkit/agent/agent.go | 2 + .../docker/swarmkit/api/types.pb.go | 428 +++++++++--------- .../docker/swarmkit/api/types.proto | 9 +- .../github.com/docker/swarmkit/ca/server.go | 9 +- .../docker/swarmkit/identity/randomid.go | 8 +- .../networkallocator/networkallocator.go | 11 + .../swarmkit/manager/controlapi/node.go | 29 +- .../swarmkit/manager/controlapi/service.go | 13 + .../swarmkit/manager/dispatcher/dispatcher.go | 12 +- .../swarmkit/manager/orchestrator/services.go | 17 + .../swarmkit/manager/scheduler/constraint.go | 19 +- .../manager/state/raft/membership/cluster.go | 12 +- .../swarmkit/manager/state/raft/raft.go | 39 +- .../swarmkit/manager/state/raft/storage.go | 1 - 16 files changed, 349 insertions(+), 268 deletions(-) diff --git a/daemon/cluster/convert/swarm.go b/daemon/cluster/convert/swarm.go index e9131a3f24..331c199c84 100644 --- a/daemon/cluster/convert/swarm.go +++ b/daemon/cluster/convert/swarm.go @@ -136,13 +136,13 @@ func SwarmSpecUpdateAcceptancePolicy(spec *swarmapi.ClusterSpec, acceptancePolic hashPwd, _ = bcrypt.GenerateFromPassword([]byte(*p.Secret), 0) hashs[*p.Secret] = hashPwd } - policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{ + policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret{ Data: hashPwd, Alg: "bcrypt", } } } else if oldSecret := getOldSecret(oldSpec, policy.Role); oldSecret != nil { // else use the old one. - policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{ + policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret{ Data: oldSecret.Data, Alg: oldSecret.Alg, } @@ -153,7 +153,7 @@ func SwarmSpecUpdateAcceptancePolicy(spec *swarmapi.ClusterSpec, acceptancePolic return nil } -func getOldSecret(oldSpec *swarmapi.ClusterSpec, role swarmapi.NodeRole) *swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret { +func getOldSecret(oldSpec *swarmapi.ClusterSpec, role swarmapi.NodeRole) *swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret { if oldSpec == nil { return nil } diff --git a/hack/vendor.sh b/hack/vendor.sh index 2bcc3a6664..c9cf119b1e 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -139,7 +139,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0 clone git github.com/docker/containerd 1b3a81545ca79456086dc2aa424357be98b962ee # cluster -clone git github.com/docker/swarmkit 24eaf0021a2eea7938fce7493ce4731f53c2b87c +clone git github.com/docker/swarmkit 16fa595d3b6fec012830179dc8e9b2d90335527d clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b diff --git a/vendor/src/github.com/docker/swarmkit/agent/agent.go b/vendor/src/github.com/docker/swarmkit/agent/agent.go index db349f1e36..c714520939 100644 --- a/vendor/src/github.com/docker/swarmkit/agent/agent.go +++ b/vendor/src/github.com/docker/swarmkit/agent/agent.go @@ -195,6 +195,8 @@ func (a *Agent) run(ctx context.Context) { log.G(ctx).WithError(err).Error("agent: closing session failed") } sessionq = nil + // if we're here before <-registered, do nothing for that event + registered = nil case <-session.closed: log.G(ctx).Debugf("agent: rebuild session") diff --git a/vendor/src/github.com/docker/swarmkit/api/types.pb.go b/vendor/src/github.com/docker/swarmkit/api/types.pb.go index 49754b8622..4c8fc5b998 100644 --- a/vendor/src/github.com/docker/swarmkit/api/types.pb.go +++ b/vendor/src/github.com/docker/swarmkit/api/types.pb.go @@ -954,7 +954,7 @@ type AcceptancePolicy_RoleAdmissionPolicy struct { Autoaccept bool `protobuf:"varint,2,opt,name=autoaccept,proto3" json:"autoaccept,omitempty"` // Secret represents a user-provided string that is necessary for new // nodes to join the cluster - Secret *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret `protobuf:"bytes,3,opt,name=secret" json:"secret,omitempty"` + Secret *AcceptancePolicy_RoleAdmissionPolicy_Secret `protobuf:"bytes,3,opt,name=secret" json:"secret,omitempty"` } func (m *AcceptancePolicy_RoleAdmissionPolicy) Reset() { *m = AcceptancePolicy_RoleAdmissionPolicy{} } @@ -963,18 +963,18 @@ func (*AcceptancePolicy_RoleAdmissionPolicy) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{23, 0} } -type AcceptancePolicy_RoleAdmissionPolicy_HashedSecret struct { - // The actual hashed content +type AcceptancePolicy_RoleAdmissionPolicy_Secret struct { + // The actual content (possibly hashed) Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` - // The type of hash we are using + // The type of hash we are using, or "plaintext" Alg string `protobuf:"bytes,2,opt,name=alg,proto3" json:"alg,omitempty"` } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Reset() { - *m = AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{} +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Reset() { + *m = AcceptancePolicy_RoleAdmissionPolicy_Secret{} } -func (*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) ProtoMessage() {} -func (*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Descriptor() ([]byte, []int) { +func (*AcceptancePolicy_RoleAdmissionPolicy_Secret) ProtoMessage() {} +func (*AcceptancePolicy_RoleAdmissionPolicy_Secret) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{23, 0, 0} } @@ -1145,7 +1145,7 @@ func init() { proto.RegisterType((*IssuanceStatus)(nil), "docker.swarmkit.v1.IssuanceStatus") proto.RegisterType((*AcceptancePolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy") proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy") - proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy.HashedSecret") + proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy_Secret)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy.Secret") proto.RegisterType((*ExternalCA)(nil), "docker.swarmkit.v1.ExternalCA") proto.RegisterType((*CAConfig)(nil), "docker.swarmkit.v1.CAConfig") proto.RegisterType((*OrchestrationConfig)(nil), "docker.swarmkit.v1.OrchestrationConfig") @@ -1593,12 +1593,12 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Copy() *AcceptancePolicy_RoleAdmi return o } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Copy() *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret { +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Copy() *AcceptancePolicy_RoleAdmissionPolicy_Secret { if m == nil { return nil } - o := &AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{ + o := &AcceptancePolicy_RoleAdmissionPolicy_Secret{ Data: m.Data, Alg: m.Alg, } @@ -2180,12 +2180,12 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) GoString() string { +func (this *AcceptancePolicy_RoleAdmissionPolicy_Secret) GoString() string { if this == nil { return "nil" } s := make([]string, 0, 6) - s = append(s, "&api.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{") + s = append(s, "&api.AcceptancePolicy_RoleAdmissionPolicy_Secret{") s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n") s = append(s, "Alg: "+fmt.Sprintf("%#v", this.Alg)+",\n") s = append(s, "}") @@ -3407,7 +3407,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) MarshalTo(data []byte) (int, erro return i, nil } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Marshal() (data []byte, err error) { +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) n, err := m.MarshalTo(data) @@ -3417,7 +3417,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Marshal() (data []by return data[:n], nil } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) MarshalTo(data []byte) (int, error) { +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) MarshalTo(data []byte) (int, error) { var i int _ = i var l int @@ -4301,7 +4301,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Size() (n int) { return n } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Size() (n int) { +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Size() (n int) { var l int _ = l l = len(m.Data) @@ -4866,16 +4866,16 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy) String() string { s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy{`, `Role:` + fmt.Sprintf("%v", this.Role) + `,`, `Autoaccept:` + fmt.Sprintf("%v", this.Autoaccept) + `,`, - `Secret:` + strings.Replace(fmt.Sprintf("%v", this.Secret), "AcceptancePolicy_RoleAdmissionPolicy_HashedSecret", "AcceptancePolicy_RoleAdmissionPolicy_HashedSecret", 1) + `,`, + `Secret:` + strings.Replace(fmt.Sprintf("%v", this.Secret), "AcceptancePolicy_RoleAdmissionPolicy_Secret", "AcceptancePolicy_RoleAdmissionPolicy_Secret", 1) + `,`, `}`, }, "") return s } -func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) String() string { +func (this *AcceptancePolicy_RoleAdmissionPolicy_Secret) String() string { if this == nil { return "nil" } - s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{`, + s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy_Secret{`, `Data:` + fmt.Sprintf("%v", this.Data) + `,`, `Alg:` + fmt.Sprintf("%v", this.Alg) + `,`, `}`, @@ -8641,7 +8641,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Unmarshal(data []byte) error { return io.ErrUnexpectedEOF } if m.Secret == nil { - m.Secret = &AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{} + m.Secret = &AcceptancePolicy_RoleAdmissionPolicy_Secret{} } if err := m.Secret.Unmarshal(data[iNdEx:postIndex]); err != nil { return err @@ -8668,7 +8668,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Unmarshal(data []byte) error { } return nil } -func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Unmarshal(data []byte) error { +func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 for iNdEx < l { @@ -8691,10 +8691,10 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Unmarshal(data []byt fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: HashedSecret: wiretype end group for non-group") + return fmt.Errorf("proto: Secret: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: HashedSecret: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Secret: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -10199,196 +10199,196 @@ var ( ) var fileDescriptorTypes = []byte{ - // 3043 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x4d, 0x6c, 0x1b, 0xc7, - 0x15, 0x16, 0x7f, 0x45, 0x0e, 0x29, 0x89, 0x5e, 0x3b, 0x8e, 0xcc, 0xa8, 0xb2, 0xbb, 0x89, 0x1b, - 0xe7, 0xa7, 0x4c, 0xac, 0xa4, 0x85, 0x9b, 0xa0, 0x4d, 0x96, 0x3f, 0xb2, 0x58, 0x4b, 0x24, 0x31, - 0x94, 0x64, 0x04, 0x45, 0x4b, 0xac, 0x96, 0x23, 0x69, 0xa3, 0xe5, 0x2e, 0xbb, 0xbb, 0x94, 0x4c, + // 3042 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0x17, 0x3f, 0x45, 0x3e, 0x52, 0x12, 0xbd, 0x76, 0x1c, 0x99, 0x51, 0x65, 0x77, 0x13, 0x37, + 0xce, 0x47, 0x99, 0x58, 0x49, 0x0b, 0x37, 0x41, 0xeb, 0x2c, 0x3f, 0x64, 0xb1, 0x96, 0x48, 0x62, + 0xa8, 0x0f, 0x04, 0x05, 0x4a, 0xac, 0x96, 0x23, 0x71, 0xa3, 0xe5, 0x2e, 0xbb, 0xbb, 0x94, 0x4c, 0x14, 0x05, 0x9c, 0x5e, 0x5a, 0x04, 0x3d, 0xf4, 0x5e, 0x04, 0x41, 0xd1, 0xa2, 0xb7, 0x9e, 0x0b, - 0xf4, 0xe4, 0xa3, 0x8f, 0x2d, 0x0a, 0x14, 0x39, 0x05, 0x4d, 0x7a, 0xe8, 0xb5, 0x40, 0x0f, 0x39, - 0xb4, 0x87, 0xbe, 0x37, 0x3f, 0xcb, 0x1f, 0xaf, 0x15, 0xa7, 0xc9, 0x81, 0xe0, 0xce, 0x7b, 0xdf, - 0x7b, 0x33, 0x6f, 0xe6, 0xcd, 0xfb, 0x19, 0x52, 0x08, 0xc7, 0x43, 0x16, 0x54, 0x86, 0xbe, 0x17, - 0x7a, 0x9a, 0xd6, 0xf7, 0xac, 0x13, 0xe6, 0x57, 0x82, 0x33, 0xd3, 0x1f, 0x9c, 0xd8, 0x61, 0xe5, - 0xf4, 0x66, 0xf9, 0x4a, 0x68, 0x0f, 0x58, 0x10, 0x9a, 0x83, 0xe1, 0x2b, 0xd1, 0x97, 0x80, 0x97, - 0x9f, 0xee, 0x8f, 0x7c, 0x33, 0xb4, 0x3d, 0xf7, 0x15, 0xf5, 0x21, 0x19, 0x97, 0x8e, 0xbc, 0x23, - 0x8f, 0x7f, 0xbe, 0x82, 0x5f, 0x82, 0xaa, 0x5f, 0x25, 0x8b, 0xfb, 0xcc, 0x0f, 0x00, 0xa6, 0x5d, - 0x22, 0x19, 0xdb, 0xed, 0xb3, 0x7b, 0xab, 0x89, 0x6b, 0x89, 0x1b, 0x69, 0x2a, 0x06, 0xfa, 0x6f, - 0x12, 0xa4, 0x60, 0xb8, 0xae, 0x17, 0x72, 0x5d, 0x81, 0xa6, 0x91, 0xb4, 0x6b, 0x0e, 0x18, 0x07, - 0xe5, 0x29, 0xff, 0xd6, 0x6a, 0x24, 0xeb, 0x98, 0x07, 0xcc, 0x09, 0x56, 0x93, 0xd7, 0x52, 0x37, - 0x0a, 0x1b, 0x2f, 0x55, 0x1e, 0x5d, 0x73, 0x65, 0x4a, 0x49, 0x65, 0x9b, 0xa3, 0x1b, 0x6e, 0xe8, - 0x8f, 0xa9, 0x14, 0x2d, 0x7f, 0x87, 0x14, 0xa6, 0xc8, 0x5a, 0x89, 0xa4, 0x4e, 0xd8, 0x58, 0x4e, - 0x83, 0x9f, 0xb8, 0xbe, 0x53, 0xd3, 0x19, 0x31, 0x98, 0x04, 0x69, 0x62, 0xf0, 0x46, 0xf2, 0x56, - 0x42, 0x7f, 0x87, 0xe4, 0x29, 0x0b, 0xbc, 0x91, 0x6f, 0xb1, 0x40, 0x7b, 0x81, 0xe4, 0x5d, 0xd3, - 0xf5, 0x7a, 0xd6, 0x70, 0x14, 0x70, 0xf1, 0x54, 0xb5, 0xf8, 0xe9, 0xc7, 0x57, 0x73, 0x2d, 0x20, - 0xd6, 0x3a, 0x7b, 0x01, 0xcd, 0x21, 0xbb, 0x06, 0x5c, 0xed, 0xeb, 0xa4, 0x38, 0x60, 0x03, 0xcf, - 0x1f, 0xf7, 0x0e, 0xc6, 0x21, 0x0b, 0xb8, 0xe2, 0x14, 0x2d, 0x08, 0x5a, 0x15, 0x49, 0xfa, 0xaf, - 0x12, 0xe4, 0x92, 0xd2, 0x4d, 0xd9, 0x8f, 0x47, 0xb6, 0xcf, 0x06, 0xcc, 0x0d, 0x03, 0xed, 0x5b, - 0x60, 0xb3, 0x3d, 0xb0, 0x43, 0x31, 0x47, 0x61, 0xe3, 0x6b, 0x71, 0x36, 0x47, 0xab, 0xa2, 0x12, - 0xac, 0x19, 0xa4, 0xe8, 0xb3, 0x80, 0xf9, 0xa7, 0x62, 0x27, 0xf8, 0x94, 0x9f, 0x2b, 0x3c, 0x23, - 0xa2, 0x6f, 0x92, 0x5c, 0xc7, 0x31, 0xc3, 0x43, 0xcf, 0x1f, 0x68, 0x3a, 0x29, 0x9a, 0xbe, 0x75, - 0x6c, 0x87, 0xcc, 0x0a, 0x47, 0xbe, 0x3a, 0x95, 0x19, 0x9a, 0x76, 0x99, 0x24, 0x3d, 0x31, 0x51, - 0xbe, 0x9a, 0x85, 0x9d, 0x48, 0xb6, 0xbb, 0x14, 0x28, 0xfa, 0x9b, 0xe4, 0x42, 0xc7, 0x19, 0x1d, - 0xd9, 0x6e, 0x9d, 0x05, 0x96, 0x6f, 0x0f, 0x51, 0x3b, 0x1e, 0x2f, 0x3a, 0x9f, 0x3a, 0x5e, 0xfc, - 0x8e, 0x8e, 0x3c, 0x39, 0x39, 0x72, 0xfd, 0xe7, 0x49, 0x72, 0xa1, 0xe1, 0x82, 0x30, 0x9b, 0x96, - 0xbe, 0x4e, 0x96, 0x19, 0x27, 0xf6, 0x4e, 0x85, 0x53, 0x49, 0x3d, 0x4b, 0x82, 0xaa, 0x3c, 0xad, - 0x39, 0xe7, 0x2f, 0x37, 0xe3, 0xcc, 0x7f, 0x44, 0x7b, 0x9c, 0xd7, 0x68, 0x0d, 0xb2, 0x38, 0xe4, - 0x46, 0x04, 0xab, 0x29, 0xae, 0xeb, 0x7a, 0x9c, 0xae, 0x47, 0xec, 0xac, 0xa6, 0x1f, 0x7e, 0x7c, - 0x75, 0x81, 0x2a, 0xd9, 0x2f, 0xe3, 0x7c, 0xff, 0x48, 0x90, 0x95, 0x96, 0xd7, 0x9f, 0xd9, 0x87, - 0x32, 0xc9, 0x1d, 0x7b, 0x41, 0x38, 0x75, 0x51, 0xa2, 0xb1, 0x76, 0x8b, 0xe4, 0x86, 0xf2, 0xf8, - 0xe4, 0xe9, 0xaf, 0xc5, 0x2f, 0x59, 0x60, 0x68, 0x84, 0xd6, 0xde, 0x24, 0x79, 0x5f, 0xf9, 0x04, - 0x58, 0xfb, 0x04, 0x8e, 0x33, 0xc1, 0x6b, 0xdf, 0x25, 0x59, 0x71, 0x08, 0xab, 0x69, 0x2e, 0x79, - 0xfd, 0x89, 0xf6, 0x9c, 0x4a, 0x21, 0xfd, 0xa3, 0x04, 0x29, 0x51, 0xf3, 0x30, 0xdc, 0x61, 0x83, - 0x03, 0xe6, 0x77, 0xe1, 0x22, 0xc3, 0xfd, 0xb9, 0x0c, 0xe7, 0xc8, 0xcc, 0x3e, 0xf3, 0xb9, 0x91, - 0x39, 0x2a, 0x47, 0xda, 0x1e, 0x3a, 0xb9, 0x69, 0x1d, 0x9b, 0x07, 0xb6, 0x63, 0x87, 0x63, 0x6e, - 0xe6, 0x72, 0xfc, 0x29, 0xcf, 0xeb, 0x84, 0xc5, 0x4f, 0x04, 0xe9, 0x8c, 0x1a, 0x6d, 0x95, 0x2c, - 0x42, 0xac, 0x0b, 0xcc, 0x23, 0xc6, 0xad, 0xcf, 0x53, 0x35, 0x04, 0x57, 0x2e, 0x4e, 0xcb, 0x69, - 0x05, 0xb2, 0xb8, 0xd7, 0xba, 0xd3, 0x6a, 0xdf, 0x6d, 0x95, 0x16, 0xb4, 0x15, 0x52, 0xd8, 0x6b, - 0xd1, 0x86, 0x51, 0xdb, 0x32, 0xaa, 0xdb, 0x8d, 0x52, 0x42, 0x5b, 0x82, 0x70, 0x11, 0x0d, 0x93, - 0xfa, 0x87, 0x09, 0x42, 0xf0, 0x00, 0xa5, 0x51, 0x6f, 0x90, 0x0c, 0xc4, 0xd3, 0x50, 0x1c, 0xdc, - 0xf2, 0xc6, 0x73, 0x71, 0xab, 0x9e, 0xc0, 0x2b, 0xf8, 0xc7, 0xa8, 0x10, 0x99, 0x5e, 0x61, 0x72, - 0x7e, 0x85, 0x19, 0x8e, 0x9c, 0x5d, 0x5a, 0x8e, 0xa4, 0xeb, 0xf8, 0x95, 0xd0, 0xf2, 0x24, 0x03, - 0x6b, 0xaa, 0xbf, 0x53, 0x4a, 0x82, 0xf3, 0x15, 0xeb, 0xcd, 0x6e, 0xad, 0xdd, 0x6a, 0x35, 0x6a, - 0xbb, 0x8d, 0x7a, 0x29, 0xa5, 0x5f, 0x27, 0x99, 0xe6, 0x00, 0xb4, 0x68, 0x6b, 0xe8, 0x01, 0x87, - 0xcc, 0x67, 0xae, 0xa5, 0x1c, 0x6b, 0x42, 0xd0, 0x7f, 0x99, 0x23, 0x99, 0x1d, 0x6f, 0xe4, 0x86, - 0xda, 0xc6, 0xd4, 0x2d, 0x5e, 0xde, 0x58, 0x8f, 0x33, 0x81, 0x03, 0x2b, 0xbb, 0x80, 0x92, 0xb7, - 0x1c, 0x0e, 0x53, 0xf8, 0x8a, 0x5c, 0xba, 0x1c, 0x21, 0x3d, 0x34, 0xfd, 0x23, 0x16, 0xca, 0x4d, - 0x97, 0x23, 0xed, 0x06, 0xc9, 0xc1, 0xe9, 0xf4, 0x3d, 0xd7, 0x19, 0x73, 0x97, 0xca, 0x89, 0x30, - 0x0b, 0xe7, 0xd0, 0x6f, 0x03, 0x8d, 0x46, 0x5c, 0x6d, 0x8b, 0x14, 0x0f, 0x20, 0x99, 0xf4, 0xbc, - 0xa1, 0x88, 0x79, 0x99, 0xc7, 0x3b, 0xa0, 0x58, 0x55, 0x15, 0xd0, 0x6d, 0x01, 0xa6, 0x85, 0x83, - 0xc9, 0x40, 0x6b, 0x91, 0xe5, 0x53, 0xcf, 0x19, 0x0d, 0x58, 0xa4, 0x2b, 0xcb, 0x75, 0x3d, 0xff, - 0x78, 0x5d, 0xfb, 0x1c, 0xaf, 0xb4, 0x2d, 0x9d, 0x4e, 0x0f, 0xcb, 0x3f, 0x4b, 0x91, 0xc2, 0xd4, - 0x64, 0x5a, 0x97, 0x14, 0x20, 0x2d, 0x0e, 0xcd, 0x23, 0x1e, 0x6a, 0xe5, 0xf6, 0xdd, 0x7c, 0xa2, - 0x85, 0x56, 0x3a, 0x13, 0x41, 0x3a, 0xad, 0x45, 0xff, 0x20, 0x49, 0x0a, 0x53, 0x4c, 0xed, 0x45, - 0x92, 0xa3, 0x1d, 0xda, 0xdc, 0x37, 0x76, 0x1b, 0xa5, 0x85, 0xf2, 0xda, 0xfb, 0x1f, 0x5c, 0x5b, - 0xe5, 0xda, 0xa6, 0x15, 0x74, 0x7c, 0xfb, 0x14, 0xbd, 0xe5, 0x06, 0x59, 0x54, 0xd0, 0x44, 0xf9, - 0x19, 0x80, 0x3e, 0x3d, 0x0f, 0x9d, 0x42, 0xd2, 0xee, 0x96, 0x41, 0xc1, 0x61, 0x92, 0xf1, 0x48, - 0xda, 0x3d, 0x36, 0x7d, 0xd6, 0xd7, 0xbe, 0x41, 0xb2, 0x12, 0x98, 0x2a, 0x97, 0x01, 0x78, 0x79, - 0x1e, 0x38, 0xc1, 0xd1, 0xee, 0xb6, 0xb1, 0xdf, 0x28, 0xa5, 0xe3, 0x71, 0xb4, 0xeb, 0x98, 0xa7, - 0x4c, 0x7b, 0x0e, 0x5c, 0x9b, 0xc3, 0x32, 0xe5, 0x2b, 0x00, 0x7b, 0xea, 0x11, 0x75, 0x88, 0x2a, - 0xaf, 0xfe, 0xe2, 0xb7, 0xeb, 0x0b, 0x7f, 0xfa, 0xdd, 0x7a, 0x69, 0x9e, 0x5d, 0xfe, 0x6f, 0x82, - 0x2c, 0xcd, 0x9c, 0x12, 0x64, 0xb5, 0xac, 0xeb, 0x59, 0xde, 0x50, 0x44, 0xe0, 0x5c, 0x95, 0x80, - 0x63, 0x65, 0x5b, 0x5e, 0x0d, 0x28, 0x54, 0x72, 0xb4, 0x3b, 0x73, 0x39, 0xe4, 0xb5, 0x27, 0x74, - 0x81, 0xd8, 0x2c, 0xf2, 0x16, 0x59, 0xea, 0xc3, 0x3e, 0x32, 0xbf, 0x67, 0x79, 0xee, 0xa1, 0x7d, - 0x24, 0xa3, 0x6b, 0x39, 0x4e, 0x67, 0x9d, 0x03, 0x69, 0x51, 0x08, 0xd4, 0x38, 0xfe, 0xcb, 0xe4, - 0x8f, 0xbb, 0x24, 0x8d, 0xb7, 0x50, 0x7b, 0x86, 0xa4, 0xab, 0xcd, 0x56, 0x1d, 0x5c, 0xe2, 0x02, - 0xec, 0xe2, 0x12, 0x5f, 0x3a, 0x32, 0xd0, 0xc7, 0xb4, 0xab, 0x24, 0xbb, 0xdf, 0xde, 0xde, 0xdb, - 0x41, 0x37, 0xb8, 0x08, 0xec, 0x95, 0x88, 0x2d, 0x8c, 0x2b, 0x5f, 0x90, 0xdb, 0x9b, 0x8f, 0x18, - 0xfa, 0x7f, 0x92, 0x64, 0x89, 0x62, 0x6d, 0xe8, 0x87, 0x1d, 0xcf, 0xb1, 0xad, 0xb1, 0xd6, 0x21, - 0x79, 0xb0, 0xaf, 0x6f, 0x4f, 0x39, 0xf7, 0xc6, 0x63, 0x12, 0xc8, 0x44, 0x4a, 0x8d, 0x6a, 0x4a, - 0x92, 0x4e, 0x94, 0x40, 0xa0, 0xc9, 0xf4, 0x99, 0x63, 0x8e, 0xcf, 0xcb, 0x64, 0x75, 0x59, 0x87, - 0x52, 0x01, 0xe5, 0x55, 0x97, 0x79, 0xaf, 0x67, 0x86, 0x21, 0x1b, 0x0c, 0x43, 0x91, 0xc9, 0xd2, - 0x50, 0x75, 0x99, 0xf7, 0x0c, 0x49, 0xd2, 0x5e, 0x27, 0xd9, 0x33, 0x30, 0xdb, 0x3b, 0x93, 0xc9, - 0xea, 0x7c, 0xbd, 0x12, 0xab, 0xbf, 0x8f, 0x39, 0x6a, 0x6e, 0xb1, 0xb8, 0xad, 0xad, 0x76, 0xab, - 0xa1, 0xb6, 0x55, 0xf2, 0xdb, 0x6e, 0xcb, 0x73, 0xd1, 0x75, 0x49, 0xbb, 0xd5, 0xdb, 0x34, 0x9a, - 0xdb, 0x7b, 0x14, 0xb7, 0xf6, 0x12, 0x40, 0x4a, 0x11, 0x64, 0xd3, 0xb4, 0x1d, 0x2c, 0xa0, 0xae, - 0x90, 0x94, 0xd1, 0x82, 0xc8, 0x5c, 0x2e, 0x01, 0xbb, 0x18, 0xb1, 0x0d, 0x77, 0x3c, 0xf1, 0xea, - 0xf9, 0x79, 0xf5, 0x77, 0x49, 0x71, 0x6f, 0xd8, 0x87, 0x9b, 0x29, 0x3c, 0x44, 0xbb, 0x06, 0xa1, - 0xc5, 0xf4, 0x4d, 0xc7, 0x61, 0x8e, 0x1d, 0x0c, 0x64, 0x8d, 0x3d, 0x4d, 0x82, 0xc2, 0xe0, 0xc9, - 0xf7, 0x52, 0xd6, 0x2f, 0x42, 0x40, 0xff, 0x29, 0x59, 0x81, 0x59, 0x42, 0x13, 0x12, 0xb5, 0x4a, - 0xcd, 0x1b, 0xa4, 0x68, 0x29, 0x52, 0xcf, 0xee, 0x0b, 0x57, 0xac, 0xae, 0xc0, 0x45, 0x2a, 0x44, - 0xd0, 0x66, 0x9d, 0x16, 0x22, 0x50, 0xb3, 0x8f, 0x76, 0x0e, 0x01, 0x8a, 0xd3, 0x67, 0xaa, 0x8b, - 0x00, 0x4d, 0x75, 0x00, 0x82, 0x34, 0xd8, 0xc5, 0x3c, 0xbb, 0x67, 0x87, 0x70, 0x3d, 0xfa, 0x22, - 0xf9, 0x66, 0x68, 0x0e, 0x09, 0x35, 0x18, 0xeb, 0xef, 0x25, 0x09, 0xd9, 0x35, 0x83, 0x13, 0x39, - 0x35, 0x94, 0x29, 0x51, 0x53, 0x72, 0x5e, 0x71, 0xbc, 0xab, 0x40, 0x74, 0x82, 0xd7, 0x5e, 0x53, - 0xd9, 0x57, 0xd4, 0x0c, 0xf1, 0x82, 0x72, 0xae, 0xb8, 0xb4, 0x3b, 0x5b, 0x18, 0xe0, 0x45, 0x64, - 0xbe, 0xcf, 0xbd, 0x08, 0x2e, 0x22, 0x7c, 0x42, 0xaf, 0x92, 0x8f, 0x6c, 0x96, 0x99, 0xe8, 0xd9, - 0xb8, 0x49, 0xe6, 0x36, 0x74, 0x6b, 0x81, 0x4e, 0xe4, 0xaa, 0x25, 0xb2, 0xec, 0xc3, 0x35, 0x83, - 0x55, 0xf7, 0x02, 0xce, 0xd6, 0xff, 0x0a, 0x7b, 0xd0, 0xec, 0x18, 0x3b, 0xf2, 0xb4, 0xeb, 0x24, - 0x7b, 0x68, 0x0e, 0x6c, 0x67, 0x2c, 0xaf, 0xd9, 0xcb, 0x71, 0x53, 0x4c, 0xf0, 0x15, 0xa3, 0xdf, - 0x87, 0x52, 0x2d, 0xd8, 0xe4, 0x32, 0x54, 0xca, 0xf2, 0x94, 0x3c, 0x3a, 0x70, 0x21, 0xf5, 0xaa, - 0x94, 0xcc, 0x47, 0x18, 0x4c, 0x7c, 0xd3, 0x8d, 0xac, 0x15, 0x03, 0xdc, 0x05, 0x88, 0xa8, 0xec, - 0xcc, 0x1c, 0x4b, 0x7b, 0xd5, 0x10, 0x12, 0x70, 0x4e, 0x74, 0x10, 0xac, 0x0f, 0x26, 0x63, 0xb4, - 0xfc, 0xbc, 0xf5, 0x50, 0x09, 0x17, 0x61, 0x32, 0x92, 0x2e, 0xbf, 0xc9, 0x43, 0xca, 0x84, 0xf5, - 0x85, 0x22, 0xdd, 0xab, 0x64, 0x69, 0xc6, 0xce, 0x47, 0x6a, 0xa1, 0x66, 0x67, 0xff, 0xf5, 0x52, - 0x5a, 0x7e, 0x7d, 0xbb, 0x94, 0xd5, 0xff, 0x0d, 0xa5, 0x59, 0xc7, 0xe3, 0xd7, 0x0a, 0x77, 0x35, - 0xbe, 0xf7, 0xcc, 0xf1, 0x4e, 0xd6, 0xf2, 0x1c, 0xe9, 0x33, 0xb1, 0xc5, 0xc0, 0x44, 0x0b, 0x26, - 0x6a, 0x0e, 0xa7, 0x91, 0x20, 0x84, 0xd7, 0x82, 0xa8, 0x6a, 0x7a, 0x43, 0xc0, 0xf1, 0x6d, 0x5d, - 0xa2, 0x44, 0x90, 0x50, 0x12, 0x1b, 0x9b, 0xe1, 0xe8, 0x00, 0xae, 0xe9, 0x31, 0xeb, 0x0b, 0x4c, - 0x9a, 0x63, 0x96, 0x22, 0x2a, 0xc2, 0xf4, 0x3a, 0xb4, 0x66, 0x4a, 0xe7, 0x2a, 0x49, 0xed, 0xd6, - 0x3a, 0x10, 0x77, 0x56, 0x20, 0x6a, 0x14, 0x14, 0x19, 0x48, 0xc8, 0xd9, 0xab, 0x77, 0x20, 0xdc, - 0xcc, 0x70, 0x80, 0x54, 0x4e, 0x63, 0x38, 0xd1, 0x7f, 0x9d, 0x20, 0x59, 0x91, 0x65, 0x62, 0x2d, - 0x36, 0xc8, 0xa2, 0xaa, 0x7e, 0x44, 0xea, 0x7b, 0xfe, 0xf1, 0x69, 0xaa, 0x22, 0xb3, 0x9e, 0x38, - 0x47, 0x25, 0x57, 0x7e, 0x83, 0x14, 0xa7, 0x19, 0x5f, 0xe8, 0x14, 0x7f, 0x42, 0x0a, 0xe8, 0x28, - 0x2a, 0x57, 0x6f, 0x90, 0xac, 0xc8, 0x84, 0xf2, 0xaa, 0x9f, 0x97, 0x33, 0x25, 0x12, 0x22, 0xdd, - 0xa2, 0xc8, 0xb3, 0xaa, 0x69, 0x5b, 0x3f, 0xdf, 0x1d, 0xa9, 0x82, 0xeb, 0x6f, 0x91, 0x74, 0x87, - 0x81, 0x86, 0x67, 0xc9, 0xa2, 0x0b, 0xa1, 0x67, 0x12, 0xd9, 0x64, 0x89, 0xd0, 0x67, 0x10, 0xb1, - 0xb2, 0xc8, 0x82, 0x78, 0x06, 0x9b, 0x67, 0x82, 0xbf, 0xa9, 0xbe, 0x15, 0xbf, 0xf5, 0x5d, 0x52, - 0xbc, 0xcb, 0xec, 0xa3, 0xe3, 0x10, 0x4e, 0x0c, 0x15, 0xbd, 0x4c, 0xd2, 0x43, 0x16, 0x2d, 0x7e, - 0x35, 0xd6, 0x75, 0x80, 0x4f, 0x39, 0x0a, 0x2f, 0xe4, 0x19, 0x97, 0x96, 0x4f, 0x05, 0x72, 0xa4, - 0xff, 0x21, 0x49, 0x96, 0x9b, 0x41, 0x30, 0x32, 0xa1, 0x0c, 0x97, 0x51, 0xf0, 0x7b, 0xb3, 0x6d, - 0xc4, 0x8d, 0x58, 0x0b, 0x67, 0x44, 0x66, 0x5b, 0x09, 0x19, 0xb9, 0x92, 0x51, 0xe4, 0xd2, 0x1f, - 0x26, 0x54, 0x0f, 0x71, 0x7d, 0xea, 0xde, 0x94, 0x57, 0xc1, 0x89, 0x2e, 0x4d, 0x6b, 0x62, 0x7b, - 0xee, 0x89, 0xeb, 0x9d, 0xb9, 0x90, 0x68, 0xa1, 0xa7, 0x68, 0x35, 0xee, 0x82, 0xa7, 0x5d, 0x06, - 0x90, 0x36, 0x03, 0xa2, 0xcc, 0x65, 0x67, 0xa8, 0xa9, 0xd3, 0x68, 0xd5, 0x9b, 0xad, 0xdb, 0x90, - 0xde, 0x1e, 0xd5, 0xd4, 0x61, 0x90, 0xce, 0xdc, 0x23, 0xd8, 0xee, 0x6c, 0xb3, 0xdb, 0xdd, 0xe3, - 0x25, 0xe3, 0xd3, 0x80, 0xba, 0x38, 0x83, 0xc2, 0x01, 0xd4, 0x8b, 0x00, 0xc2, 0x4c, 0x0a, 0xa0, - 0x74, 0x0c, 0x08, 0x93, 0x29, 0x04, 0x10, 0xe1, 0xe1, 0xff, 0x4c, 0x92, 0x92, 0x61, 0x59, 0x6c, - 0x18, 0x22, 0x5f, 0x56, 0x27, 0xbb, 0x70, 0x93, 0xf1, 0xcb, 0x66, 0xf8, 0xa6, 0x82, 0x6e, 0x71, - 0x2b, 0xf6, 0x1d, 0x69, 0x4e, 0xae, 0x42, 0x3d, 0x87, 0x19, 0xfd, 0x81, 0x1d, 0xe0, 0xdb, 0x82, - 0xa0, 0xd1, 0x48, 0x13, 0x56, 0x97, 0x17, 0x63, 0x10, 0xda, 0xab, 0x24, 0xed, 0x03, 0x59, 0x1e, - 0xcf, 0xda, 0xe3, 0xba, 0x3c, 0x14, 0xa5, 0x1c, 0xa9, 0xad, 0x13, 0x62, 0x8e, 0x42, 0xcf, 0xe4, - 0xf3, 0xf3, 0x83, 0xc9, 0xd1, 0x29, 0x8a, 0xf6, 0x43, 0x88, 0xd6, 0xcc, 0xf2, 0x65, 0xa3, 0x54, - 0xd8, 0x68, 0xfc, 0xbf, 0xab, 0xaf, 0x6c, 0x99, 0x18, 0x51, 0xba, 0x5c, 0x19, 0x95, 0x4a, 0xcb, - 0xaf, 0x93, 0xe2, 0x34, 0x1d, 0xbd, 0x1b, 0xca, 0x0b, 0x93, 0x1b, 0x50, 0xa4, 0xfc, 0x1b, 0x9d, - 0xc6, 0x74, 0x8e, 0x94, 0xd3, 0xc0, 0xa7, 0xfe, 0x21, 0xe4, 0xa5, 0xc6, 0xbd, 0x90, 0xf9, 0xae, - 0xe9, 0xd4, 0x0c, 0xad, 0x31, 0x15, 0x2d, 0x85, 0xe5, 0x2f, 0xc4, 0xbe, 0x03, 0x44, 0x12, 0x95, - 0x9a, 0x11, 0x13, 0x2f, 0xa1, 0x52, 0x18, 0xf9, 0x8e, 0x7c, 0x53, 0xe2, 0x95, 0xc2, 0x1e, 0xdd, - 0xa6, 0x48, 0xc3, 0x07, 0x19, 0x15, 0x9d, 0x52, 0x8f, 0x7f, 0x0c, 0x9c, 0x9a, 0xe0, 0xab, 0x8f, - 0x50, 0x2f, 0x13, 0x32, 0x59, 0x35, 0x1c, 0x5b, 0xa6, 0xb6, 0xd9, 0xed, 0x6e, 0xc3, 0x55, 0xe1, - 0x95, 0xf3, 0x84, 0xc5, 0xc9, 0xfa, 0xef, 0x13, 0x24, 0x57, 0x33, 0x64, 0x86, 0xd9, 0x24, 0x25, - 0x1e, 0x57, 0x2c, 0xe6, 0x87, 0x3d, 0x76, 0x6f, 0x68, 0xfb, 0x63, 0x19, 0x1a, 0xce, 0x2f, 0x41, - 0x97, 0x51, 0xaa, 0x06, 0x42, 0x0d, 0x2e, 0xa3, 0x51, 0x52, 0x64, 0xd2, 0xc4, 0x9e, 0x65, 0xaa, - 0x40, 0xbd, 0x7e, 0xfe, 0x56, 0x88, 0xf2, 0x6c, 0x32, 0x86, 0xe6, 0x57, 0x29, 0xa9, 0x99, 0x81, - 0xbe, 0x4f, 0x2e, 0xb6, 0x7d, 0xeb, 0x18, 0x0a, 0x25, 0x31, 0xa9, 0x5c, 0xf2, 0x5b, 0x64, 0x2d, - 0x84, 0x82, 0xa8, 0x77, 0x6c, 0x07, 0x21, 0x3e, 0x65, 0x82, 0x6f, 0x30, 0x17, 0xf9, 0x3d, 0xfe, - 0xe4, 0x28, 0x9e, 0x40, 0xe9, 0x15, 0xc4, 0x6c, 0x09, 0x08, 0x55, 0x88, 0x6d, 0x04, 0xe8, 0x3f, - 0x20, 0xa5, 0xba, 0x1d, 0x0c, 0xcd, 0x10, 0x74, 0xcb, 0x7e, 0x46, 0xbb, 0x4d, 0x4a, 0xc7, 0x0c, - 0xea, 0xd9, 0x03, 0x66, 0x42, 0x4e, 0x64, 0xbe, 0xed, 0xf5, 0x9f, 0x68, 0x1f, 0x56, 0x22, 0xa9, - 0x0e, 0x17, 0xd2, 0x3f, 0x83, 0x0c, 0x8e, 0x6f, 0x3c, 0x52, 0xef, 0x4b, 0xe4, 0x42, 0xe0, 0x9a, - 0xc3, 0xe0, 0xd8, 0x0b, 0x7b, 0xb6, 0x1b, 0xe2, 0xa3, 0xa6, 0x23, 0x6b, 0xe1, 0x92, 0x62, 0x34, - 0x25, 0x1d, 0x62, 0xb3, 0x76, 0xc2, 0xd8, 0xb0, 0xe7, 0x39, 0xfd, 0x9e, 0x62, 0x8a, 0x87, 0x4c, - 0x40, 0x23, 0xa7, 0xed, 0xf4, 0xbb, 0x8a, 0xae, 0x55, 0xc9, 0xba, 0xe3, 0x1d, 0xf5, 0xc0, 0x32, - 0x1f, 0xee, 0x7d, 0xef, 0xd0, 0xf3, 0x7b, 0x81, 0xe3, 0x9d, 0xc1, 0x87, 0x03, 0x7f, 0xcc, 0x57, - 0x8d, 0x46, 0x19, 0x50, 0x0d, 0x01, 0xda, 0xf4, 0xfc, 0x2e, 0xf0, 0x36, 0x15, 0x02, 0xd3, 0xfc, - 0xc4, 0xec, 0xd0, 0xb6, 0x4e, 0x54, 0x9a, 0x8f, 0xa8, 0xbb, 0x40, 0x84, 0x48, 0xb7, 0xc4, 0x1c, - 0x66, 0xf1, 0x4d, 0xe6, 0xa8, 0x0c, 0x47, 0x15, 0x15, 0x11, 0x41, 0xfa, 0x37, 0x49, 0xbe, 0xe3, - 0x98, 0x16, 0x7f, 0x2e, 0xc6, 0xea, 0x1f, 0x52, 0x18, 0x9e, 0x1c, 0x58, 0x2d, 0xc2, 0x5b, 0x9e, - 0x4e, 0x93, 0xf4, 0xf7, 0x20, 0xe9, 0x53, 0xcf, 0x0b, 0xe1, 0x92, 0x5e, 0x23, 0x59, 0xcb, 0xec, - 0x29, 0x77, 0x2f, 0x56, 0xf3, 0xe0, 0x16, 0x99, 0x9a, 0x71, 0x87, 0x8d, 0x69, 0xc6, 0x32, 0xe1, - 0x0f, 0xd3, 0x1f, 0x20, 0xd0, 0x49, 0xf9, 0x76, 0x14, 0x45, 0xfa, 0x03, 0x2f, 0x06, 0x0a, 0x05, - 0x61, 0xfc, 0x87, 0x08, 0x57, 0x94, 0xa0, 0xde, 0x31, 0x04, 0x0e, 0x51, 0x2c, 0x56, 0x97, 0x01, - 0x49, 0x04, 0x12, 0xc3, 0x09, 0x25, 0x02, 0x8d, 0xdf, 0xfa, 0xdf, 0x12, 0xa4, 0x80, 0x03, 0xfb, - 0xd0, 0xb6, 0x30, 0xcf, 0x7c, 0xf1, 0x18, 0x09, 0x81, 0xc1, 0x0a, 0x7c, 0xb9, 0x28, 0x1e, 0x18, - 0x6a, 0x5d, 0x4a, 0x91, 0xa6, 0xbd, 0x0d, 0xe1, 0x91, 0xe7, 0x39, 0x19, 0x1e, 0xf5, 0xcf, 0xcf, - 0x88, 0xb2, 0xcb, 0x91, 0x72, 0x7c, 0x13, 0x27, 0xab, 0xe3, 0x47, 0x53, 0xa4, 0xd3, 0x24, 0x7c, - 0xea, 0xb6, 0x5c, 0x7e, 0x1a, 0xf2, 0xa9, 0xbb, 0xd6, 0xa2, 0x40, 0xd1, 0xff, 0x92, 0x20, 0x4b, - 0x0d, 0xd7, 0xf2, 0xc7, 0x3c, 0xa4, 0xe0, 0x0e, 0xae, 0x91, 0x3c, 0x14, 0xd3, 0xc1, 0x38, 0x80, - 0x8e, 0x53, 0xbd, 0xa4, 0x45, 0x04, 0xad, 0x49, 0xf2, 0x10, 0x3c, 0x3d, 0xdf, 0x0e, 0x8f, 0x07, - 0xb2, 0xaa, 0x8c, 0x0f, 0x63, 0xd3, 0x3a, 0x2b, 0x86, 0x12, 0xa1, 0x13, 0x69, 0x15, 0xb8, 0x52, - 0x7c, 0xb1, 0x3c, 0x70, 0x41, 0xff, 0xeb, 0x40, 0xab, 0x03, 0xf5, 0x62, 0x0f, 0x3b, 0x08, 0x6e, - 0x07, 0xb4, 0x82, 0x92, 0x86, 0x5d, 0x91, 0xae, 0x93, 0x7c, 0xa4, 0x0c, 0xdf, 0x2f, 0x8d, 0x46, - 0xb7, 0x77, 0x73, 0xe3, 0x56, 0xef, 0x76, 0x6d, 0x07, 0xc2, 0x98, 0xc8, 0xa1, 0x7f, 0x04, 0x9b, - 0x76, 0x4c, 0x17, 0xba, 0x1c, 0xd5, 0xf3, 0x81, 0x57, 0xf8, 0x70, 0xd5, 0x54, 0x51, 0x94, 0x16, - 0x5e, 0x81, 0xb7, 0x0f, 0x8b, 0x22, 0x64, 0xc5, 0x17, 0x45, 0x53, 0xef, 0xb8, 0xa9, 0x73, 0xdf, - 0x71, 0xd3, 0x5f, 0xc9, 0x3b, 0xee, 0x8b, 0x9f, 0xa5, 0x48, 0x3e, 0xea, 0xe1, 0xd0, 0x65, 0xb0, - 0x46, 0x59, 0x10, 0xdd, 0x75, 0x44, 0x6f, 0xf1, 0xea, 0x24, 0x6f, 0x6c, 0x6f, 0xb7, 0x6b, 0x06, - 0x3e, 0x83, 0xbe, 0x2d, 0x8a, 0x98, 0x08, 0x60, 0xc0, 0xa5, 0xc5, 0x43, 0xef, 0x6b, 0xfa, 0xa4, - 0x88, 0xb9, 0x2f, 0x7b, 0xf8, 0x08, 0xa5, 0x2a, 0x98, 0xe7, 0x48, 0xce, 0xe8, 0x76, 0x9b, 0xb7, - 0x5b, 0xa0, 0xe9, 0x41, 0xa2, 0xfc, 0x14, 0x80, 0x2e, 0x4c, 0x54, 0x41, 0xf2, 0x3d, 0x72, 0x41, - 0x13, 0xa2, 0x6a, 0xb5, 0x46, 0x07, 0xe7, 0xbb, 0x9f, 0x9c, 0x47, 0xf1, 0xd4, 0xcd, 0x1f, 0xc6, - 0xf2, 0x1d, 0xda, 0xe8, 0x18, 0x14, 0x67, 0x7c, 0x90, 0x9c, 0x5b, 0x57, 0xc7, 0x67, 0xd0, 0xd5, - 0xe3, 0x9c, 0xeb, 0xea, 0x4d, 0xf7, 0x7e, 0xaa, 0xac, 0x01, 0x66, 0x79, 0xd2, 0xb8, 0xc2, 0xfe, - 0x8e, 0x71, 0xb6, 0xee, 0xae, 0x41, 0x77, 0xb9, 0x9a, 0xd4, 0xdc, 0x6c, 0x5d, 0x7c, 0x4e, 0x40, - 0x2d, 0x60, 0x1d, 0xdd, 0x6b, 0xb5, 0xb8, 0x75, 0xe9, 0x39, 0xeb, 0xe8, 0xc8, 0x75, 0x11, 0x73, - 0x1d, 0x52, 0x58, 0x7b, 0xa7, 0xb3, 0xdd, 0xd8, 0x6d, 0x94, 0x1e, 0xa4, 0xe7, 0x16, 0x54, 0xf3, - 0x06, 0x43, 0x87, 0x85, 0xc2, 0xbc, 0xee, 0xd6, 0xde, 0x2e, 0x7f, 0x72, 0xbe, 0x9f, 0x99, 0x9f, - 0xf0, 0x78, 0x14, 0xf6, 0xb1, 0x6c, 0xbc, 0x16, 0xd5, 0x71, 0x0f, 0x32, 0x22, 0x65, 0x46, 0x18, - 0x51, 0xc4, 0xa1, 0x1e, 0xda, 0xf8, 0xbe, 0x78, 0x9d, 0xbe, 0x9f, 0x9d, 0xd3, 0x43, 0xd9, 0xbb, - 0x10, 0x05, 0xa1, 0xd4, 0x8b, 0x9e, 0xa4, 0x22, 0xd6, 0x8b, 0x3f, 0x22, 0x39, 0x15, 0x30, 0x60, - 0x77, 0xb2, 0x77, 0xdb, 0xf4, 0x4e, 0x83, 0xc2, 0xd1, 0xf3, 0xdd, 0x51, 0x9c, 0xbb, 0x9e, 0x0f, - 0xde, 0x05, 0xcb, 0x58, 0xdc, 0x31, 0x5a, 0xc6, 0x6d, 0x00, 0xc8, 0x37, 0x2f, 0x05, 0x90, 0x5e, - 0x5f, 0x2e, 0xc9, 0x09, 0x22, 0x9d, 0xd5, 0xb5, 0x87, 0x9f, 0xac, 0x2f, 0x7c, 0x04, 0xbf, 0x7f, - 0x7d, 0xb2, 0x9e, 0xb8, 0xff, 0xe9, 0x7a, 0xe2, 0x21, 0xfc, 0xfe, 0x0c, 0xbf, 0xbf, 0xc3, 0xef, - 0x20, 0xcb, 0xeb, 0x97, 0xd7, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x61, 0x4f, 0x38, 0x13, 0x5f, - 0x1d, 0x00, 0x00, + 0xf4, 0xe4, 0xa3, 0x8f, 0x2d, 0x0a, 0x14, 0x01, 0x0a, 0x04, 0x4d, 0xfa, 0x0f, 0x04, 0xe8, 0x21, + 0x87, 0xf6, 0xd0, 0x37, 0x1f, 0xbb, 0x4b, 0xd2, 0x6b, 0xc5, 0x6e, 0x72, 0x20, 0xb8, 0xf3, 0xde, + 0xef, 0xbd, 0x99, 0x37, 0xf3, 0xe6, 0x7d, 0x0c, 0x14, 0xfc, 0xc9, 0x88, 0x7a, 0x95, 0x91, 0xeb, + 0xf8, 0x8e, 0xa2, 0xf4, 0x1d, 0xe3, 0x84, 0xba, 0x15, 0xef, 0x4c, 0x77, 0x87, 0x27, 0xa6, 0x5f, + 0x39, 0xbd, 0x59, 0xbe, 0xe2, 0x9b, 0x43, 0xea, 0xf9, 0xfa, 0x70, 0xf4, 0x5a, 0xf8, 0x25, 0xe0, + 0xe5, 0x67, 0xfb, 0x63, 0x57, 0xf7, 0x4d, 0xc7, 0x7e, 0x2d, 0xf8, 0x90, 0x8c, 0x4b, 0xc7, 0xce, + 0xb1, 0xc3, 0x3f, 0x5f, 0x63, 0x5f, 0x82, 0xaa, 0x5e, 0x85, 0xc5, 0x7d, 0xea, 0x7a, 0x08, 0x53, + 0x2e, 0x41, 0xc6, 0xb4, 0xfb, 0xf4, 0xde, 0x6a, 0xe2, 0x5a, 0xe2, 0x46, 0x9a, 0x88, 0x81, 0xfa, + 0xdb, 0x04, 0x14, 0x34, 0xdb, 0x76, 0x7c, 0xae, 0xcb, 0x53, 0x14, 0x48, 0xdb, 0xfa, 0x90, 0x72, + 0x50, 0x9e, 0xf0, 0x6f, 0xa5, 0x06, 0x59, 0x4b, 0x3f, 0xa4, 0x96, 0xb7, 0x9a, 0xbc, 0x96, 0xba, + 0x51, 0xd8, 0x78, 0xa5, 0xf2, 0xe8, 0x9a, 0x2b, 0x53, 0x4a, 0x2a, 0xdb, 0x1c, 0xdd, 0xb0, 0x7d, + 0x77, 0x42, 0xa4, 0x68, 0xf9, 0x7b, 0x50, 0x98, 0x22, 0x2b, 0x25, 0x48, 0x9d, 0xd0, 0x89, 0x9c, + 0x86, 0x7d, 0xb2, 0xf5, 0x9d, 0xea, 0xd6, 0x98, 0xe2, 0x24, 0x8c, 0x26, 0x06, 0x6f, 0x25, 0x6f, + 0x25, 0xd4, 0x77, 0x21, 0x4f, 0xa8, 0xe7, 0x8c, 0x5d, 0x83, 0x7a, 0xca, 0x4b, 0x90, 0xb7, 0x75, + 0xdb, 0xe9, 0x19, 0xa3, 0xb1, 0xc7, 0xc5, 0x53, 0xd5, 0xe2, 0x67, 0x9f, 0x5c, 0xcd, 0xb5, 0x90, + 0x58, 0xeb, 0xec, 0x79, 0x24, 0xc7, 0xd8, 0x35, 0xe4, 0x2a, 0xdf, 0x84, 0xe2, 0x90, 0x0e, 0x1d, + 0x77, 0xd2, 0x3b, 0x9c, 0xf8, 0xd4, 0xe3, 0x8a, 0x53, 0xa4, 0x20, 0x68, 0x55, 0x46, 0x52, 0x7f, + 0x9d, 0x80, 0x4b, 0x81, 0x6e, 0x42, 0x7f, 0x32, 0x36, 0x5d, 0x3a, 0xa4, 0xb6, 0xef, 0x29, 0xdf, + 0x41, 0x9b, 0xcd, 0xa1, 0xe9, 0x8b, 0x39, 0x0a, 0x1b, 0xdf, 0x88, 0xb3, 0x39, 0x5c, 0x15, 0x91, + 0x60, 0x45, 0x83, 0xa2, 0x4b, 0x3d, 0xea, 0x9e, 0x8a, 0x9d, 0xe0, 0x53, 0x7e, 0xa9, 0xf0, 0x8c, + 0x88, 0xba, 0x09, 0xb9, 0x8e, 0xa5, 0xfb, 0x47, 0x8e, 0x3b, 0x54, 0x54, 0x28, 0xea, 0xae, 0x31, + 0x30, 0x7d, 0x6a, 0xf8, 0x63, 0x37, 0x38, 0x95, 0x19, 0x9a, 0x72, 0x19, 0x92, 0x8e, 0x98, 0x28, + 0x5f, 0xcd, 0xe2, 0x4e, 0x24, 0xdb, 0x5d, 0x82, 0x14, 0xf5, 0x6d, 0xb8, 0xd0, 0xb1, 0xc6, 0xc7, + 0xa6, 0x5d, 0xa7, 0x9e, 0xe1, 0x9a, 0x23, 0xa6, 0x9d, 0x1d, 0x2f, 0x73, 0xbe, 0xe0, 0x78, 0xd9, + 0x77, 0x78, 0xe4, 0xc9, 0xe8, 0xc8, 0xd5, 0x5f, 0x24, 0xe1, 0x42, 0xc3, 0x46, 0x61, 0x3a, 0x2d, + 0x7d, 0x1d, 0x96, 0x29, 0x27, 0xf6, 0x4e, 0x85, 0x53, 0x49, 0x3d, 0x4b, 0x82, 0x1a, 0x78, 0x5a, + 0x73, 0xce, 0x5f, 0x6e, 0xc6, 0x99, 0xff, 0x88, 0xf6, 0x38, 0xaf, 0x51, 0x1a, 0xb0, 0x38, 0xe2, + 0x46, 0x78, 0xab, 0x29, 0xae, 0xeb, 0x7a, 0x9c, 0xae, 0x47, 0xec, 0xac, 0xa6, 0x1f, 0x7e, 0x72, + 0x75, 0x81, 0x04, 0xb2, 0x5f, 0xc5, 0xf9, 0xfe, 0x95, 0x80, 0x95, 0x96, 0xd3, 0x9f, 0xd9, 0x87, + 0x32, 0xe4, 0x06, 0x8e, 0xe7, 0x4f, 0x5d, 0x94, 0x70, 0xac, 0xdc, 0x82, 0xdc, 0x48, 0x1e, 0x9f, + 0x3c, 0xfd, 0xb5, 0xf8, 0x25, 0x0b, 0x0c, 0x09, 0xd1, 0xca, 0xdb, 0x90, 0x77, 0x03, 0x9f, 0x40, + 0x6b, 0x9f, 0xc0, 0x71, 0x22, 0xbc, 0xf2, 0x7d, 0xc8, 0x8a, 0x43, 0x58, 0x4d, 0x73, 0xc9, 0xeb, + 0x4f, 0xb4, 0xe7, 0x44, 0x0a, 0xa9, 0x1f, 0x27, 0xa0, 0x44, 0xf4, 0x23, 0x7f, 0x87, 0x0e, 0x0f, + 0xa9, 0xdb, 0xc5, 0x8b, 0x8c, 0xf7, 0xe7, 0x32, 0x9e, 0x23, 0xd5, 0xfb, 0xd4, 0xe5, 0x46, 0xe6, + 0x88, 0x1c, 0x29, 0x7b, 0xcc, 0xc9, 0x75, 0x63, 0xa0, 0x1f, 0x9a, 0x96, 0xe9, 0x4f, 0xb8, 0x99, + 0xcb, 0xf1, 0xa7, 0x3c, 0xaf, 0x13, 0x17, 0x1f, 0x09, 0x92, 0x19, 0x35, 0xca, 0x2a, 0x2c, 0x62, + 0xac, 0xf3, 0xf4, 0x63, 0xca, 0xad, 0xcf, 0x93, 0x60, 0x88, 0xae, 0x5c, 0x9c, 0x96, 0x53, 0x0a, + 0xb0, 0xb8, 0xd7, 0xba, 0xdb, 0x6a, 0x1f, 0xb4, 0x4a, 0x0b, 0xca, 0x0a, 0x14, 0xf6, 0x5a, 0xa4, + 0xa1, 0xd5, 0xb6, 0xb4, 0xea, 0x76, 0xa3, 0x94, 0x50, 0x96, 0x30, 0x5c, 0x84, 0xc3, 0xa4, 0xfa, + 0x51, 0x02, 0x80, 0x1d, 0xa0, 0x34, 0xea, 0x2d, 0xc8, 0x60, 0x3c, 0xf5, 0xc5, 0xc1, 0x2d, 0x6f, + 0xbc, 0x10, 0xb7, 0xea, 0x08, 0x5e, 0x61, 0x7f, 0x94, 0x08, 0x91, 0xe9, 0x15, 0x26, 0xe7, 0x57, + 0x98, 0xe1, 0xc8, 0xd9, 0xa5, 0xe5, 0x20, 0x5d, 0x67, 0x5f, 0x09, 0x25, 0x0f, 0x19, 0x5c, 0x53, + 0xfd, 0xdd, 0x52, 0x12, 0x9d, 0xaf, 0x58, 0x6f, 0x76, 0x6b, 0xed, 0x56, 0xab, 0x51, 0xdb, 0x6d, + 0xd4, 0x4b, 0x29, 0xf5, 0x3a, 0x64, 0x9a, 0x43, 0xd4, 0xa2, 0xac, 0x31, 0x0f, 0x38, 0xa2, 0x2e, + 0xb5, 0x8d, 0xc0, 0xb1, 0x22, 0x82, 0xfa, 0xab, 0x1c, 0x64, 0x76, 0x9c, 0xb1, 0xed, 0x2b, 0x1b, + 0x53, 0xb7, 0x78, 0x79, 0x63, 0x3d, 0xce, 0x04, 0x0e, 0xac, 0xec, 0x22, 0x4a, 0xde, 0x72, 0x3c, + 0x4c, 0xe1, 0x2b, 0x72, 0xe9, 0x72, 0xc4, 0xe8, 0xbe, 0xee, 0x1e, 0x53, 0x5f, 0x6e, 0xba, 0x1c, + 0x29, 0x37, 0x20, 0x87, 0xa7, 0xd3, 0x77, 0x6c, 0x6b, 0xc2, 0x5d, 0x2a, 0x27, 0xc2, 0x2c, 0x9e, + 0x43, 0xbf, 0x8d, 0x34, 0x12, 0x72, 0x95, 0x2d, 0x28, 0x1e, 0x62, 0x32, 0xe9, 0x39, 0x23, 0x11, + 0xf3, 0x32, 0x8f, 0x77, 0x40, 0xb1, 0xaa, 0x2a, 0xa2, 0xdb, 0x02, 0x4c, 0x0a, 0x87, 0xd1, 0x40, + 0x69, 0xc1, 0xf2, 0xa9, 0x63, 0x8d, 0x87, 0x34, 0xd4, 0x95, 0xe5, 0xba, 0x5e, 0x7c, 0xbc, 0xae, + 0x7d, 0x8e, 0x0f, 0xb4, 0x2d, 0x9d, 0x4e, 0x0f, 0xcb, 0x3f, 0x4f, 0x41, 0x61, 0x6a, 0x32, 0xa5, + 0x0b, 0x05, 0x4c, 0x8b, 0x23, 0xfd, 0x98, 0x87, 0x5a, 0xb9, 0x7d, 0x37, 0x9f, 0x68, 0xa1, 0x95, + 0x4e, 0x24, 0x48, 0xa6, 0xb5, 0xa8, 0x1f, 0x26, 0xa1, 0x30, 0xc5, 0x54, 0x5e, 0x86, 0x1c, 0xe9, + 0x90, 0xe6, 0xbe, 0xb6, 0xdb, 0x28, 0x2d, 0x94, 0xd7, 0x3e, 0xf8, 0xf0, 0xda, 0x2a, 0xd7, 0x36, + 0xad, 0xa0, 0xe3, 0x9a, 0xa7, 0xcc, 0x5b, 0x6e, 0xc0, 0x62, 0x00, 0x4d, 0x94, 0x9f, 0x43, 0xe8, + 0xb3, 0xf3, 0xd0, 0x29, 0x24, 0xe9, 0x6e, 0x69, 0x04, 0x1d, 0x26, 0x19, 0x8f, 0x24, 0xdd, 0x81, + 0xee, 0xd2, 0xbe, 0xf2, 0x2d, 0xc8, 0x4a, 0x60, 0xaa, 0x5c, 0x46, 0xe0, 0xe5, 0x79, 0x60, 0x84, + 0x23, 0xdd, 0x6d, 0x6d, 0xbf, 0x51, 0x4a, 0xc7, 0xe3, 0x48, 0xd7, 0xd2, 0x4f, 0xa9, 0xf2, 0x02, + 0xba, 0x36, 0x87, 0x65, 0xca, 0x57, 0x10, 0xf6, 0xcc, 0x23, 0xea, 0x18, 0xaa, 0xbc, 0xfa, 0xcb, + 0xdf, 0xad, 0x2f, 0xfc, 0xf9, 0xf7, 0xeb, 0xa5, 0x79, 0x76, 0xf9, 0xbf, 0x09, 0x58, 0x9a, 0x39, + 0x25, 0xcc, 0x6a, 0x59, 0xdb, 0x31, 0x9c, 0x91, 0x88, 0xc0, 0xb9, 0x2a, 0xa0, 0x63, 0x65, 0x5b, + 0x4e, 0x0d, 0x29, 0x44, 0x72, 0x94, 0xbb, 0x73, 0x39, 0xe4, 0x8d, 0x27, 0x74, 0x81, 0xd8, 0x2c, + 0x72, 0x1b, 0x96, 0xfa, 0xb8, 0x8f, 0xd4, 0xed, 0x19, 0x8e, 0x7d, 0x64, 0x1e, 0xcb, 0xe8, 0x5a, + 0x8e, 0xd3, 0x59, 0xe7, 0x40, 0x52, 0x14, 0x02, 0x35, 0x8e, 0xff, 0x2a, 0xf9, 0xe3, 0x00, 0xd2, + 0xec, 0x16, 0x2a, 0xcf, 0x41, 0xba, 0xda, 0x6c, 0xd5, 0xd1, 0x25, 0x2e, 0xe0, 0x2e, 0x2e, 0xf1, + 0xa5, 0x33, 0x06, 0xf3, 0x31, 0xe5, 0x2a, 0x64, 0xf7, 0xdb, 0xdb, 0x7b, 0x3b, 0xcc, 0x0d, 0x2e, + 0x22, 0x7b, 0x25, 0x64, 0x0b, 0xe3, 0xca, 0x17, 0xe4, 0xf6, 0xe6, 0x43, 0x86, 0xfa, 0x9f, 0x24, + 0x2c, 0x11, 0x56, 0x1b, 0xba, 0x7e, 0xc7, 0xb1, 0x4c, 0x63, 0xa2, 0x74, 0x20, 0x8f, 0xf6, 0xf5, + 0xcd, 0x29, 0xe7, 0xde, 0x78, 0x4c, 0x02, 0x89, 0xa4, 0x82, 0x51, 0x2d, 0x90, 0x24, 0x91, 0x12, + 0x0c, 0x34, 0x99, 0x3e, 0xb5, 0xf4, 0xc9, 0x79, 0x99, 0xac, 0x2e, 0xeb, 0x50, 0x22, 0xa0, 0xbc, + 0xea, 0xd2, 0xef, 0xf5, 0x74, 0xdf, 0xa7, 0xc3, 0x91, 0x2f, 0x32, 0x59, 0x1a, 0xab, 0x2e, 0xfd, + 0x9e, 0x26, 0x49, 0xca, 0x9b, 0x90, 0x3d, 0x43, 0xb3, 0x9d, 0x33, 0x99, 0xac, 0xce, 0xd7, 0x2b, + 0xb1, 0xea, 0x07, 0x2c, 0x47, 0xcd, 0x2d, 0x96, 0x6d, 0x6b, 0xab, 0xdd, 0x6a, 0x04, 0xdb, 0x2a, + 0xf9, 0x6d, 0xbb, 0xe5, 0xd8, 0xcc, 0x75, 0xa1, 0xdd, 0xea, 0x6d, 0x6a, 0xcd, 0xed, 0x3d, 0xc2, + 0xb6, 0xf6, 0x12, 0x42, 0x4a, 0x21, 0x64, 0x53, 0x37, 0x2d, 0x56, 0x40, 0x5d, 0x81, 0x94, 0xd6, + 0xc2, 0xc8, 0x5c, 0x2e, 0x21, 0xbb, 0x18, 0xb2, 0x35, 0x7b, 0x12, 0x79, 0xf5, 0xfc, 0xbc, 0xea, + 0x7b, 0x50, 0xdc, 0x1b, 0xf5, 0xf1, 0x66, 0x0a, 0x0f, 0x51, 0xae, 0x61, 0x68, 0xd1, 0x5d, 0xdd, + 0xb2, 0xa8, 0x65, 0x7a, 0x43, 0x59, 0x63, 0x4f, 0x93, 0xb0, 0x30, 0x78, 0xf2, 0xbd, 0x94, 0xf5, + 0x8b, 0x10, 0x50, 0x7f, 0x06, 0x2b, 0x38, 0x8b, 0xaf, 0x63, 0xa2, 0x0e, 0x52, 0xf3, 0x06, 0x14, + 0x8d, 0x80, 0xd4, 0x33, 0xfb, 0xc2, 0x15, 0xab, 0x2b, 0x78, 0x91, 0x0a, 0x21, 0xb4, 0x59, 0x27, + 0x85, 0x10, 0xd4, 0xec, 0x33, 0x3b, 0x47, 0x08, 0x65, 0xd3, 0x67, 0xaa, 0x8b, 0x08, 0x4d, 0x75, + 0x10, 0xc2, 0x68, 0xb8, 0x8b, 0x79, 0x7a, 0xcf, 0xf4, 0xf1, 0x7a, 0xf4, 0x45, 0xf2, 0xcd, 0x90, + 0x1c, 0x23, 0xd4, 0x70, 0xac, 0xbe, 0x9f, 0x04, 0xd8, 0xd5, 0xbd, 0x13, 0x39, 0x35, 0x96, 0x29, + 0x61, 0x53, 0x72, 0x5e, 0x71, 0xbc, 0x1b, 0x80, 0x48, 0x84, 0x57, 0xde, 0x08, 0xb2, 0xaf, 0xa8, + 0x19, 0xe2, 0x05, 0xe5, 0x5c, 0x71, 0x69, 0x77, 0xb6, 0x30, 0x60, 0x17, 0x91, 0xba, 0x2e, 0xf7, + 0x22, 0xbc, 0x88, 0xf8, 0x89, 0xbd, 0x4a, 0x3e, 0xb4, 0x59, 0x66, 0xa2, 0xe7, 0xe3, 0x26, 0x99, + 0xdb, 0xd0, 0xad, 0x05, 0x12, 0xc9, 0x55, 0x4b, 0xb0, 0xec, 0xe2, 0x35, 0xc3, 0x55, 0xf7, 0x3c, + 0xce, 0x56, 0xff, 0x86, 0x7b, 0xd0, 0xec, 0x68, 0x3b, 0xf2, 0xb4, 0xeb, 0x90, 0x3d, 0xd2, 0x87, + 0xa6, 0x35, 0x91, 0xd7, 0xec, 0xd5, 0xb8, 0x29, 0x22, 0x7c, 0x45, 0xeb, 0xf7, 0xb1, 0x54, 0xf3, + 0x36, 0xb9, 0x0c, 0x91, 0xb2, 0x3c, 0x25, 0x8f, 0x0f, 0x6d, 0x4c, 0xbd, 0x41, 0x4a, 0xe6, 0x23, + 0x16, 0x4c, 0x5c, 0xdd, 0x0e, 0xad, 0x15, 0x03, 0xb6, 0x0b, 0x18, 0x51, 0xe9, 0x99, 0x3e, 0x91, + 0xf6, 0x06, 0x43, 0x4c, 0xc0, 0x39, 0xd1, 0x41, 0xd0, 0x3e, 0x9a, 0xcc, 0xa2, 0xe5, 0x97, 0xad, + 0x87, 0x48, 0xb8, 0x08, 0x93, 0xa1, 0x74, 0xf9, 0x6d, 0x1e, 0x52, 0x22, 0xd6, 0x53, 0x45, 0xba, + 0xd7, 0x61, 0x69, 0xc6, 0xce, 0x47, 0x6a, 0xa1, 0x66, 0x67, 0xff, 0xcd, 0x52, 0x5a, 0x7e, 0x7d, + 0xb7, 0x94, 0x55, 0xff, 0x8d, 0xa5, 0x59, 0xc7, 0xe1, 0xd7, 0x8a, 0xed, 0x6a, 0x7c, 0xef, 0x99, + 0xe3, 0x9d, 0xac, 0xe1, 0x58, 0xd2, 0x67, 0x62, 0x8b, 0x81, 0x48, 0x0b, 0x4b, 0xd4, 0x1c, 0x4e, + 0x42, 0x41, 0x0c, 0xaf, 0x05, 0x51, 0xd5, 0xf4, 0x46, 0x88, 0xe3, 0xdb, 0xba, 0x44, 0x40, 0x90, + 0x98, 0x24, 0x6b, 0x6c, 0x46, 0xe3, 0x43, 0xbc, 0xa6, 0x03, 0xda, 0x17, 0x98, 0x34, 0xc7, 0x2c, + 0x85, 0x54, 0x06, 0x53, 0xeb, 0xd8, 0x9a, 0x05, 0x3a, 0x57, 0x21, 0xb5, 0x5b, 0xeb, 0x60, 0xdc, + 0x59, 0xc1, 0xa8, 0x51, 0x08, 0xc8, 0x48, 0x62, 0x9c, 0xbd, 0x7a, 0x07, 0xc3, 0xcd, 0x0c, 0x07, + 0x49, 0xe5, 0x34, 0x0b, 0x27, 0xea, 0x6f, 0x12, 0x90, 0x15, 0x59, 0x26, 0xd6, 0x62, 0x0d, 0x16, + 0x83, 0xea, 0x47, 0xa4, 0xbe, 0x17, 0x1f, 0x9f, 0xa6, 0x2a, 0x32, 0xeb, 0x89, 0x73, 0x0c, 0xe4, + 0xca, 0x6f, 0x41, 0x71, 0x9a, 0xf1, 0x54, 0xa7, 0xf8, 0x53, 0x28, 0x30, 0x47, 0x09, 0x72, 0xf5, + 0x06, 0x64, 0x45, 0x26, 0x94, 0x57, 0xfd, 0xbc, 0x9c, 0x29, 0x91, 0x18, 0xe9, 0x16, 0x45, 0x9e, + 0x0d, 0x9a, 0xb6, 0xf5, 0xf3, 0xdd, 0x91, 0x04, 0x70, 0xf5, 0x36, 0xa4, 0x3b, 0x14, 0x35, 0x3c, + 0x0f, 0x8b, 0x36, 0x86, 0x9e, 0x28, 0xb2, 0xc9, 0x12, 0xa1, 0x4f, 0x31, 0x62, 0x65, 0x19, 0x0b, + 0xe3, 0x19, 0x6e, 0x9e, 0x8e, 0xfe, 0x16, 0xf4, 0xad, 0xec, 0x5b, 0xdd, 0x85, 0xe2, 0x01, 0x35, + 0x8f, 0x07, 0x3e, 0x9e, 0x18, 0x53, 0xf4, 0x2a, 0xa4, 0x47, 0x34, 0x5c, 0xfc, 0x6a, 0xac, 0xeb, + 0x20, 0x9f, 0x70, 0x14, 0xbb, 0x90, 0x67, 0x5c, 0x5a, 0x3e, 0x15, 0xc8, 0x91, 0xfa, 0xc7, 0x24, + 0x2c, 0x37, 0x3d, 0x6f, 0xac, 0x63, 0x19, 0x2e, 0xa3, 0xe0, 0x0f, 0x66, 0xdb, 0x88, 0x1b, 0xb1, + 0x16, 0xce, 0x88, 0xcc, 0xb6, 0x12, 0x32, 0x72, 0x25, 0xc3, 0xc8, 0xa5, 0x3e, 0x4c, 0x04, 0x3d, + 0xc4, 0xf5, 0xa9, 0x7b, 0x53, 0x5e, 0x45, 0x27, 0xba, 0x34, 0xad, 0x89, 0xee, 0xd9, 0x27, 0xb6, + 0x73, 0x66, 0x63, 0xa2, 0xc5, 0x9e, 0xa2, 0xd5, 0x38, 0x40, 0x4f, 0xbb, 0x8c, 0x20, 0x65, 0x06, + 0x44, 0xa8, 0x4d, 0xcf, 0x98, 0xa6, 0x4e, 0xa3, 0x55, 0x6f, 0xb6, 0xee, 0x60, 0x7a, 0x7b, 0x54, + 0x53, 0x87, 0x62, 0x3a, 0xb3, 0x8f, 0x71, 0xbb, 0xb3, 0xcd, 0x6e, 0x77, 0x8f, 0x97, 0x8c, 0xcf, + 0x22, 0xea, 0xe2, 0x0c, 0x8a, 0x0d, 0xb0, 0x5e, 0x44, 0x10, 0xcb, 0xa4, 0x08, 0x4a, 0xc7, 0x80, + 0x58, 0x32, 0xc5, 0x00, 0x22, 0x3c, 0xfc, 0x1f, 0x49, 0x28, 0x69, 0x86, 0x41, 0x47, 0x3e, 0xe3, + 0xcb, 0xea, 0x64, 0x17, 0x6f, 0x32, 0xfb, 0x32, 0x29, 0x7b, 0x53, 0x61, 0x6e, 0x71, 0x2b, 0xf6, + 0x1d, 0x69, 0x4e, 0xae, 0x42, 0x1c, 0x8b, 0x6a, 0xfd, 0xa1, 0xe9, 0xb1, 0xb7, 0x05, 0x41, 0x23, + 0xa1, 0xa6, 0xf2, 0xe7, 0x09, 0xb8, 0x18, 0x83, 0x50, 0x5e, 0x87, 0xb4, 0x8b, 0x64, 0x79, 0x3c, + 0x6b, 0x8f, 0xeb, 0xf2, 0x98, 0x28, 0xe1, 0x48, 0x65, 0x1d, 0x40, 0x1f, 0xfb, 0x8e, 0xce, 0xe7, + 0xe7, 0x07, 0x93, 0x23, 0x53, 0x14, 0xe5, 0x00, 0xa3, 0x35, 0x35, 0x5c, 0xd9, 0x28, 0x15, 0x36, + 0x6e, 0xff, 0xbf, 0xab, 0xaf, 0x74, 0xb9, 0x1a, 0x22, 0xd5, 0x95, 0x2b, 0x58, 0xb0, 0xf3, 0x2f, + 0xe6, 0xd1, 0x58, 0x52, 0xe8, 0x7c, 0xd1, 0x45, 0xc2, 0xbf, 0x99, 0xa3, 0xe8, 0xd6, 0x71, 0xe0, + 0x28, 0xf8, 0xa9, 0x7e, 0x84, 0xb9, 0xa8, 0x71, 0xcf, 0xa7, 0xae, 0xad, 0x5b, 0x35, 0x4d, 0x69, + 0x4c, 0x45, 0x48, 0x61, 0xed, 0x4b, 0xb1, 0xbd, 0x7f, 0x28, 0x51, 0xa9, 0x69, 0x31, 0x31, 0x12, + 0xab, 0x83, 0xb1, 0x6b, 0xc9, 0x77, 0x24, 0x5e, 0x1d, 0xec, 0x91, 0x6d, 0xc2, 0x68, 0xec, 0x11, + 0x26, 0x88, 0x48, 0xa9, 0xc7, 0x3f, 0x00, 0x4e, 0x4d, 0xf0, 0xf5, 0x47, 0xa5, 0x57, 0x01, 0xa2, + 0x55, 0xe3, 0x51, 0x65, 0x6a, 0x9b, 0xdd, 0xee, 0x36, 0x5e, 0x0f, 0x5e, 0x2d, 0x47, 0x2c, 0x4e, + 0x56, 0xff, 0x90, 0x80, 0x5c, 0x4d, 0x93, 0x59, 0x65, 0x13, 0x4a, 0x3c, 0x96, 0x18, 0xd4, 0xf5, + 0x7b, 0xf4, 0xde, 0xc8, 0x74, 0x27, 0x32, 0x1c, 0x9c, 0x5f, 0x76, 0x2e, 0x33, 0xa9, 0x1a, 0x0a, + 0x35, 0xb8, 0x8c, 0x42, 0xa0, 0x48, 0xa5, 0x89, 0x3d, 0x43, 0x0f, 0x82, 0xf3, 0xfa, 0xf9, 0x5b, + 0x21, 0x4a, 0xb2, 0x68, 0x8c, 0x0d, 0x6f, 0xa0, 0xa4, 0xa6, 0x7b, 0xea, 0x3e, 0x5c, 0x6c, 0xbb, + 0xc6, 0x00, 0x8b, 0x23, 0x31, 0xa9, 0x5c, 0xf2, 0x6d, 0x58, 0xf3, 0xb1, 0x08, 0xea, 0x0d, 0x4c, + 0xcf, 0x67, 0xcf, 0x97, 0xe8, 0x1b, 0xd4, 0x66, 0xfc, 0x1e, 0x7f, 0x66, 0x14, 0xcf, 0x9e, 0xe4, + 0x0a, 0xc3, 0x6c, 0x09, 0x08, 0x09, 0x10, 0xdb, 0x0c, 0xa0, 0xfe, 0x08, 0x4a, 0x75, 0xd3, 0x1b, + 0xe9, 0x3e, 0xea, 0x96, 0x3d, 0x8c, 0x72, 0x07, 0x4a, 0x03, 0x8a, 0x35, 0xec, 0x21, 0xd5, 0x31, + 0x0f, 0x52, 0xd7, 0x74, 0xfa, 0x4f, 0xb4, 0x0f, 0x2b, 0xa1, 0x54, 0x87, 0x0b, 0xa9, 0x5f, 0x60, + 0xd6, 0x66, 0xef, 0x3a, 0x52, 0xef, 0x2b, 0x70, 0xc1, 0xb3, 0xf5, 0x91, 0x37, 0x70, 0xfc, 0x9e, + 0x69, 0xfb, 0xec, 0x21, 0xd3, 0x92, 0xf5, 0x6f, 0x29, 0x60, 0x34, 0x25, 0x1d, 0xe3, 0xb1, 0x72, + 0x42, 0xe9, 0xa8, 0xe7, 0x58, 0xfd, 0x5e, 0xc0, 0x14, 0x8f, 0x97, 0x88, 0x66, 0x9c, 0xb6, 0xd5, + 0xef, 0x06, 0x74, 0xa5, 0x0a, 0xeb, 0x96, 0x73, 0xdc, 0x43, 0xcb, 0x5c, 0xbc, 0xeb, 0xbd, 0x23, + 0xc7, 0xed, 0x79, 0x96, 0x73, 0x86, 0x1f, 0x16, 0xfe, 0x51, 0x37, 0x68, 0x2e, 0xca, 0x88, 0x6a, + 0x08, 0xd0, 0xa6, 0xe3, 0x76, 0x91, 0xb7, 0x19, 0x20, 0x58, 0x6a, 0x8f, 0xcc, 0xf6, 0x4d, 0xe3, + 0x24, 0x48, 0xed, 0x21, 0x75, 0x17, 0x89, 0x18, 0xdd, 0x96, 0xa8, 0x45, 0x0d, 0xbe, 0xc9, 0x1c, + 0x95, 0xe1, 0xa8, 0x62, 0x40, 0x64, 0x20, 0xf5, 0xdb, 0x90, 0xef, 0x58, 0xba, 0xc1, 0x9f, 0x88, + 0x59, 0xc5, 0x8f, 0x69, 0x8b, 0x9d, 0x1c, 0x5a, 0x2d, 0x42, 0x5a, 0x9e, 0x4c, 0x93, 0xd4, 0xf7, + 0x31, 0xd1, 0x13, 0xc7, 0xf1, 0xf1, 0x92, 0x5e, 0x83, 0xac, 0xa1, 0xf7, 0x02, 0x77, 0x2f, 0x56, + 0xf3, 0xe8, 0x16, 0x99, 0x9a, 0x76, 0x97, 0x4e, 0x48, 0xc6, 0xd0, 0xf1, 0x8f, 0xa5, 0x3c, 0x44, + 0x30, 0x27, 0xe5, 0xdb, 0x51, 0x14, 0x29, 0x0f, 0xbd, 0x18, 0x29, 0x04, 0x85, 0xd9, 0x3f, 0x46, + 0xb5, 0xa2, 0x04, 0xf5, 0x06, 0xba, 0x37, 0x10, 0x05, 0x62, 0x75, 0x19, 0x91, 0x20, 0x90, 0x5b, + 0x48, 0x25, 0x20, 0xd0, 0xec, 0x5b, 0xfd, 0x7b, 0x02, 0x0a, 0x6c, 0x60, 0x1e, 0x99, 0x06, 0xcb, + 0x2d, 0x4f, 0x1f, 0x17, 0x31, 0x30, 0x18, 0x9e, 0x2b, 0x17, 0xc5, 0x03, 0x43, 0xad, 0x4b, 0x08, + 0xa3, 0x29, 0xef, 0x60, 0x48, 0xe4, 0xb9, 0x4d, 0x86, 0x44, 0xf5, 0xcb, 0xb3, 0xa0, 0xec, 0x6c, + 0xa4, 0x1c, 0xdf, 0xc4, 0x68, 0x75, 0xfc, 0x68, 0x8a, 0x64, 0x9a, 0xc4, 0x9e, 0xb7, 0x0d, 0x9b, + 0x9f, 0x86, 0x7c, 0xde, 0xae, 0xb5, 0x08, 0x52, 0xd4, 0xbf, 0x26, 0x60, 0xa9, 0x61, 0x1b, 0xee, + 0x84, 0x87, 0x14, 0xb6, 0x83, 0x6b, 0x90, 0xc7, 0x02, 0xda, 0x9b, 0x78, 0xd8, 0x65, 0x06, 0xaf, + 0x67, 0x21, 0x41, 0x69, 0x42, 0x1e, 0x83, 0xa7, 0xe3, 0x9a, 0xfe, 0x60, 0x28, 0x2b, 0xc9, 0xf8, + 0x30, 0x36, 0xad, 0xb3, 0xa2, 0x05, 0x22, 0x24, 0x92, 0x0e, 0x02, 0x57, 0x8a, 0x2f, 0x96, 0x07, + 0x2e, 0xec, 0x79, 0x2d, 0x6c, 0x6f, 0xb0, 0x46, 0xec, 0xb1, 0xae, 0x81, 0xdb, 0x81, 0xed, 0x9f, + 0xa4, 0xb1, 0x4e, 0x48, 0x55, 0x21, 0x1f, 0x2a, 0x63, 0x6f, 0x96, 0x5a, 0xa3, 0xdb, 0xbb, 0xb9, + 0x71, 0xab, 0x77, 0xa7, 0xb6, 0x83, 0x61, 0x4c, 0xe4, 0xcd, 0x3f, 0xa1, 0x4d, 0x3b, 0xba, 0x8d, + 0x9d, 0x4d, 0xd0, 0xe7, 0xa1, 0x57, 0xb8, 0x78, 0xd5, 0x82, 0x42, 0x28, 0x2d, 0xbc, 0x82, 0xdd, + 0x3e, 0x56, 0x08, 0x31, 0x56, 0x7c, 0x21, 0x34, 0xf5, 0x76, 0x9b, 0x3a, 0xf7, 0xed, 0x36, 0xfd, + 0xb5, 0xbc, 0xdd, 0xbe, 0xfc, 0x45, 0x0a, 0xf2, 0x61, 0xdf, 0xc6, 0x5c, 0x86, 0xd5, 0x25, 0x0b, + 0xa2, 0xa3, 0x0e, 0xe9, 0x2d, 0x5e, 0x91, 0xe4, 0xb5, 0xed, 0xed, 0x76, 0x4d, 0x63, 0x4f, 0x9f, + 0xef, 0x88, 0xc2, 0x25, 0x04, 0x68, 0x78, 0x69, 0xd9, 0xa1, 0xf7, 0x15, 0x35, 0x2a, 0x5c, 0xee, + 0xcb, 0xbe, 0x3d, 0x44, 0x05, 0x55, 0xcb, 0x0b, 0x90, 0xd3, 0xba, 0xdd, 0xe6, 0x9d, 0x16, 0x6a, + 0x7a, 0x90, 0x28, 0x3f, 0x83, 0xa0, 0x0b, 0x91, 0x2a, 0x4c, 0xb8, 0xc7, 0x36, 0x6a, 0x62, 0xa8, + 0x5a, 0xad, 0xd1, 0x61, 0xf3, 0xdd, 0x4f, 0xce, 0xa3, 0x78, 0xba, 0xe6, 0x8f, 0x61, 0xf9, 0x0e, + 0x69, 0x74, 0x34, 0xc2, 0x66, 0x7c, 0x90, 0x9c, 0x5b, 0x57, 0xc7, 0xa5, 0xd8, 0xc9, 0xb3, 0x39, + 0xd7, 0x83, 0x77, 0xdc, 0xfb, 0xa9, 0xb2, 0x82, 0x98, 0xe5, 0xa8, 0x59, 0xc5, 0xfd, 0x9d, 0xb0, + 0xd9, 0xba, 0xbb, 0x1a, 0xd9, 0xe5, 0x6a, 0x52, 0x73, 0xb3, 0x75, 0xd9, 0x13, 0x02, 0xd3, 0x82, + 0xd6, 0x91, 0xbd, 0x56, 0x8b, 0x5b, 0x97, 0x9e, 0xb3, 0x8e, 0x8c, 0x6d, 0x9b, 0x61, 0xae, 0x63, + 0x0a, 0x6b, 0xef, 0x74, 0xb6, 0x1b, 0xbb, 0x8d, 0xd2, 0x83, 0xf4, 0xdc, 0x82, 0x6a, 0xce, 0x70, + 0x64, 0x51, 0x5f, 0x98, 0xd7, 0xdd, 0xda, 0xdb, 0xe5, 0xcf, 0xcc, 0xf7, 0x33, 0xf3, 0x13, 0x0e, + 0xc6, 0x7e, 0x9f, 0x95, 0x8a, 0xd7, 0xc2, 0xda, 0xed, 0x41, 0x46, 0xa4, 0xcc, 0x10, 0x23, 0x0a, + 0x37, 0xa6, 0x87, 0x34, 0x7e, 0x28, 0x5e, 0xa4, 0xef, 0x67, 0xe7, 0xf4, 0x10, 0xfa, 0x1e, 0x46, + 0x41, 0x2c, 0xef, 0xc2, 0x67, 0xa8, 0x90, 0xf5, 0xf2, 0x8f, 0x21, 0x17, 0x04, 0x0c, 0xdc, 0x9d, + 0xec, 0x41, 0x9b, 0xdc, 0x6d, 0x10, 0x3c, 0x7a, 0xbe, 0x3b, 0x01, 0xe7, 0xc0, 0x71, 0xd1, 0xbb, + 0x70, 0x19, 0x8b, 0x3b, 0x5a, 0x4b, 0xbb, 0x83, 0x00, 0xf9, 0xce, 0x15, 0x00, 0xa4, 0xd7, 0x97, + 0x4b, 0x72, 0x82, 0x50, 0x67, 0x75, 0xed, 0xe1, 0xa7, 0xeb, 0x0b, 0x1f, 0xe3, 0xef, 0xf3, 0x4f, + 0xd7, 0x13, 0xf7, 0x3f, 0x5b, 0x4f, 0x3c, 0xc4, 0xdf, 0x5f, 0xf0, 0xf7, 0x4f, 0xfc, 0x1d, 0x66, + 0x79, 0xfd, 0xf2, 0xc6, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x79, 0xcb, 0x88, 0x96, 0x53, 0x1d, + 0x00, 0x00, } diff --git a/vendor/src/github.com/docker/swarmkit/api/types.proto b/vendor/src/github.com/docker/swarmkit/api/types.proto index b673b2dbe8..dd33139910 100644 --- a/vendor/src/github.com/docker/swarmkit/api/types.proto +++ b/vendor/src/github.com/docker/swarmkit/api/types.proto @@ -426,10 +426,10 @@ message IssuanceStatus { message AcceptancePolicy { message RoleAdmissionPolicy { - message HashedSecret { - // The actual hashed content + message Secret { + // The actual content (possibly hashed) bytes data = 1; - // The type of hash we are using + // The type of hash we are using, or "plaintext" string alg = 2; } @@ -439,13 +439,12 @@ message AcceptancePolicy { bool autoaccept = 2; // Secret represents a user-provided string that is necessary for new // nodes to join the cluster - HashedSecret secret = 3; + Secret secret = 3; } repeated RoleAdmissionPolicy policies = 1; } - message ExternalCA { enum CAProtocol { CFSSL = 0 [(gogoproto.enumvalue_customname) = "CAProtocolCFSSL"]; diff --git a/vendor/src/github.com/docker/swarmkit/ca/server.go b/vendor/src/github.com/docker/swarmkit/ca/server.go index ee6325375c..229df2e4f4 100644 --- a/vendor/src/github.com/docker/swarmkit/ca/server.go +++ b/vendor/src/github.com/docker/swarmkit/ca/server.go @@ -1,6 +1,8 @@ package ca import ( + "crypto/subtle" + "errors" "fmt" "strings" "sync" @@ -255,7 +257,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod } // checkSecretValidity verifies if a secret string matches the secret hash stored in the -// Acceptance Policy. It currently only supports bcrypted hashes. +// Acceptance Policy. It currently only supports bcrypted hashes and plaintext. func checkSecretValidity(policy *api.AcceptancePolicy_RoleAdmissionPolicy, secret string) error { if policy == nil || secret == "" { return fmt.Errorf("invalid policy or secret") @@ -264,6 +266,11 @@ func checkSecretValidity(policy *api.AcceptancePolicy_RoleAdmissionPolicy, secre switch strings.ToLower(policy.Secret.Alg) { case "bcrypt": return bcrypt.CompareHashAndPassword(policy.Secret.Data, []byte(secret)) + case "plaintext": + if subtle.ConstantTimeCompare(policy.Secret.Data, []byte(secret)) == 1 { + return nil + } + return errors.New("incorrect secret") } return fmt.Errorf("hash algorithm not supported: %s", policy.Secret.Alg) diff --git a/vendor/src/github.com/docker/swarmkit/identity/randomid.go b/vendor/src/github.com/docker/swarmkit/identity/randomid.go index eecc612f37..f59179b164 100644 --- a/vendor/src/github.com/docker/swarmkit/identity/randomid.go +++ b/vendor/src/github.com/docker/swarmkit/identity/randomid.go @@ -16,9 +16,8 @@ var ( // parameters for random identifier generation. We can tweak this when there is // time for further analysis. const ( - randomIDEntropyBytes = 16 - randomNodeIDEntropyBytes = 8 - randomIDBase = 36 + randomIDEntropyBytes = 16 + randomIDBase = 36 // To ensure that all identifiers are fixed length, we make sure they // get padded out to 25 characters, which is the maximum for the base36 @@ -28,8 +27,7 @@ const ( // was calculated from floor(log(2^128-1, 36)) + 1. // // See http://mathworld.wolfram.com/NumberLength.html for more information. - maxRandomIDLength = 25 - maxRandomNodeIDLength = 13 + maxRandomIDLength = 25 ) // NewID generates a new identifier for use where random identifiers with low diff --git a/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go b/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go index c9c32d48a3..8bf181cd07 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go +++ b/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go @@ -8,6 +8,8 @@ import ( "github.com/docker/libnetwork/drivers/overlay/ovmanager" "github.com/docker/libnetwork/drvregistry" "github.com/docker/libnetwork/ipamapi" + builtinIpam "github.com/docker/libnetwork/ipams/builtin" + nullIpam "github.com/docker/libnetwork/ipams/null" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "golang.org/x/net/context" @@ -76,6 +78,15 @@ func New() (*NetworkAllocator, error) { return nil, err } + for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){ + builtinIpam.Init, + nullIpam.Init, + } { + if err := fn(reg, nil, nil); err != nil { + return nil, err + } + } + pa, err := newPortAllocator() if err != nil { return nil, err diff --git a/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go b/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go index b2ec541eee..b730224fb6 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go @@ -2,6 +2,7 @@ package controlapi import ( "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/manager/state/raft/membership" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" "google.golang.org/grpc" @@ -188,8 +189,10 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) var ( node *api.Node + member *membership.Member demote bool ) + err := s.store.Update(func(tx store.Tx) error { node = store.GetNode(tx, request.NodeID) if node == nil { @@ -199,6 +202,8 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) // Demotion sanity checks. if node.Spec.Role == api.NodeRoleManager && request.Spec.Role == api.NodeRoleWorker { demote = true + + // Check for manager entries in Store. managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager)) if err != nil { return grpc.Errorf(codes.Internal, "internal store error: %v", err) @@ -206,6 +211,16 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) if len(managers) == 1 && managers[0].ID == node.ID { return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") } + + // Check for node in memberlist + if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil { + return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist") + } + + // Quorum safeguard + if !s.raft.CanRemoveMember(member.RaftID) { + return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") + } } node.Meta.Version = *request.NodeVersion @@ -220,14 +235,12 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) } if demote && s.raft != nil { - memberlist := s.raft.GetMemberlist() - for raftID, member := range memberlist { - if member.NodeID == request.NodeID { - if err := s.raft.RemoveMember(ctx, raftID); err != nil { - return nil, err - } - break - } + // TODO(abronan): the remove can potentially fail and leave the node with + // an incorrect role (worker rather than manager), we need to reconcile the + // memberlist with the desired state rather than attempting to remove the + // member once. + if err := s.raft.RemoveMember(ctx, member.RaftID); err != nil { + return nil, grpc.Errorf(codes.Internal, "cannot demote manager to worker: %v", err) } } diff --git a/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go b/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go index 253f71b724..20a8468777 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go @@ -7,6 +7,7 @@ import ( "github.com/docker/engine-api/types/reference" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/scheduler" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" @@ -75,6 +76,14 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return nil } +func validatePlacement(placement *api.Placement) error { + if placement == nil { + return nil + } + _, err := scheduler.ParseExprs(placement.Constraints) + return err +} + func validateUpdate(uc *api.UpdateConfig) error { if uc == nil { return nil @@ -101,6 +110,10 @@ func validateTask(taskSpec api.TaskSpec) error { return err } + if err := validatePlacement(taskSpec.Placement); err != nil { + return err + } + if taskSpec.GetRuntime() == nil { return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") } diff --git a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 4f040b50dd..64dd08872f 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -328,14 +328,14 @@ func (d *Dispatcher) isRunning() bool { } // register is used for registration of node with particular dispatcher. -func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, string, error) { +func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) { // prevent register until we're ready to accept it if err := d.isRunningLocked(); err != nil { - return "", "", err + return "", err } if err := d.nodes.CheckRateLimit(nodeID); err != nil { - return "", "", err + return "", err } // create or update node in store @@ -355,7 +355,7 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a }) if err != nil { - return "", "", err + return "", err } expireFunc := func() { @@ -377,7 +377,7 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a // time a node registers, we invalidate the session and issue a new // session, once identity is proven. This will cause misbehaved agents to // be kicked when multiple connections are made. - return rn.Node.ID, rn.SessionID, nil + return rn.SessionID, nil } // UpdateTaskStatus updates status of task. Node should send such updates @@ -650,7 +650,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio } // register the node. - nodeID, sessionID, err := d.register(stream.Context(), nodeID, r.Description) + sessionID, err := d.register(stream.Context(), nodeID, r.Description) if err != nil { return err } diff --git a/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go b/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go index 6dd7e0d66f..267e3d1da5 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go +++ b/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go @@ -70,6 +70,15 @@ func (r *ReplicatedOrchestrator) resolveService(ctx context.Context, task *api.T return service } +type tasksByRunningState []*api.Task + +func (ts tasksByRunningState) Len() int { return len(ts) } +func (ts tasksByRunningState) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } + +func (ts tasksByRunningState) Less(i, j int) bool { + return ts[i].Status.State == api.TaskStateRunning && ts[j].Status.State != api.TaskStateRunning +} + type taskWithIndex struct { task *api.Task @@ -139,6 +148,14 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser // Preferentially remove tasks on the nodes that have the most // copies of this service, to leave a more balanced result. + + // First sort tasks such that tasks which are currently running + // (in terms of observed state) appear before non-running tasks. + // This will cause us to prefer to remove non-running tasks, all + // other things being equal in terms of node balance. + + sort.Sort(tasksByRunningState(runningTasks)) + // Assign each task an index that counts it as the nth copy of // of the service on its node (1, 2, 3, ...), and sort the // tasks by this counter value. diff --git a/vendor/src/github.com/docker/swarmkit/manager/scheduler/constraint.go b/vendor/src/github.com/docker/swarmkit/manager/scheduler/constraint.go index 066f195329..61be3b907a 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/scheduler/constraint.go +++ b/vendor/src/github.com/docker/swarmkit/manager/scheduler/constraint.go @@ -18,14 +18,19 @@ type ConstraintFilter struct { // SetTask returns true when the filter is enable for a given task. func (f *ConstraintFilter) SetTask(t *api.Task) bool { - if t.Spec.Placement != nil && len(t.Spec.Placement.Constraints) > 0 { - constraints, err := ParseExprs(t.Spec.Placement.Constraints) - if err == nil { - f.constraints = constraints - return true - } + if t.Spec.Placement == nil || len(t.Spec.Placement.Constraints) == 0 { + return false } - return false + + constraints, err := ParseExprs(t.Spec.Placement.Constraints) + if err != nil { + // constraints have been validated at controlapi + // if in any case it finds an error here, treat this task + // as constraint filter disabled. + return false + } + f.constraints = constraints + return true } // Check returns true if the task's constraint is supported by the given node. diff --git a/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go b/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go index 89c91b8a7e..ae5deec6aa 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go +++ b/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go @@ -165,19 +165,11 @@ func (c *Cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { // that might block or harm the Cluster on Member recovery func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool { members := c.Members() - - nmembers := 0 nreachable := 0 for _, m := range members { - // Skip the node that is going to be deleted - if m.RaftID == id { - continue - } - // Local node from where the remove is issued if m.RaftID == from { - nmembers++ nreachable++ continue } @@ -186,8 +178,6 @@ func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool { if err == nil && connState == grpc.Ready { nreachable++ } - - nmembers++ } // Special case of 2 managers @@ -195,7 +185,7 @@ func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool { return false } - nquorum := nmembers/2 + 1 + nquorum := (len(members)+1)/2 + 1 if nreachable < nquorum { return false } diff --git a/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go b/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go index a7875d1ec6..3cb2405264 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -94,6 +94,7 @@ type Node struct { wal *wal.WAL snapshotter *snap.Snapshotter wasLeader bool + restored bool isMember uint32 joinAddr string @@ -394,6 +395,18 @@ func (n *Node) Run(ctx context.Context) error { } } + // If we are the only registered member after + // restoring from the state, campaign to be the + // leader. + if !n.restored { + if len(n.cluster.Members()) <= 1 { + if err := n.Campaign(n.Ctx); err != nil { + panic("raft: cannot campaign to be the leader on node restore") + } + } + n.restored = true + } + // Advance the state machine n.Advance() @@ -638,7 +651,14 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp return &api.LeaveResponse{}, nil } -// RemoveMember submits a configuration change to remove a member from the raft cluster. +// CanRemoveMember checks if a member can be removed from +// the context of the current node. +func (n *Node) CanRemoveMember(id uint64) bool { + return n.cluster.CanRemoveMember(n.Config.ID, id) +} + +// RemoveMember submits a configuration change to remove a member from the raft cluster +// after checking if the operation would not result in a loss of quorum. func (n *Node) RemoveMember(ctx context.Context, id uint64) error { n.membershipLock.Lock() defer n.membershipLock.Unlock() @@ -828,6 +848,18 @@ func (n *Node) GetMemberlist() map[uint64]*api.RaftMember { return memberlist } +// GetMemberByNodeID returns member information based +// on its generic Node ID. +func (n *Node) GetMemberByNodeID(nodeID string) *membership.Member { + members := n.cluster.Members() + for _, member := range members { + if member.NodeID == nodeID { + return member + } + } + return nil +} + // IsMember checks if the raft node has effectively joined // a cluster of existing members. func (n *Node) IsMember() bool { @@ -1172,11 +1204,6 @@ func (n *Node) applyRemoveNode(cc raftpb.ConfChange) (err error) { } } - // Do not unregister yourself - if n.Config.ID == cc.NodeID { - return nil - } - return n.cluster.RemoveMember(cc.NodeID) } diff --git a/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go b/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go index c0d4b454aa..95074ffc19 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -65,7 +65,6 @@ func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error { return err } - n.Node = raft.RestartNode(n.Config) return nil } From 6dfba780cc490ef7579e95169b5c71617f8e5a47 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Thu, 7 Jul 2016 20:17:39 +0100 Subject: [PATCH 4/4] Fix test case for docker_api_swarm_test.go Signed-off-by: Justin Cormack --- integration-cli/docker_api_swarm_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration-cli/docker_api_swarm_test.go b/integration-cli/docker_api_swarm_test.go index 7b8d0431ff..dfcd1727d3 100644 --- a/integration-cli/docker_api_swarm_test.go +++ b/integration-cli/docker_api_swarm_test.go @@ -277,8 +277,6 @@ func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) { } if i > 100 { c.Errorf("node did not turn into manager") - } else { - break } time.Sleep(100 * time.Millisecond) } @@ -296,8 +294,6 @@ func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) { } if i > 100 { c.Errorf("node did not turn into worker") - } else { - break } time.Sleep(100 * time.Millisecond) }