From 1847bb899a07d3dd324e75a3ed9b3489fcfc302f Mon Sep 17 00:00:00 2001 From: Ying Li Date: Tue, 2 May 2017 14:58:57 -0700 Subject: [PATCH 1/3] Propagate the desired CA certificate and CAConfig ForceRotate parameter in the Docker REST APIs when viewing or updating the swarm spec info, and also propagate the desired CA key in the Docker REST APIs when updating swarm spec info only (it is not available for viewing). Signed-off-by: Ying Li --- api/swagger.yaml | 8 ++++++++ api/types/swarm/swarm.go | 10 ++++++++++ daemon/cluster/convert/swarm.go | 13 +++++++++++++ docs/api/version-history.md | 5 ++++- 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 18971c4b07..9e33a6c6fd 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1886,6 +1886,14 @@ definitions: CACert: description: "The root CA certificate (in PEM format) this external CA uses to issue TLS certificates (assumed to be to the current swarm root CA certificate if not provided)." type: "string" + SigningCACert: + description: "The desired signing CA certificate for all swarm node TLS leaf certificates, in PEM format." + type: "string" + SigningCAKey: + description: "The desired signing CA key for all swarm node TLS leaf certificates, in PEM format." + type: "string" + ForceRotate: + description: "An integer whose purpose is to force swarm to generate a new signing CA certificate and key, if none have been specified in `SigningCACert` and `SigningCAKey`" EncryptionConfig: description: "Parameters related to encryption-at-rest." type: "object" diff --git a/api/types/swarm/swarm.go b/api/types/swarm/swarm.go index bdb3042337..5b74f14b11 100644 --- a/api/types/swarm/swarm.go +++ b/api/types/swarm/swarm.go @@ -109,6 +109,16 @@ type CAConfig struct { // ExternalCAs is a list of CAs to which a manager node will make // certificate signing requests for node certificates. ExternalCAs []*ExternalCA `json:",omitempty"` + + // SigningCACert and SigningCAKey specify the desired signing root CA and + // root CA key for the swarm. When inspecting the cluster, the key will + // be redacted. + SigningCACert string `json:",omitempty"` + SigningCAKey string `json:",omitempty"` + + // If this value changes, and there is no specified signing cert and key, + // then the swarm is forced to generate a new root certificate ane key. + ForceRotate uint64 `json:",omitempty"` } // ExternalCAProtocol represents type of external CA. diff --git a/daemon/cluster/convert/swarm.go b/daemon/cluster/convert/swarm.go index 64fc7f72d9..c6e1f3652b 100644 --- a/daemon/cluster/convert/swarm.go +++ b/daemon/cluster/convert/swarm.go @@ -30,6 +30,11 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm { EncryptionConfig: types.EncryptionConfig{ AutoLockManagers: c.Spec.EncryptionConfig.AutoLockManagers, }, + CAConfig: types.CAConfig{ + // do not include the signing CA key (it should already be redacted via the swarm APIs) + SigningCACert: string(c.Spec.CAConfig.SigningCACert), + ForceRotate: c.Spec.CAConfig.ForceRotate, + }, }, TLSInfo: types.TLSInfo{ TrustRoot: string(c.RootCA.CACert), @@ -114,6 +119,14 @@ func MergeSwarmSpecToGRPC(s types.Spec, spec swarmapi.ClusterSpec) (swarmapi.Clu if s.CAConfig.NodeCertExpiry != 0 { spec.CAConfig.NodeCertExpiry = gogotypes.DurationProto(s.CAConfig.NodeCertExpiry) } + if s.CAConfig.SigningCACert != "" { + spec.CAConfig.SigningCACert = []byte(s.CAConfig.SigningCACert) + } + if s.CAConfig.SigningCAKey != "" { + // do prpagate the signing CA key here because we want to provide it TO the swarm APIs + spec.CAConfig.SigningCAKey = []byte(s.CAConfig.SigningCAKey) + } + spec.CAConfig.ForceRotate = s.CAConfig.ForceRotate for _, ca := range s.CAConfig.ExternalCAs { protocol, ok := swarmapi.ExternalCA_CAProtocol_value[strings.ToUpper(string(ca.Protocol))] diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 359813d87e..4e02e7193a 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -19,11 +19,14 @@ keywords: "API, Docker, rcli, REST, documentation" * `GET /info` now returns the list of supported logging drivers, including plugins. * `GET /info` and `GET /swarm` now returns the cluster-wide swarm CA info if the node is in a swarm: the cluster root CA certificate, and the cluster TLS - leaf certificate issuer's subject and public key. + leaf certificate issuer's subject and public key. It also displays the desired CA signing certificate, if any was provided as part of the spec. * `POST /build/` now (when not silent) produces an `Aux` message in the JSON output stream with payload `types.BuildResult` for each image produced. The final such message will reference the image resulting from the build. * `GET /nodes` and `GET /nodes/{id}` now returns additional information about swarm TLS info if the node is part of a swarm: the trusted root CA, and the issuer's subject and public key. * `GET /distribution/(name)/json` is a new endpoint that returns a JSON output stream with payload `types.DistributionInspect` for an image name. It includes a descriptor with the digest, and supported platforms retrieved from directly contacting the registry. +* `POST /swarm/update` now accepts 3 additional parameters as part of the swarm spec's CA configuration; the desired CA certificate for + the swarm, the desired CA key for the swarm (if not using an external certificate), and an optional parameter to force swarm to + generate and rotate to a new CA certificate/key pair. ## v1.29 API changes From a771c16834c92cb39142078c64e253423f0fb4e3 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Tue, 2 May 2017 21:22:56 -0700 Subject: [PATCH 2/3] Update the stream formatter to display custom unit numbers. Signed-off-by: Ying Li --- daemon/cluster/convert/swarm.go | 2 +- pkg/jsonmessage/jsonmessage.go | 28 +++++++++++++--- pkg/jsonmessage/jsonmessage_test.go | 44 +++++++++++++++++++++----- pkg/progress/progress.go | 2 ++ pkg/streamformatter/streamformatter.go | 2 +- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/daemon/cluster/convert/swarm.go b/daemon/cluster/convert/swarm.go index c6e1f3652b..0d5c8738c9 100644 --- a/daemon/cluster/convert/swarm.go +++ b/daemon/cluster/convert/swarm.go @@ -123,7 +123,7 @@ func MergeSwarmSpecToGRPC(s types.Spec, spec swarmapi.ClusterSpec) (swarmapi.Clu spec.CAConfig.SigningCACert = []byte(s.CAConfig.SigningCACert) } if s.CAConfig.SigningCAKey != "" { - // do prpagate the signing CA key here because we want to provide it TO the swarm APIs + // do propagate the signing CA key here because we want to provide it TO the swarm APIs spec.CAConfig.SigningCAKey = []byte(s.CAConfig.SigningCAKey) } spec.CAConfig.ForceRotate = s.CAConfig.ForceRotate diff --git a/pkg/jsonmessage/jsonmessage.go b/pkg/jsonmessage/jsonmessage.go index 2b8e98c429..dc785d6187 100644 --- a/pkg/jsonmessage/jsonmessage.go +++ b/pkg/jsonmessage/jsonmessage.go @@ -36,7 +36,8 @@ type JSONProgress struct { Total int64 `json:"total,omitempty"` Start int64 `json:"start,omitempty"` // If true, don't show xB/yB - HideCounts bool `json:"hidecounts,omitempty"` + HideCounts bool `json:"hidecounts,omitempty"` + Units string `json:"units,omitempty"` } func (p *JSONProgress) String() string { @@ -55,11 +56,16 @@ func (p *JSONProgress) String() string { if p.Current <= 0 && p.Total <= 0 { return "" } - current := units.HumanSize(float64(p.Current)) if p.Total <= 0 { - return fmt.Sprintf("%8v", current) + switch p.Units { + case "": + current := units.HumanSize(float64(p.Current)) + return fmt.Sprintf("%8v", current) + default: + return fmt.Sprintf("%d %s", p.Current, p.Units) + } } - total := units.HumanSize(float64(p.Total)) + percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 if percentage > 50 { percentage = 50 @@ -73,13 +79,25 @@ func (p *JSONProgress) String() string { pbBox = fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces)) } - if !p.HideCounts { + switch { + case p.HideCounts: + case p.Units == "": // no units, use bytes + current := units.HumanSize(float64(p.Current)) + total := units.HumanSize(float64(p.Total)) + numbersBox = fmt.Sprintf("%8v/%v", current, total) if p.Current > p.Total { // remove total display if the reported current is wonky. numbersBox = fmt.Sprintf("%8v", current) } + default: + numbersBox = fmt.Sprintf("%d/%d %s", p.Current, p.Total, p.Units) + + if p.Current > p.Total { + // remove total display if the reported current is wonky. + numbersBox = fmt.Sprintf("%d %s", p.Current, p.Units) + } } if p.Current > 0 && p.Start > 0 && percentage < 50 { diff --git a/pkg/jsonmessage/jsonmessage_test.go b/pkg/jsonmessage/jsonmessage_test.go index ce3b6de8c0..c3ed6c046a 100644 --- a/pkg/jsonmessage/jsonmessage_test.go +++ b/pkg/jsonmessage/jsonmessage_test.go @@ -65,22 +65,50 @@ func TestProgress(t *testing.T) { if jp5.String() != expected { t.Fatalf("Expected %q, got %q", expected, jp5.String()) } + + expected = "[=========================> ] 50/100 units" + if termsz != nil && termsz.Width <= 110 { + expected = " 50/100 units" + } + jp6 := JSONProgress{Current: 50, Total: 100, Units: "units"} + if jp6.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp6.String()) + } + + // this number can't be negative + expected = "[==================================================>] 50 units" + if termsz != nil && termsz.Width <= 110 { + expected = " 50 units" + } + jp7 := JSONProgress{Current: 50, Total: 40, Units: "units"} + if jp7.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp7.String()) + } + + expected = "[=========================> ] " + if termsz != nil && termsz.Width <= 110 { + expected = "" + } + jp8 := JSONProgress{Current: 50, Total: 100, HideCounts: true} + if jp8.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp8.String()) + } } func TestJSONMessageDisplay(t *testing.T) { now := time.Now() messages := map[JSONMessage][]string{ // Empty - JSONMessage{}: {"\n", "\n"}, + {}: {"\n", "\n"}, // Status - JSONMessage{ + { Status: "status", }: { "status\n", "status\n", }, // General - JSONMessage{ + { Time: now.Unix(), ID: "ID", From: "From", @@ -90,7 +118,7 @@ func TestJSONMessageDisplay(t *testing.T) { fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)), }, // General, with nano precision time - JSONMessage{ + { TimeNano: now.UnixNano(), ID: "ID", From: "From", @@ -100,7 +128,7 @@ func TestJSONMessageDisplay(t *testing.T) { fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), }, // General, with both times Nano is preferred - JSONMessage{ + { Time: now.Unix(), TimeNano: now.UnixNano(), ID: "ID", @@ -111,7 +139,7 @@ func TestJSONMessageDisplay(t *testing.T) { fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), }, // Stream over status - JSONMessage{ + { Status: "status", Stream: "stream", }: { @@ -119,7 +147,7 @@ func TestJSONMessageDisplay(t *testing.T) { "stream", }, // With progress message - JSONMessage{ + { Status: "status", ProgressMessage: "progressMessage", }: { @@ -127,7 +155,7 @@ func TestJSONMessageDisplay(t *testing.T) { "status progressMessage", }, // With progress, stream empty - JSONMessage{ + { Status: "status", Stream: "", Progress: &JSONProgress{Current: 1}, diff --git a/pkg/progress/progress.go b/pkg/progress/progress.go index e78fc120b6..7c3d3a5145 100644 --- a/pkg/progress/progress.go +++ b/pkg/progress/progress.go @@ -18,6 +18,8 @@ type Progress struct { // If true, don't show xB/yB HideCounts bool + // If not empty, use units instead of bytes for counts + Units string // Aux contains extra information not presented to the user, such as // digests for push signing. diff --git a/pkg/streamformatter/streamformatter.go b/pkg/streamformatter/streamformatter.go index 48ba65503c..c4f55755ec 100644 --- a/pkg/streamformatter/streamformatter.go +++ b/pkg/streamformatter/streamformatter.go @@ -117,7 +117,7 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error { if prog.Message != "" { formatted = out.sf.formatStatus(prog.ID, prog.Message) } else { - jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total, HideCounts: prog.HideCounts} + jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total, HideCounts: prog.HideCounts, Units: prog.Units} formatted = out.sf.formatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux) } _, err := out.out.Write(formatted) From 376c75d13cedd22c578197a140ffc10e27e84d01 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Mon, 8 May 2017 17:14:34 -0700 Subject: [PATCH 3/3] Add API test to rotate the swarm CA certificate Signed-off-by: Ying Li --- integration-cli/docker_api_swarm_test.go | 72 ++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/integration-cli/docker_api_swarm_test.go b/integration-cli/docker_api_swarm_test.go index 028785a01c..9f3f3e53ca 100644 --- a/integration-cli/docker_api_swarm_test.go +++ b/integration-cli/docker_api_swarm_test.go @@ -14,12 +14,15 @@ import ( "sync" "time" + "github.com/cloudflare/cfssl/csr" "github.com/cloudflare/cfssl/helpers" + "github.com/cloudflare/cfssl/initca" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/swarmkit/ca" "github.com/go-check/check" ) @@ -930,3 +933,72 @@ func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) } + +func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { + m := s.AddDaemon(c, true, true) + w := s.AddDaemon(c, true, false) + + info, err := m.SwarmInfo() + c.Assert(err, checker.IsNil) + + currentTrustRoot := info.Cluster.TLSInfo.TrustRoot + + // rotate multiple times + for i := 0; i < 4; i++ { + var cert, key []byte + if i%2 != 0 { + cert, _, key, err = initca.New(&csr.CertificateRequest{ + CN: "newRoot", + KeyRequest: csr.NewBasicKeyRequest(), + CA: &csr.CAConfig{Expiry: ca.RootCAExpiration}, + }) + c.Assert(err, checker.IsNil) + } + expectedCert := string(cert) + m.UpdateSwarm(c, func(s *swarm.Spec) { + s.CAConfig.SigningCACert = expectedCert + s.CAConfig.SigningCAKey = string(key) + s.CAConfig.ForceRotate++ + }) + + // poll to make sure update succeeds + var clusterTLSInfo swarm.TLSInfo + for j := 0; j < 18; j++ { + info, err := m.SwarmInfo() + c.Assert(err, checker.IsNil) + c.Assert(info.Cluster.Spec.CAConfig.SigningCACert, checker.Equals, expectedCert) + // the desired CA key is always redacted + c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "") + + clusterTLSInfo = info.Cluster.TLSInfo + + if !info.Cluster.RootRotationInProgress { + break + } + + // root rotation not done + time.Sleep(250 * time.Millisecond) + } + c.Assert(clusterTLSInfo.TrustRoot, checker.Not(checker.Equals), currentTrustRoot) + if cert != nil { + c.Assert(clusterTLSInfo.TrustRoot, checker.Equals, expectedCert) + } + // could take another second or two for the nodes to trust the new roots after the've all gotten + // new TLS certificates + for j := 0; j < 18; j++ { + mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo + wInfo := m.GetNode(c, w.NodeID).Description.TLSInfo + + if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot { + break + } + + // nodes don't trust root certs yet + time.Sleep(250 * time.Millisecond) + } + + c.Assert(m.GetNode(c, m.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + c.Assert(m.GetNode(c, w.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + currentTrustRoot = clusterTLSInfo.TrustRoot + } +}