From 7ce9f0275130b66241f9077485d14e25c641cc3d Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Thu, 15 Oct 2015 15:43:24 -0700 Subject: [PATCH] updating notary and gotuf with latest bugfixes Signed-off-by: David Lawrence --- hack/vendor.sh | 4 +- .../docker/notary/client/changelist/change.go | 22 ++++++ .../github.com/docker/notary/client/client.go | 73 +++++++++++++++++-- .../docker/notary/client/helpers.go | 46 ++++++++++-- .../endophage/gotuf/client/client.go | 61 +++++++--------- .../endophage/gotuf/client/errors.go | 8 ++ .../github.com/endophage/gotuf/data/keys.go | 2 +- .../github.com/endophage/gotuf/data/roles.go | 3 +- .../endophage/gotuf/store/httpstore.go | 1 + vendor/src/github.com/endophage/gotuf/tuf.go | 36 +++++++-- 10 files changed, 199 insertions(+), 57 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 13a42f5076..052303394d 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -41,8 +41,8 @@ clone git github.com/boltdb/bolt v1.0 clone git github.com/docker/distribution 20c4b7a1805a52753dfd593ee1cc35558722a0ce # docker/1.9 branch clone git github.com/vbatts/tar-split v0.9.10 -clone git github.com/docker/notary ac05822d7d71ef077df3fc24f506672282a1feea -clone git github.com/endophage/gotuf 9bcdad0308e34a49f38448b8ad436ad8860825ce +clone git github.com/docker/notary 089d8450d8928aa1c58fd03f09cabbde9bcb4590 +clone git github.com/endophage/gotuf 876c31a61bc4aa0dae09bb8ef3946dc26dd04924 clone git github.com/jfrazelle/go 6e461eb70cb4187b41a84e9a567d7137bdbe0f16 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c diff --git a/vendor/src/github.com/docker/notary/client/changelist/change.go b/vendor/src/github.com/docker/notary/client/changelist/change.go index 867c230517..dfdaed5c36 100644 --- a/vendor/src/github.com/docker/notary/client/changelist/change.go +++ b/vendor/src/github.com/docker/notary/client/changelist/change.go @@ -1,5 +1,9 @@ package changelist +import ( + "github.com/endophage/gotuf/data" +) + // Scopes for TufChanges are simply the TUF roles. // Unfortunately because of targets delegations, we can only // cover the base roles. @@ -10,6 +14,17 @@ const ( ScopeTimestamp = "timestamp" ) +// Types for TufChanges are namespaced by the Role they +// are relevant for. The Root and Targets roles are the +// only ones for which user action can cause a change, as +// all changes in Snapshot and Timestamp are programatically +// generated base on Root and Targets changes. +const ( + TypeRootRole = "role" + TypeTargetsTarget = "target" + TypeTargetsDelegation = "delegation" +) + // TufChange represents a change to a TUF repo type TufChange struct { // Abbreviated because Go doesn't permit a field and method of the same name @@ -20,6 +35,13 @@ type TufChange struct { Data []byte `json:"data"` } +// TufRootData represents a modification of the keys associated +// with a role that appears in the root.json +type TufRootData struct { + Keys []data.TUFKey `json:"keys"` + RoleName string `json:"role"` +} + // NewTufChange initializes a tufChange object func NewTufChange(action string, role, changeType, changePath string, content []byte) *TufChange { return &TufChange{ diff --git a/vendor/src/github.com/docker/notary/client/client.go b/vendor/src/github.com/docker/notary/client/client.go index ee376053c3..ca57a89f1a 100644 --- a/vendor/src/github.com/docker/notary/client/client.go +++ b/vendor/src/github.com/docker/notary/client/client.go @@ -245,6 +245,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error { if err != nil { return err } + defer cl.Close() logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length) meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes} @@ -258,7 +259,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error { if err != nil { return err } - return cl.Close() + return nil } // RemoveTarget creates a new changelist entry to remove a target from the repository @@ -326,6 +327,17 @@ func (r *NotaryRepository) GetTargetByName(name string) (*Target, error) { return &Target{Name: name, Hashes: meta.Hashes, Length: meta.Length}, nil } +// GetChangelist returns the list of the repository's unpublished changes +func (r *NotaryRepository) GetChangelist() (changelist.Changelist, error) { + changelistDir := filepath.Join(r.tufRepoPath, "changelist") + cl, err := changelist.NewFileChangelist(changelistDir) + if err != nil { + logrus.Debug("Error initializing changelist") + return nil, err + } + return cl, nil +} + // Publish pushes the local changes in signed material to the remote notary-server // Conceptually it performs an operation similar to a `git rebase` func (r *NotaryRepository) Publish() error { @@ -371,11 +383,8 @@ func (r *NotaryRepository) Publish() error { return err } } - // load the changelist for this repo - changelistDir := filepath.Join(r.tufRepoPath, "changelist") - cl, err := changelist.NewFileChangelist(changelistDir) + cl, err := r.GetChangelist() if err != nil { - logrus.Debug("Error initializing changelist") return err } // apply the changelist to the repo @@ -445,7 +454,7 @@ func (r *NotaryRepository) Publish() error { // This is not a critical problem when only a single host is pushing // but will cause weird behaviour if changelist cleanup is failing // and there are multiple hosts writing to the repo. - logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", changelistDir) + logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", filepath.Join(r.tufRepoPath, "changelist")) } return nil } @@ -596,3 +605,55 @@ func (r *NotaryRepository) bootstrapClient() (*tufclient.Client, error) { r.fileStore, ), nil } + +// RotateKeys removes all existing keys associated with role and adds +// the keys specified by keyIDs to the role. These changes are staged +// in a changelist until publish is called. +func (r *NotaryRepository) RotateKeys() error { + for _, role := range []string{"targets", "snapshot"} { + key, err := r.cryptoService.Create(role, data.ECDSAKey) + if err != nil { + return err + } + err = r.rootFileKeyChange(role, changelist.ActionCreate, key) + if err != nil { + return err + } + } + return nil +} + +func (r *NotaryRepository) rootFileKeyChange(role, action string, key data.PublicKey) error { + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + k, ok := key.(*data.TUFKey) + if !ok { + return errors.New("Invalid key type found during rotation.") + } + + meta := changelist.TufRootData{ + RoleName: role, + Keys: []data.TUFKey{*k}, + } + metaJSON, err := json.Marshal(meta) + if err != nil { + return err + } + + c := changelist.NewTufChange( + action, + changelist.ScopeRoot, + changelist.TypeRootRole, + role, + metaJSON, + ) + err = cl.Add(c) + if err != nil { + return err + } + return nil +} diff --git a/vendor/src/github.com/docker/notary/client/helpers.go b/vendor/src/github.com/docker/notary/client/helpers.go index 476ef08b71..50be86c6ce 100644 --- a/vendor/src/github.com/docker/notary/client/helpers.go +++ b/vendor/src/github.com/docker/notary/client/helpers.go @@ -7,7 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/notary/client/changelist" - "github.com/endophage/gotuf" + tuf "github.com/endophage/gotuf" "github.com/endophage/gotuf/data" "github.com/endophage/gotuf/keys" "github.com/endophage/gotuf/store" @@ -38,14 +38,16 @@ func applyChangelist(repo *tuf.TufRepo, cl changelist.Changelist) error { } switch c.Scope() { case changelist.ScopeTargets: - err := applyTargetsChange(repo, c) - if err != nil { - return err - } + err = applyTargetsChange(repo, c) + case changelist.ScopeRoot: + err = applyRootChange(repo, c) default: logrus.Debug("scope not supported: ", c.Scope()) } index++ + if err != nil { + return err + } } logrus.Debugf("applied %d change(s)", index) return nil @@ -75,6 +77,40 @@ func applyTargetsChange(repo *tuf.TufRepo, c changelist.Change) error { return nil } +func applyRootChange(repo *tuf.TufRepo, c changelist.Change) error { + var err error + switch c.Type() { + case changelist.TypeRootRole: + err = applyRootRoleChange(repo, c) + default: + logrus.Debug("type of root change not yet supported: ", c.Type()) + } + return err // might be nil +} + +func applyRootRoleChange(repo *tuf.TufRepo, c changelist.Change) error { + switch c.Action() { + case changelist.ActionCreate: + // replaces all keys for a role + d := &changelist.TufRootData{} + err := json.Unmarshal(c.Content(), d) + if err != nil { + return err + } + k := []data.PublicKey{} + for _, key := range d.Keys { + k = append(k, data.NewPublicKey(key.Algorithm(), key.Public())) + } + err = repo.ReplaceBaseKeys(d.RoleName, k...) + if err != nil { + return err + } + default: + logrus.Debug("action not yet supported for root: ", c.Action()) + } + return nil +} + func nearExpiry(r *data.SignedRoot) bool { plus6mo := time.Now().AddDate(0, 6, 0) return r.Signed.Expires.Before(plus6mo) diff --git a/vendor/src/github.com/endophage/gotuf/client/client.go b/vendor/src/github.com/endophage/gotuf/client/client.go index 2bcda4b741..532e474789 100644 --- a/vendor/src/github.com/endophage/gotuf/client/client.go +++ b/vendor/src/github.com/endophage/gotuf/client/client.go @@ -175,19 +175,7 @@ func (c *Client) downloadRoot() error { var s *data.Signed var raw []byte if download { - logrus.Debug("downloading new root") - raw, err = c.remote.GetMeta(role, size) - if err != nil { - return err - } - hash := sha256.Sum256(raw) - if expectedSha256 != nil && !bytes.Equal(hash[:], expectedSha256) { - // if we don't have an expected sha256, we're going to trust the root - // based purely on signature and expiry time validation - return fmt.Errorf("Remote root sha256 did not match snapshot root sha256: %#x vs. %#x", hash, []byte(expectedSha256)) - } - s = &data.Signed{} - err = json.Unmarshal(raw, s) + raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return err } @@ -247,6 +235,8 @@ func (c Client) verifyRoot(role string, s *data.Signed, minVersion int) error { } // downloadTimestamp is responsible for downloading the timestamp.json +// Timestamps are special in that we ALWAYS attempt to download and only +// use cache if the download fails (and the cache is still valid). func (c *Client) downloadTimestamp() error { logrus.Debug("downloadTimestamp") role := data.RoleName("timestamp") @@ -271,7 +261,6 @@ func (c *Client) downloadTimestamp() error { } // unlike root, targets and snapshot, always try and download timestamps // from remote, only using the cache one if we couldn't reach remote. - logrus.Debug("Downloading timestamp") raw, err := c.remote.GetMeta(role, maxSize) var s *data.Signed if err != nil || len(raw) == 0 { @@ -286,6 +275,7 @@ func (c *Client) downloadTimestamp() error { } return err } + logrus.Debug("using cached timestamp") s = old } else { download = true @@ -351,17 +341,7 @@ func (c *Client) downloadSnapshot() error { } var s *data.Signed if download { - logrus.Debug("downloading new snapshot") - raw, err = c.remote.GetMeta(role, size) - if err != nil { - return err - } - genHash := sha256.Sum256(raw) - if !bytes.Equal(genHash[:], expectedSha256) { - return fmt.Errorf("Retrieved snapshot did not verify against hash in timestamp.") - } - s = &data.Signed{} - err = json.Unmarshal(raw, s) + raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return err } @@ -390,8 +370,7 @@ func (c *Client) downloadSnapshot() error { } // downloadTargets is responsible for downloading any targets file -// including delegates roles. It will download the whole tree of -// delegated roles below the given one +// including delegates roles. func (c *Client) downloadTargets(role string) error { role = data.RoleName(role) // this will really only do something for base targets role snap := c.local.Snapshot.Signed @@ -418,6 +397,24 @@ func (c *Client) downloadTargets(role string) error { return nil } +func (c *Client) downloadSigned(role string, size int64, expectedSha256 []byte) ([]byte, *data.Signed, error) { + logrus.Debugf("downloading new %s", role) + raw, err := c.remote.GetMeta(role, size) + if err != nil { + return nil, nil, err + } + genHash := sha256.Sum256(raw) + if !bytes.Equal(genHash[:], expectedSha256) { + return nil, nil, ErrChecksumMismatch{role: role} + } + s := &data.Signed{} + err = json.Unmarshal(raw, s) + if err != nil { + return nil, nil, err + } + return raw, s, nil +} + func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.Files, consistent bool, threshold int) (*data.Signed, error) { // require role exists in snapshots roleMeta, ok := snapshotMeta[role] @@ -454,25 +451,19 @@ func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.F } else { download = true } - } + size := snapshotMeta[role].Length var s *data.Signed if download { rolePath, err := c.RoleTargetsPath(role, hex.EncodeToString(expectedSha256), consistent) if err != nil { return nil, err } - raw, err = c.remote.GetMeta(rolePath, snapshotMeta[role].Length) + raw, s, err = c.downloadSigned(rolePath, size, expectedSha256) if err != nil { return nil, err } - s = &data.Signed{} - err = json.Unmarshal(raw, s) - if err != nil { - logrus.Error("Error unmarshalling targets file:", err) - return nil, err - } } else { logrus.Debug("using cached ", role) s = old diff --git a/vendor/src/github.com/endophage/gotuf/client/errors.go b/vendor/src/github.com/endophage/gotuf/client/errors.go index 776e6a69ea..311e74a8de 100644 --- a/vendor/src/github.com/endophage/gotuf/client/errors.go +++ b/vendor/src/github.com/endophage/gotuf/client/errors.go @@ -10,6 +10,14 @@ var ( ErrInsufficientKeys = errors.New("tuf: insufficient keys to meet threshold") ) +type ErrChecksumMismatch struct { + role string +} + +func (e ErrChecksumMismatch) Error() string { + return fmt.Sprintf("tuf: checksum for %s did not match", e.role) +} + type ErrMissingRemoteMetadata struct { Name string } diff --git a/vendor/src/github.com/endophage/gotuf/data/keys.go b/vendor/src/github.com/endophage/gotuf/data/keys.go index 3df1ce05c7..eccccc420d 100644 --- a/vendor/src/github.com/endophage/gotuf/data/keys.go +++ b/vendor/src/github.com/endophage/gotuf/data/keys.go @@ -71,7 +71,7 @@ func (k TUFKey) Public() []byte { return k.Value.Public } -func (k *TUFKey) Private() []byte { +func (k TUFKey) Private() []byte { return k.Value.Private } diff --git a/vendor/src/github.com/endophage/gotuf/data/roles.go b/vendor/src/github.com/endophage/gotuf/data/roles.go index d3047d7844..1034393e1d 100644 --- a/vendor/src/github.com/endophage/gotuf/data/roles.go +++ b/vendor/src/github.com/endophage/gotuf/data/roles.go @@ -24,7 +24,7 @@ var ValidRoles = map[string]string{ func SetValidRoles(rs map[string]string) { // iterate ValidRoles - for k, _ := range ValidRoles { + for k := range ValidRoles { if v, ok := rs[k]; ok { ValidRoles[k] = v } @@ -88,6 +88,7 @@ type Role struct { Name string `json:"name"` Paths []string `json:"paths,omitempty"` PathHashPrefixes []string `json:"path_hash_prefixes,omitempty"` + Email string `json:"email,omitempty"` } func NewRole(name string, threshold int, keyIDs, paths, pathHashPrefixes []string) (*Role, error) { diff --git a/vendor/src/github.com/endophage/gotuf/store/httpstore.go b/vendor/src/github.com/endophage/gotuf/store/httpstore.go index 1a82b094cd..7304382675 100644 --- a/vendor/src/github.com/endophage/gotuf/store/httpstore.go +++ b/vendor/src/github.com/endophage/gotuf/store/httpstore.go @@ -90,6 +90,7 @@ func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) { if resp.StatusCode == http.StatusNotFound { return nil, ErrMetaNotFound{} } else if resp.StatusCode != http.StatusOK { + logrus.Debugf("received HTTP status %d when requesting %s.", resp.StatusCode, name) return nil, ErrServerUnavailable{code: resp.StatusCode} } if resp.ContentLength > size { diff --git a/vendor/src/github.com/endophage/gotuf/tuf.go b/vendor/src/github.com/endophage/gotuf/tuf.go index 4d226acebb..39af54018b 100644 --- a/vendor/src/github.com/endophage/gotuf/tuf.go +++ b/vendor/src/github.com/endophage/gotuf/tuf.go @@ -71,24 +71,46 @@ func NewTufRepo(keysDB *keys.KeyDB, cryptoService signed.CryptoService) *TufRepo } // AddBaseKeys is used to add keys to the role in root.json -func (tr *TufRepo) AddBaseKeys(role string, keys ...*data.TUFKey) error { +func (tr *TufRepo) AddBaseKeys(role string, keys ...data.PublicKey) error { if tr.Root == nil { return ErrNotLoaded{role: "root"} } + ids := []string{} for _, k := range keys { // Store only the public portion - pubKey := *k - pubKey.Value.Private = nil - tr.Root.Signed.Keys[pubKey.ID()] = &pubKey - tr.keysDB.AddKey(&pubKey) + pubKey := data.NewPrivateKey(k.Algorithm(), k.Public(), nil) + tr.Root.Signed.Keys[pubKey.ID()] = pubKey + tr.keysDB.AddKey(k) tr.Root.Signed.Roles[role].KeyIDs = append(tr.Root.Signed.Roles[role].KeyIDs, pubKey.ID()) + ids = append(ids, pubKey.ID()) } + r, err := data.NewRole( + role, + tr.Root.Signed.Roles[role].Threshold, + ids, + nil, + nil, + ) + if err != nil { + return err + } + tr.keysDB.AddRole(r) tr.Root.Dirty = true return nil } -// RemoveKeys is used to remove keys from the roles in root.json +// ReplaceBaseKeys is used to replace all keys for the given role with the new keys +func (tr *TufRepo) ReplaceBaseKeys(role string, keys ...data.PublicKey) error { + r := tr.keysDB.GetRole(role) + err := tr.RemoveBaseKeys(role, r.KeyIDs...) + if err != nil { + return err + } + return tr.AddBaseKeys(role, keys...) +} + +// RemoveBaseKeys is used to remove keys from the roles in root.json func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error { if tr.Root == nil { return ErrNotLoaded{role: "root"} @@ -119,7 +141,7 @@ func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error { } // remove keys no longer in use by any roles - for k, _ := range toDelete { + for k := range toDelete { delete(tr.Root.Signed.Keys, k) } tr.Root.Dirty = true