diff --git a/hack/vendor.sh b/hack/vendor.sh index 51ff8ca32f..349a9b32b9 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -37,9 +37,9 @@ clone git github.com/hashicorp/consul v0.5.2 clone git github.com/docker/distribution ec87e9b6971d831f0eff752ddb54fb64693e51cd # docker/1.8 branch clone git github.com/vbatts/tar-split v0.9.6 -clone git github.com/docker/notary 8e8122eb5528f621afcd4e2854c47302f17392f7 -clone git github.com/endophage/gotuf a592b03b28b02bb29bb5878308fb1abed63383b5 -clone git github.com/tent/canonical-json-go 96e4ba3a7613a1216cbd1badca4efe382adea337 +clone git github.com/docker/notary ac05822d7d71ef077df3fc24f506672282a1feea +clone git github.com/endophage/gotuf 9bcdad0308e34a49f38448b8ad436ad8860825ce +clone git github.com/jfrazelle/go 6e461eb70cb4187b41a84e9a567d7137bdbe0f16 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c clone git github.com/opencontainers/runc v0.0.3 # libcontainer diff --git a/vendor/src/github.com/docker/notary/client/changelist/changelist.go b/vendor/src/github.com/docker/notary/client/changelist/changelist.go index 80cd2461f6..a91b16cb38 100644 --- a/vendor/src/github.com/docker/notary/client/changelist/changelist.go +++ b/vendor/src/github.com/docker/notary/client/changelist/changelist.go @@ -32,3 +32,28 @@ func (cl *memChangelist) Clear(archive string) error { func (cl *memChangelist) Close() error { return nil } + +func (cl *memChangelist) NewIterator() (ChangeIterator, error) { + return &MemChangeListIterator{index: 0, collection: cl.changes}, nil +} + +// MemChangeListIterator is a concrete instance of ChangeIterator +type MemChangeListIterator struct { + index int + collection []Change // Same type as memChangeList.changes +} + +// Next returns the next Change +func (m *MemChangeListIterator) Next() (item Change, err error) { + if m.index >= len(m.collection) { + return nil, IteratorBoundsError(m.index) + } + item = m.collection[m.index] + m.index++ + return item, err +} + +// HasNext indicates whether the iterator is exhausted +func (m *MemChangeListIterator) HasNext() bool { + return m.index < len(m.collection) +} diff --git a/vendor/src/github.com/docker/notary/client/changelist/file_changelist.go b/vendor/src/github.com/docker/notary/client/changelist/file_changelist.go index afee0cbcb9..5ab237d82e 100644 --- a/vendor/src/github.com/docker/notary/client/changelist/file_changelist.go +++ b/vendor/src/github.com/docker/notary/client/changelist/file_changelist.go @@ -28,30 +28,51 @@ func NewFileChangelist(dir string) (*FileChangelist, error) { return &FileChangelist{dir: dir}, nil } +// getFileNames reads directory, filtering out child directories +func getFileNames(dirName string) ([]os.FileInfo, error) { + var dirListing, fileInfos []os.FileInfo + dir, err := os.Open(dirName) + if err != nil { + return fileInfos, err + } + defer dir.Close() + dirListing, err = dir.Readdir(0) + if err != nil { + return fileInfos, err + } + for _, f := range dirListing { + if f.IsDir() { + continue + } + fileInfos = append(fileInfos, f) + } + return fileInfos, nil +} + +// Read a JSON formatted file from disk; convert to TufChange struct +func unmarshalFile(dirname string, f os.FileInfo) (*TufChange, error) { + c := &TufChange{} + raw, err := ioutil.ReadFile(path.Join(dirname, f.Name())) + if err != nil { + return c, err + } + err = json.Unmarshal(raw, c) + if err != nil { + return c, err + } + return c, nil +} + // List returns a list of sorted changes func (cl FileChangelist) List() []Change { var changes []Change - dir, err := os.Open(cl.dir) - if err != nil { - return changes - } - defer dir.Close() - fileInfos, err := dir.Readdir(0) + fileInfos, err := getFileNames(cl.dir) if err != nil { return changes } sort.Sort(fileChanges(fileInfos)) for _, f := range fileInfos { - if f.IsDir() { - continue - } - raw, err := ioutil.ReadFile(path.Join(cl.dir, f.Name())) - if err != nil { - logrus.Warn(err.Error()) - continue - } - c := &TufChange{} - err = json.Unmarshal(raw, c) + c, err := unmarshalFile(cl.dir, f) if err != nil { logrus.Warn(err.Error()) continue @@ -94,6 +115,47 @@ func (cl FileChangelist) Close() error { return nil } +// NewIterator creates an iterator from FileChangelist +func (cl FileChangelist) NewIterator() (ChangeIterator, error) { + fileInfos, err := getFileNames(cl.dir) + if err != nil { + return &FileChangeListIterator{}, err + } + sort.Sort(fileChanges(fileInfos)) + return &FileChangeListIterator{dirname: cl.dir, collection: fileInfos}, nil +} + +// IteratorBoundsError is an Error type used by Next() +type IteratorBoundsError int + +// Error implements the Error interface +func (e IteratorBoundsError) Error() string { + return fmt.Sprintf("Iterator index (%d) out of bounds", e) +} + +// FileChangeListIterator is a concrete instance of ChangeIterator +type FileChangeListIterator struct { + index int + dirname string + collection []os.FileInfo +} + +// Next returns the next Change in the FileChangeList +func (m *FileChangeListIterator) Next() (item Change, err error) { + if m.index >= len(m.collection) { + return nil, IteratorBoundsError(m.index) + } + f := m.collection[m.index] + m.index++ + item, err = unmarshalFile(m.dirname, f) + return +} + +// HasNext indicates whether iterator is exhausted +func (m *FileChangeListIterator) HasNext() bool { + return m.index < len(m.collection) +} + type fileChanges []os.FileInfo // Len returns the length of a file change list diff --git a/vendor/src/github.com/docker/notary/client/changelist/interface.go b/vendor/src/github.com/docker/notary/client/changelist/interface.go index a9b09b71fe..4369623c88 100644 --- a/vendor/src/github.com/docker/notary/client/changelist/interface.go +++ b/vendor/src/github.com/docker/notary/client/changelist/interface.go @@ -18,6 +18,10 @@ type Changelist interface { // Close syncronizes any pending writes to the underlying // storage and closes the file/connection Close() error + + // NewIterator returns an iterator for walking through the list + // of changes currently stored + NewIterator() (ChangeIterator, error) } const ( @@ -57,3 +61,10 @@ type Change interface { // action, it will be nil. Content() []byte } + +// ChangeIterator is the interface for iterating across collections of +// TUF Change items +type ChangeIterator interface { + Next() (Change, error) + HasNext() bool +} diff --git a/vendor/src/github.com/docker/notary/client/client.go b/vendor/src/github.com/docker/notary/client/client.go index 6d59af720a..ee376053c3 100644 --- a/vendor/src/github.com/docker/notary/client/client.go +++ b/vendor/src/github.com/docker/notary/client/client.go @@ -173,6 +173,9 @@ func (r *NotaryRepository) Initialize(uCryptoService *cryptoservice.UnlockedCryp // All the timestamp keys are generated by the remote server. remote, err := getRemoteStore(r.baseURL, r.gun, r.roundTrip) + if err != nil { + return err + } rawTSKey, err := remote.GetKey("timestamp") if err != nil { return err @@ -456,17 +459,20 @@ func (r *NotaryRepository) bootstrapRepo() error { if err != nil { return err } - root := &data.Signed{} + root := &data.SignedRoot{} err = json.Unmarshal(rootJSON, root) if err != nil { return err } - tufRepo.SetRoot(root) + err = tufRepo.SetRoot(root) + if err != nil { + return err + } targetsJSON, err := r.fileStore.GetMeta("targets", 0) if err != nil { return err } - targets := &data.Signed{} + targets := &data.SignedTargets{} err = json.Unmarshal(targetsJSON, targets) if err != nil { return err @@ -476,7 +482,7 @@ func (r *NotaryRepository) bootstrapRepo() error { if err != nil { return err } - snapshot := &data.Signed{} + snapshot := &data.SignedSnapshot{} err = json.Unmarshal(snapshotJSON, snapshot) if err != nil { return err @@ -557,6 +563,9 @@ func (r *NotaryRepository) bootstrapClient() (*tufclient.Client, error) { return nil, store.ErrMetaNotFound{} } } + // can't just unmarshal into SignedRoot because validate root + // needs the root.Signed field to still be []byte for signature + // validation root := &data.Signed{} err = json.Unmarshal(rootJSON, root) if err != nil { @@ -571,7 +580,11 @@ func (r *NotaryRepository) bootstrapClient() (*tufclient.Client, error) { kdb := keys.NewDB() r.tufRepo = tuf.NewTufRepo(kdb, r.cryptoService) - err = r.tufRepo.SetRoot(root) + signedRoot, err := data.RootFromSigned(root) + if err != nil { + return nil, err + } + err = r.tufRepo.SetRoot(signedRoot) if err != nil { return nil, err } diff --git a/vendor/src/github.com/docker/notary/client/helpers.go b/vendor/src/github.com/docker/notary/client/helpers.go index 93040b41da..476ef08b71 100644 --- a/vendor/src/github.com/docker/notary/client/helpers.go +++ b/vendor/src/github.com/docker/notary/client/helpers.go @@ -26,9 +26,16 @@ func getRemoteStore(baseURL, gun string, rt http.RoundTripper) (store.RemoteStor } func applyChangelist(repo *tuf.TufRepo, cl changelist.Changelist) error { - changes := cl.List() - logrus.Debugf("applying %d changes", len(changes)) - for _, c := range changes { + it, err := cl.NewIterator() + if err != nil { + return err + } + index := 0 + for it.HasNext() { + c, err := it.Next() + if err != nil { + return err + } switch c.Scope() { case changelist.ScopeTargets: err := applyTargetsChange(repo, c) @@ -38,7 +45,9 @@ func applyChangelist(repo *tuf.TufRepo, cl changelist.Changelist) error { default: logrus.Debug("scope not supported: ", c.Scope()) } + index++ } + logrus.Debugf("applied %d change(s)", index) return nil } diff --git a/vendor/src/github.com/docker/notary/keystoremanager/import_export.go b/vendor/src/github.com/docker/notary/keystoremanager/import_export.go index 55c736cdad..7b0cee94f5 100644 --- a/vendor/src/github.com/docker/notary/keystoremanager/import_export.go +++ b/vendor/src/github.com/docker/notary/keystoremanager/import_export.go @@ -16,6 +16,11 @@ import ( "github.com/docker/notary/trustmanager" ) +const ( + zipSymlinkAttr = 0xA1ED0000 + zipMadeByUNIX = 3 << 8 +) + var ( // ErrNoValidPrivateKey is returned if a key being imported doesn't // look like a private key @@ -126,15 +131,32 @@ func moveKeys(oldKeyStore, newKeyStore *trustmanager.KeyFileStore) error { } } + // Recreate symlinks + for _, relKeyPath := range oldKeyStore.ListFiles(true) { + fullKeyPath := filepath.Join(oldKeyStore.BaseDir(), relKeyPath) + + fi, err := os.Lstat(fullKeyPath) + if err != nil { + return err + } + + if (fi.Mode() & os.ModeSymlink) != 0 { + target, err := os.Readlink(fullKeyPath) + if err != nil { + return err + } + os.Symlink(target, filepath.Join(newKeyStore.BaseDir(), relKeyPath)) + } + } + return nil } func addKeysToArchive(zipWriter *zip.Writer, newKeyStore *trustmanager.KeyFileStore, subDir string) error { - // List all files but no symlinks - for _, relKeyPath := range newKeyStore.ListFiles(false) { + for _, relKeyPath := range newKeyStore.ListFiles(true) { fullKeyPath := filepath.Join(newKeyStore.BaseDir(), relKeyPath) - fi, err := os.Stat(fullKeyPath) + fi, err := os.Lstat(fullKeyPath) if err != nil { return err } @@ -145,17 +167,40 @@ func addKeysToArchive(zipWriter *zip.Writer, newKeyStore *trustmanager.KeyFileSt } infoHeader.Name = filepath.Join(subDir, relKeyPath) - zipFileEntryWriter, err := zipWriter.CreateHeader(infoHeader) - if err != nil { - return err - } - fileContents, err := ioutil.ReadFile(fullKeyPath) - if err != nil { - return err - } - if _, err = zipFileEntryWriter.Write(fileContents); err != nil { - return err + // Is this a symlink? If so, encode properly in the zip file. + if (fi.Mode() & os.ModeSymlink) != 0 { + infoHeader.CreatorVersion = zipMadeByUNIX + infoHeader.ExternalAttrs = zipSymlinkAttr + + zipFileEntryWriter, err := zipWriter.CreateHeader(infoHeader) + if err != nil { + return err + } + + target, err := os.Readlink(fullKeyPath) + if err != nil { + return err + } + + // Write relative path + if _, err = zipFileEntryWriter.Write([]byte(target)); err != nil { + return err + } + } else { + zipFileEntryWriter, err := zipWriter.CreateHeader(infoHeader) + if err != nil { + return err + } + + fileContents, err := ioutil.ReadFile(fullKeyPath) + if err != nil { + return err + } + + if _, err = zipFileEntryWriter.Write(fileContents); err != nil { + return err + } } } @@ -197,7 +242,6 @@ func (km *KeyStoreManager) ExportAllKeys(dest io.Writer, newPassphraseRetriever return err } if err := addKeysToArchive(zipWriter, tempNonRootKeyStore, privNonRootKeysSubdir); err != nil { - return err } @@ -206,7 +250,13 @@ func (km *KeyStoreManager) ExportAllKeys(dest io.Writer, newPassphraseRetriever return nil } -// ImportKeysZip imports keys from a zip file provided as an io.ReaderAt. The +// IsZipSymlink returns true if the file described by the zip file header is a +// symlink. +func IsZipSymlink(f *zip.File) bool { + return f.CreatorVersion&0xFF00 == zipMadeByUNIX && f.ExternalAttrs == zipSymlinkAttr +} + +// ImportKeysZip imports keys from a zip file provided as an zip.Reader. The // keys in the root_keys directory are left encrypted, but the other keys are // decrypted with the specified passphrase. func (km *KeyStoreManager) ImportKeysZip(zipReader zip.Reader) error { @@ -239,17 +289,33 @@ func (km *KeyStoreManager) ImportKeysZip(zipReader zip.Reader) error { // Note that using / as a separator is okay here - the zip // package guarantees that the separator will be / if strings.HasPrefix(fNameTrimmed, rootKeysPrefix) { - if err = checkRootKeyIsEncrypted(fileBytes); err != nil { - rc.Close() - return err + if IsZipSymlink(f) { + newName := filepath.Join(km.rootKeyStore.BaseDir(), strings.TrimPrefix(f.Name, rootKeysPrefix)) + err = os.Symlink(string(fileBytes), newName) + if err != nil { + return err + } + } else { + if err = checkRootKeyIsEncrypted(fileBytes); err != nil { + rc.Close() + return err + } + // Root keys are preserved without decrypting + keyName := strings.TrimPrefix(fNameTrimmed, rootKeysPrefix) + newRootKeys[keyName] = fileBytes } - // Root keys are preserved without decrypting - keyName := strings.TrimPrefix(fNameTrimmed, rootKeysPrefix) - newRootKeys[keyName] = fileBytes } else if strings.HasPrefix(fNameTrimmed, nonRootKeysPrefix) { - // Nonroot keys are preserved without decrypting - keyName := strings.TrimPrefix(fNameTrimmed, nonRootKeysPrefix) - newNonRootKeys[keyName] = fileBytes + if IsZipSymlink(f) { + newName := filepath.Join(km.nonRootKeyStore.BaseDir(), strings.TrimPrefix(f.Name, nonRootKeysPrefix)) + err = os.Symlink(string(fileBytes), newName) + if err != nil { + return err + } + } else { + // Nonroot keys are preserved without decrypting + keyName := strings.TrimPrefix(fNameTrimmed, nonRootKeysPrefix) + newNonRootKeys[keyName] = fileBytes + } } else { // This path inside the zip archive doesn't look like a // root key, non-root key, or alias. To avoid adding a file @@ -281,7 +347,6 @@ func (km *KeyStoreManager) ImportKeysZip(zipReader zip.Reader) error { func moveKeysByGUN(oldKeyStore, newKeyStore *trustmanager.KeyFileStore, gun string) error { // List all files but no symlinks for relKeyPath := range oldKeyStore.ListKeys() { - // Skip keys that aren't associated with this GUN if !strings.HasPrefix(relKeyPath, filepath.FromSlash(gun)) { continue diff --git a/vendor/src/github.com/docker/notary/pkg/passphrase/passphrase.go b/vendor/src/github.com/docker/notary/pkg/passphrase/passphrase.go index e89092b891..9359e0efc0 100644 --- a/vendor/src/github.com/docker/notary/pkg/passphrase/passphrase.go +++ b/vendor/src/github.com/docker/notary/pkg/passphrase/passphrase.go @@ -150,7 +150,7 @@ func PromptRetrieverWithInOut(in io.Reader, out io.Writer, aliasMap map[string]s } if len(retPass) < 8 { - fmt.Fprintln(out, "Please use a password manager to generate and store a good random passphrase.") + fmt.Fprintln(out, "Passphrase is too short. Please use a password manager to generate and store a good random passphrase.") return "", false, ErrTooShort } diff --git a/vendor/src/github.com/docker/notary/trustmanager/filestore.go b/vendor/src/github.com/docker/notary/trustmanager/filestore.go index d573888d6a..5aa3e7aef8 100644 --- a/vendor/src/github.com/docker/notary/trustmanager/filestore.go +++ b/vendor/src/github.com/docker/notary/trustmanager/filestore.go @@ -189,7 +189,7 @@ func (f *SimpleFileStore) genFileName(name string) string { return fmt.Sprintf("%s.%s", name, f.fileExt) } -// Link creates a symlink beetween the ID of the certificate used by a repository +// Link creates a symlink between the ID of the certificate used by a repository // and the ID of the root key that is being used. // We use full path for the source and local for the destination to use relative // path for the symlink diff --git a/vendor/src/github.com/endophage/gotuf/client/client.go b/vendor/src/github.com/endophage/gotuf/client/client.go index 9cb910717b..2bcda4b741 100644 --- a/vendor/src/github.com/endophage/gotuf/client/client.go +++ b/vendor/src/github.com/endophage/gotuf/client/client.go @@ -223,7 +223,12 @@ func (c Client) verifyRoot(role string, s *data.Signed, minVersion int) error { // This will cause keyDB to get updated, overwriting any keyIDs associated // with the roles in root.json logrus.Debug("updating known root roles and keys") - err = c.local.SetRoot(s) + root, err := data.RootFromSigned(s) + if err != nil { + logrus.Error(err.Error()) + return err + } + err = c.local.SetRoot(root) if err != nil { logrus.Error(err.Error()) return err @@ -298,7 +303,11 @@ func (c *Client) downloadTimestamp() error { if download { c.cache.SetMeta(role, raw) } - c.local.SetTimestamp(s) + ts, err := data.TimestampFromSigned(s) + if err != nil { + return err + } + c.local.SetTimestamp(ts) return nil } @@ -366,7 +375,11 @@ func (c *Client) downloadSnapshot() error { return err } logrus.Debug("successfully verified snapshot") - c.local.SetSnapshot(s) + snap, err := data.SnapshotFromSigned(s) + if err != nil { + return err + } + c.local.SetSnapshot(snap) if download { err = c.cache.SetMeta(role, raw) if err != nil { @@ -393,7 +406,11 @@ func (c *Client) downloadTargets(role string) error { logrus.Error("Error getting targets file:", err) return err } - err = c.local.SetTargets(role, s) + t, err := data.TargetsFromSigned(s) + if err != nil { + return err + } + err = c.local.SetTargets(role, t) if err != nil { return err } diff --git a/vendor/src/github.com/endophage/gotuf/data/keys.go b/vendor/src/github.com/endophage/gotuf/data/keys.go index 118617ce24..3df1ce05c7 100644 --- a/vendor/src/github.com/endophage/gotuf/data/keys.go +++ b/vendor/src/github.com/endophage/gotuf/data/keys.go @@ -5,7 +5,7 @@ import ( "encoding/hex" "github.com/Sirupsen/logrus" - cjson "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) type Key interface { @@ -57,7 +57,7 @@ func (k TUFKey) Algorithm() KeyAlgorithm { func (k *TUFKey) ID() string { if k.id == "" { pubK := NewPublicKey(k.Algorithm(), k.Public()) - data, err := cjson.Marshal(&pubK) + data, err := json.MarshalCanonical(&pubK) if err != nil { logrus.Error("Error generating key ID:", err) } diff --git a/vendor/src/github.com/endophage/gotuf/data/root.go b/vendor/src/github.com/endophage/gotuf/data/root.go index b290322d44..5730e9bf0c 100644 --- a/vendor/src/github.com/endophage/gotuf/data/root.go +++ b/vendor/src/github.com/endophage/gotuf/data/root.go @@ -1,10 +1,9 @@ package data import ( - "encoding/json" "time" - cjson "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) type SignedRoot struct { @@ -58,7 +57,7 @@ func NewRoot(keys map[string]PublicKey, roles map[string]*RootRole, consistent b } func (r SignedRoot) ToSigned() (*Signed, error) { - s, err := cjson.Marshal(r.Signed) + s, err := json.MarshalCanonical(r.Signed) if err != nil { return nil, err } diff --git a/vendor/src/github.com/endophage/gotuf/data/snapshot.go b/vendor/src/github.com/endophage/gotuf/data/snapshot.go index 1a88915eae..5b61e46f49 100644 --- a/vendor/src/github.com/endophage/gotuf/data/snapshot.go +++ b/vendor/src/github.com/endophage/gotuf/data/snapshot.go @@ -2,11 +2,10 @@ package data import ( "bytes" - "encoding/json" "time" "github.com/Sirupsen/logrus" - cjson "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) type SignedSnapshot struct { @@ -61,7 +60,7 @@ func (sp *SignedSnapshot) hashForRole(role string) []byte { } func (sp SignedSnapshot) ToSigned() (*Signed, error) { - s, err := cjson.Marshal(sp.Signed) + s, err := json.MarshalCanonical(sp.Signed) if err != nil { return nil, err } diff --git a/vendor/src/github.com/endophage/gotuf/data/targets.go b/vendor/src/github.com/endophage/gotuf/data/targets.go index 74c2c0b176..a51f60d208 100644 --- a/vendor/src/github.com/endophage/gotuf/data/targets.go +++ b/vendor/src/github.com/endophage/gotuf/data/targets.go @@ -3,10 +3,8 @@ package data import ( "crypto/sha256" "encoding/hex" - "encoding/json" - "time" - cjson "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) type SignedTargets struct { @@ -16,9 +14,7 @@ type SignedTargets struct { } type Targets struct { - Type string `json:"_type"` - Version int `json:"version"` - Expires time.Time `json:"expires"` + SignedCommon Targets Files `json:"targets"` Delegations Delegations `json:"delegations,omitempty"` } @@ -27,9 +23,11 @@ func NewTargets() *SignedTargets { return &SignedTargets{ Signatures: make([]Signature, 0), Signed: Targets{ - Type: TUFTypes["targets"], - Version: 0, - Expires: DefaultExpires("targets"), + SignedCommon: SignedCommon{ + Type: TUFTypes["targets"], + Version: 0, + Expires: DefaultExpires("targets"), + }, Targets: make(Files), Delegations: *NewDelegations(), }, @@ -86,7 +84,7 @@ func (t *SignedTargets) AddDelegation(role *Role, keys []*PublicKey) error { } func (t SignedTargets) ToSigned() (*Signed, error) { - s, err := cjson.Marshal(t.Signed) + s, err := json.MarshalCanonical(t.Signed) if err != nil { return nil, err } diff --git a/vendor/src/github.com/endophage/gotuf/data/timestamp.go b/vendor/src/github.com/endophage/gotuf/data/timestamp.go index 3a26df1b3b..e8a0de7cd2 100644 --- a/vendor/src/github.com/endophage/gotuf/data/timestamp.go +++ b/vendor/src/github.com/endophage/gotuf/data/timestamp.go @@ -2,10 +2,9 @@ package data import ( "bytes" - "encoding/json" "time" - cjson "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) type SignedTimestamp struct { @@ -44,7 +43,7 @@ func NewTimestamp(snapshot *Signed) (*SignedTimestamp, error) { } func (ts SignedTimestamp) ToSigned() (*Signed, error) { - s, err := cjson.Marshal(ts.Signed) + s, err := json.MarshalCanonical(ts.Signed) if err != nil { return nil, err } diff --git a/vendor/src/github.com/endophage/gotuf/data/types.go b/vendor/src/github.com/endophage/gotuf/data/types.go index 98d55f32be..7cc84874d5 100644 --- a/vendor/src/github.com/endophage/gotuf/data/types.go +++ b/vendor/src/github.com/endophage/gotuf/data/types.go @@ -3,7 +3,6 @@ package data import ( "crypto/sha256" "crypto/sha512" - "encoding/json" "fmt" "hash" "io" @@ -12,6 +11,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/jfrazelle/go/canonical/json" ) type KeyAlgorithm string @@ -85,6 +85,17 @@ type Signed struct { Signatures []Signature `json:"signatures"` } +type SignedCommon struct { + Type string `json:"_type"` + Expires time.Time `json:"expires"` + Version int `json:"version"` +} + +type SignedMeta struct { + Signed SignedCommon `json:"signed"` + Signatures []Signature `json:"signatures"` +} + type Signature struct { KeyID string `json:"keyid"` Method SigAlgorithm `json:"method"` diff --git a/vendor/src/github.com/endophage/gotuf/signed/verify.go b/vendor/src/github.com/endophage/gotuf/signed/verify.go index fe79563f32..b1ef6a3aea 100644 --- a/vendor/src/github.com/endophage/gotuf/signed/verify.go +++ b/vendor/src/github.com/endophage/gotuf/signed/verify.go @@ -1,7 +1,6 @@ package signed import ( - "encoding/json" "errors" "strings" "time" @@ -9,7 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/endophage/gotuf/data" "github.com/endophage/gotuf/keys" - "github.com/tent/canonical-json-go" + "github.com/jfrazelle/go/canonical/json" ) var ( @@ -21,12 +20,6 @@ var ( ErrWrongType = errors.New("tuf: meta file has wrong type") ) -type signedMeta struct { - Type string `json:"_type"` - Expires time.Time `json:"expires"` - Version int `json:"version"` -} - // VerifyRoot checks if a given root file is valid against a known set of keys. // Threshold is always assumed to be 1 func VerifyRoot(s *data.Signed, minVersion int, keys map[string]data.PublicKey) error { @@ -38,7 +31,7 @@ func VerifyRoot(s *data.Signed, minVersion int, keys map[string]data.PublicKey) if err := json.Unmarshal(s.Signed, &decoded); err != nil { return err } - msg, err := cjson.Marshal(decoded) + msg, err := json.MarshalCanonical(decoded) if err != nil { return err } @@ -76,7 +69,7 @@ func Verify(s *data.Signed, role string, minVersion int, db *keys.KeyDB) error { } func verifyMeta(s *data.Signed, role string, minVersion int) error { - sm := &signedMeta{} + sm := &data.SignedCommon{} if err := json.Unmarshal(s.Signed, sm); err != nil { return err } @@ -117,7 +110,7 @@ func VerifySignatures(s *data.Signed, role string, db *keys.KeyDB) error { if err := json.Unmarshal(s.Signed, &decoded); err != nil { return err } - msg, err := cjson.Marshal(decoded) + msg, err := json.MarshalCanonical(decoded) if err != nil { return err } diff --git a/vendor/src/github.com/endophage/gotuf/store/dbstore.go b/vendor/src/github.com/endophage/gotuf/store/dbstore.go index f67f3729e0..8d65d0db21 100644 --- a/vendor/src/github.com/endophage/gotuf/store/dbstore.go +++ b/vendor/src/github.com/endophage/gotuf/store/dbstore.go @@ -3,7 +3,6 @@ package store import ( "database/sql" "encoding/hex" - "encoding/json" "fmt" "io/ioutil" "os" @@ -12,6 +11,7 @@ import ( logrus "github.com/Sirupsen/logrus" "github.com/endophage/gotuf/data" "github.com/endophage/gotuf/utils" + "github.com/jfrazelle/go/canonical/json" ) const ( diff --git a/vendor/src/github.com/endophage/gotuf/store/httpstore.go b/vendor/src/github.com/endophage/gotuf/store/httpstore.go index 1287a6d7ab..1a82b094cd 100644 --- a/vendor/src/github.com/endophage/gotuf/store/httpstore.go +++ b/vendor/src/github.com/endophage/gotuf/store/httpstore.go @@ -14,10 +14,12 @@ import ( "github.com/Sirupsen/logrus" ) -type ErrServerUnavailable struct{} +type ErrServerUnavailable struct { + code int +} func (err ErrServerUnavailable) Error() string { - return "Unable to reach trust server at this time." + return fmt.Sprintf("Unable to reach trust server at this time: %d.", err.code) } type ErrShortRead struct{} @@ -85,13 +87,15 @@ func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) { return nil, err } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, ErrMetaNotFound{} + } else if resp.StatusCode != http.StatusOK { + return nil, ErrServerUnavailable{code: resp.StatusCode} + } if resp.ContentLength > size { return nil, ErrMaliciousServer{} } logrus.Debugf("%d when retrieving metadata for %s", resp.StatusCode, name) - if resp.StatusCode == http.StatusNotFound { - return nil, ErrMetaNotFound{} - } b := io.LimitReader(resp.Body, size) body, err := ioutil.ReadAll(b) if resp.ContentLength > 0 && int64(len(body)) < resp.ContentLength { @@ -113,8 +117,17 @@ func (s HTTPStore) SetMeta(name string, blob []byte) error { if err != nil { return err } - _, err = s.roundTrip.RoundTrip(req) - return err + resp, err := s.roundTrip.RoundTrip(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrMetaNotFound{} + } else if resp.StatusCode != http.StatusOK { + return ErrServerUnavailable{code: resp.StatusCode} + } + return nil } func (s HTTPStore) SetMultiMeta(metas map[string][]byte) error { @@ -140,8 +153,17 @@ func (s HTTPStore) SetMultiMeta(metas map[string][]byte) error { if err != nil { return err } - _, err = s.roundTrip.RoundTrip(req) - return err + resp, err := s.roundTrip.RoundTrip(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrMetaNotFound{} + } else if resp.StatusCode != http.StatusOK { + return ErrServerUnavailable{code: resp.StatusCode} + } + return nil } func (s HTTPStore) buildMetaURL(name string) (*url.URL, error) { @@ -188,6 +210,12 @@ func (s HTTPStore) GetTarget(path string) (io.ReadCloser, error) { if err != nil { return nil, err } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, ErrMetaNotFound{} + } else if resp.StatusCode != http.StatusOK { + return nil, ErrServerUnavailable{code: resp.StatusCode} + } return resp.Body, nil } @@ -205,6 +233,11 @@ func (s HTTPStore) GetKey(role string) ([]byte, error) { return nil, err } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, ErrMetaNotFound{} + } else if resp.StatusCode != http.StatusOK { + return nil, ErrServerUnavailable{code: resp.StatusCode} + } body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err diff --git a/vendor/src/github.com/endophage/gotuf/tuf.go b/vendor/src/github.com/endophage/gotuf/tuf.go index 23d04ca4a0..4d226acebb 100644 --- a/vendor/src/github.com/endophage/gotuf/tuf.go +++ b/vendor/src/github.com/endophage/gotuf/tuf.go @@ -258,16 +258,12 @@ func (tr *TufRepo) InitTimestamp() error { // SetRoot parses the Signed object into a SignedRoot object, sets // the keys and roles in the KeyDB, and sets the TufRepo.Root field // to the SignedRoot object. -func (tr *TufRepo) SetRoot(s *data.Signed) error { - r, err := data.RootFromSigned(s) - if err != nil { - return err - } - for _, key := range r.Signed.Keys { +func (tr *TufRepo) SetRoot(s *data.SignedRoot) error { + for _, key := range s.Signed.Keys { logrus.Debug("Adding key ", key.ID()) tr.keysDB.AddKey(key) } - for roleName, role := range r.Signed.Roles { + for roleName, role := range s.Signed.Roles { logrus.Debugf("Adding role %s with keys %s", roleName, strings.Join(role.KeyIDs, ",")) baseRole, err := data.NewRole( roleName, @@ -284,48 +280,35 @@ func (tr *TufRepo) SetRoot(s *data.Signed) error { return err } } - tr.Root = r + tr.Root = s return nil } // SetTimestamp parses the Signed object into a SignedTimestamp object // and sets the TufRepo.Timestamp field. -func (tr *TufRepo) SetTimestamp(s *data.Signed) error { - ts, err := data.TimestampFromSigned(s) - if err != nil { - return err - } - tr.Timestamp = ts +func (tr *TufRepo) SetTimestamp(s *data.SignedTimestamp) error { + tr.Timestamp = s return nil } // SetSnapshot parses the Signed object into a SignedSnapshots object // and sets the TufRepo.Snapshot field. -func (tr *TufRepo) SetSnapshot(s *data.Signed) error { - snap, err := data.SnapshotFromSigned(s) - if err != nil { - return err - } - - tr.Snapshot = snap +func (tr *TufRepo) SetSnapshot(s *data.SignedSnapshot) error { + tr.Snapshot = s return nil } // SetTargets parses the Signed object into a SignedTargets object, // reads the delegated roles and keys into the KeyDB, and sets the // SignedTargets object agaist the role in the TufRepo.Targets map. -func (tr *TufRepo) SetTargets(role string, s *data.Signed) error { - t, err := data.TargetsFromSigned(s) - if err != nil { - return err - } - for _, k := range t.Signed.Delegations.Keys { +func (tr *TufRepo) SetTargets(role string, s *data.SignedTargets) error { + for _, k := range s.Signed.Delegations.Keys { tr.keysDB.AddKey(k) } - for _, r := range t.Signed.Delegations.Roles { + for _, r := range s.Signed.Delegations.Roles { tr.keysDB.AddRole(r) } - tr.Targets[role] = t + tr.Targets[role] = s return nil } diff --git a/vendor/src/github.com/endophage/gotuf/utils/utils.go b/vendor/src/github.com/endophage/gotuf/utils/utils.go index 38d46b031a..ca164a1aa4 100644 --- a/vendor/src/github.com/endophage/gotuf/utils/utils.go +++ b/vendor/src/github.com/endophage/gotuf/utils/utils.go @@ -3,6 +3,7 @@ package utils import ( "bytes" "crypto/sha256" + "crypto/sha512" "crypto/tls" "fmt" "io" @@ -78,3 +79,15 @@ type NoopCloser struct { func (nc *NoopCloser) Close() error { return nil } + +func DoHash(alg string, d []byte) []byte { + switch alg { + case "sha256": + digest := sha256.Sum256(d) + return digest[:] + case "sha512": + digest := sha512.Sum512(d) + return digest[:] + } + return nil +} diff --git a/vendor/src/github.com/jfrazelle/go/canonical/json/decode.go b/vendor/src/github.com/jfrazelle/go/canonical/json/decode.go new file mode 100644 index 0000000000..51b4ffea9d --- /dev/null +++ b/vendor/src/github.com/jfrazelle/go/canonical/json/decode.go @@ -0,0 +1,1085 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "errors" + "fmt" + "reflect" + "runtime" + "strconv" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshalling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +// +func Unmarshal(data []byte, v interface{}) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by objects +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to +} + +func (e *UnmarshalTypeError) Error() string { + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// (No longer used; kept for compatibility.) +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) (err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + d.value(rv) + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + savedError error + useNumber bool + canonical bool +} + +// errPhase is used for errors that should not happen unless +// there is a bug in the JSON decoder or something is editing +// the data slice while the decoder executes. +var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + return d +} + +// error aborts the decoding by panicking with err. +func (d *decodeState) error(err error) { + panic(err) +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = err + } +} + +// next cuts off and returns the next full JSON value in d.data[d.off:]. +// The next value is known to be an object or array, not a literal. +func (d *decodeState) next() []byte { + c := d.data[d.off] + item, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // Our scanner has seen the opening brace/bracket + // and thinks we're still in the middle of the object. + // invent a closing brace/bracket to get it out. + if c == '{' { + d.scan.step(&d.scan, '}') + } else { + d.scan.step(&d.scan, ']') + } + + return item +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +// It updates d.off and returns the new scan code. +func (d *decodeState) scanWhile(op int) int { + var newOp int + for { + if d.off >= len(d.data) { + newOp = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } else { + c := int(d.data[d.off]) + d.off++ + newOp = d.scan.step(&d.scan, c) + } + if newOp != op { + break + } + } + return newOp +} + +// value decodes a JSON value from d.data[d.off:] into the value. +// it updates d.off to point past the decoded value. +func (d *decodeState) value(v reflect.Value) { + if !v.IsValid() { + _, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // d.scan thinks we're still at the beginning of the item. + // Feed in an empty string - the shortest, simplest value - + // so that it knows we got to the end of the value. + if d.scan.redo { + // rewind. + d.scan.redo = false + d.scan.step = stateBeginValue + } + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + + n := len(d.scan.parseState) + if n > 0 && d.scan.parseState[n-1] == parseObjectKey { + // d.scan thinks we just read an object key; finish the object + d.scan.step(&d.scan, ':') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '}') + } + + return + } + + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(v) + + case scanBeginObject: + d.object(v) + + case scanBeginLiteral: + d.literal(v) + } +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() interface{} { + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(reflect.Value{}) + + case scanBeginObject: + d.object(reflect.Value{}) + + case scanBeginLiteral: + switch v := d.literalInterface().(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into the value v. +// the first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) { + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + } + + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + v.Set(reflect.ValueOf(d.arrayInterface())) + return + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + case reflect.Array: + case reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + // Get element of array, growing if necessary. + if v.Kind() == reflect.Slice { + // Grow slice if necessary + if i >= v.Cap() { + newcap := v.Cap() + v.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) + reflect.Copy(newv, v) + v.Set(newv) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + d.value(v.Index(i)) + } else { + // Ran out of fixed array: skip. + d.value(reflect.Value{}) + } + i++ + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + // Array. Zero the rest. + z := reflect.Zero(v.Type().Elem()) + for ; i < v.Len(); i++ { + v.Index(i).Set(z) + } + } else { + v.SetLen(i) + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } +} + +var nullLiteral = []byte("null") + +// object consumes an object from d.data[d.off-1:], decoding into the value v. +// the first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) { + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.off-- + d.next() // skip over { } in input + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + v.Set(reflect.ValueOf(d.objectInterface())) + return + } + + // Check type of target: struct or map[string]T + switch v.Kind() { + case reflect.Map: + // map must have string kind + t := v.Type() + if t.Key().Kind() != reflect.String { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.off-- + d.next() // skip over { } in input + return + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + + default: + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.off-- + d.next() // skip over { } in input + return + } + + var mapElem reflect.Value + + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquoteBytes(item) + if !ok { + d.error(errPhase) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := v.Type().Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.Set(reflect.Zero(elemType)) + } + subv = mapElem + } else { + var f *field + fields := cachedTypeFields(v.Type(), false) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, key) { + f = ff + break + } + if f == nil && ff.equalFold(ff.nameBytes, key) { + f = ff + } + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + } + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + d.literalStore(nullLiteral, subv, false) + case string: + d.literalStore([]byte(qv), subv, true) + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", item, v.Type())) + } + } else { + d.value(subv) + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kv := reflect.ValueOf(key).Convert(v.Type().Key()) + v.SetMapIndex(kv, subv) + } + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } +} + +// literal consumes a literal from d.data[d.off-1:], decoding into the value v. +// The first byte of the literal has been read already +// (that's how the caller knows it's a literal). +func (d *decodeState) literal(v reflect.Value) { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + + d.literalStore(d.data[start:d.off], v, false) +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { + // Check for unmarshaler. + if len(item) == 0 { + //Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return + } + wantptr := item[0] == 'n' // null + u, ut, pv := d.indirect(v, wantptr) + if u != nil { + err := u.UnmarshalJSON(item) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + } + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + err := ut.UnmarshalText(s) + if err != nil { + d.error(err) + } + return + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + switch v.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + v.Set(reflect.Zero(v.Type())) + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := c == 't' + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case reflect.Slice: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.ValueOf(b[0:n])) + case reflect.String: + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + s := string(item) + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + v.SetString(s) + break + } + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(&UnmarshalTypeError{"number", v.Type()}) + } + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{"number", v.Type()}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetFloat(n) + } + } +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() interface{} { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + panic("unreachable") + case scanBeginArray: + return d.arrayInterface() + case scanBeginObject: + return d.objectInterface() + case scanBeginLiteral: + return d.literalInterface() + } +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// literalInterface is like literal but returns an interface value. +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + d.error(errPhase) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + r, err := strconv.ParseUint(string(s[2:6]), 16, 64) + if err != nil { + return -1 + } + return rune(r) +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/vendor/src/github.com/jfrazelle/go/canonical/json/encode.go b/vendor/src/github.com/jfrazelle/go/canonical/json/encode.go new file mode 100644 index 0000000000..655f1a6661 --- /dev/null +++ b/vendor/src/github.com/jfrazelle/go/canonical/json/encode.go @@ -0,0 +1,1234 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON objects as defined in +// RFC 4627. The mapping between JSON objects and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// http://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "math" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" +// to keep some browsers from misinterpreting JSON output as HTML. +// Ampersand "&" is also escaped to "\u0026" for the same reason. +// +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string, and a nil slice +// encodes as the null JSON object. +// +// Struct values encode as JSON objects. Each exported struct field +// becomes a member of the object unless +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option. +// The empty values are false, 0, any +// nil pointer or interface value, and any array, slice, map, or string of +// length zero. The object's default key string is the struct field name +// but can be specified in the struct field's tag value. The "json" key in +// the struct field's tag value is the key name, followed by an optional comma +// and options. Examples: +// +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "myName". +// Field int `json:"myName"` +// +// // Field appears in JSON as key "myName" and +// // the field is omitted from the object if its value is empty, +// // as defined above. +// Field int `json:"myName,omitempty"` +// +// // Field appears in JSON as key "Field" (the default), but +// // the field is skipped if empty. +// // Note the leading comma. +// Field int `json:",omitempty"` +// +// The "string" option signals that a field is stored as JSON inside a +// JSON-encoded string. It applies only to fields of string, floating point, +// or integer types. This extra level of encoding is sometimes used when +// communicating with JavaScript programs: +// +// Int64String int64 `json:",string"` +// +// The key name will be used if it's a non-empty string consisting of +// only Unicode letters, digits, dollar signs, percent signs, hyphens, +// underscores and slashes. +// +// Anonymous struct fields are usually marshaled as if their inner exported fields +// were fields in the outer struct, subject to the usual Go visibility rules amended +// as described in the next paragraph. +// An anonymous struct field with a name given in its JSON tag is treated as +// having that name, rather than being anonymous. +// An anonymous struct field of interface type is treated the same as having +// that type as its name, rather than being anonymous. +// +// The Go visibility rules for struct fields are amended for JSON when +// deciding which field to marshal or unmarshal. If there are +// multiple fields at the same level, and that level is the least +// nested (and would therefore be the nesting level selected by the +// usual Go rules), the following extra rules apply: +// +// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, +// even if there are multiple untagged fields that would otherwise conflict. +// 2) If there is exactly one field (tagged or not according to the first rule), that is selected. +// 3) Otherwise there are multiple fields, and all are ignored; no error occurs. +// +// Handling of anonymous struct fields is new in Go 1.1. +// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of +// an anonymous struct field in both current and earlier versions, give the field +// a JSON tag of "-". +// +// Map values encode as JSON objects. +// The map's key type must be string; the object keys are used directly +// as map keys. +// +// Pointer values encode as the value pointed to. +// A nil pointer encodes as the null JSON object. +// +// Interface values encode as the value contained in the interface. +// A nil interface value encodes as the null JSON object. +// +// Channel, complex, and function values cannot be encoded in JSON. +// Attempting to encode such a value causes Marshal to return +// an UnsupportedTypeError. +// +// JSON cannot represent cyclic data structures and Marshal does not +// handle them. Passing cyclic structures to Marshal will result in +// an infinite recursion. +// +func Marshal(v interface{}) ([]byte, error) { + return marshal(v, false) +} + +// MarshalIndent is like Marshal but applies Indent to format the output. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + b, err := Marshal(v) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = Indent(&buf, b, prefix, indent) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MarshalCanonical is like Marshal but encodes into Canonical JSON. +// Read more at: http://wiki.laptop.org/go/Canonical_JSON +func MarshalCanonical(v interface{}) ([]byte, error) { + return marshal(v, true) +} + +func marshal(v interface{}, canonical bool) ([]byte, error) { + e := &encodeState{canonical: canonical} + err := e.marshal(v) + if err != nil { + return nil, err + } + return e.Bytes(), nil +} + +// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 +// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 +// so that the JSON will be safe to embed inside HTML