From 9322636c8926f155a10d9679e364e3f8e86203d6 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Fri, 4 Nov 2016 12:20:02 -0700 Subject: [PATCH] Revendor go-memdb to floridoo/go-memdb's fork (608dda3b1410a73eaf3ac8b517c9ae7ebab6aa87) We are temporarily using a fork containing a PR that hasn't been merged upstream yet. Signed-off-by: Andrea Luzzardi --- vendor.conf | 2 +- vendor/github.com/hashicorp/go-memdb/index.go | 130 +++++++++++++----- .../github.com/hashicorp/go-memdb/schema.go | 9 ++ vendor/github.com/hashicorp/go-memdb/txn.go | 121 ++++++++-------- 4 files changed, 175 insertions(+), 87 deletions(-) diff --git a/vendor.conf b/vendor.conf index 10b559b40b..9c8ec81555 100644 --- a/vendor.conf +++ b/vendor.conf @@ -108,7 +108,7 @@ github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b65068 golang.org/x/crypto 3fbbcd23f1cb824e69491a5930cfeff09b12f4d2 golang.org/x/time a4bde12657593d5e90d0533a3e4fd95e635124cb github.com/mreiferson/go-httpclient 63fe23f7434723dc904c901043af07931f293c47 -github.com/hashicorp/go-memdb 98f52f52d7a476958fa9da671354d270c50661a7 +github.com/hashicorp/go-memdb 608dda3b1410a73eaf3ac8b517c9ae7ebab6aa87 https://github.com/floridoo/go-memdb github.com/hashicorp/go-immutable-radix 8e8ed81f8f0bf1bdd829593fdd5c29922c1ea990 github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 github.com/coreos/pkg fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 diff --git a/vendor/github.com/hashicorp/go-memdb/index.go b/vendor/github.com/hashicorp/go-memdb/index.go index 7237f33e27..0e9062d20d 100644 --- a/vendor/github.com/hashicorp/go-memdb/index.go +++ b/vendor/github.com/hashicorp/go-memdb/index.go @@ -9,15 +9,27 @@ import ( // Indexer is an interface used for defining indexes type Indexer interface { - // FromObject is used to extract an index value from an - // object or to indicate that the index value is missing. - FromObject(raw interface{}) (bool, []byte, error) - // ExactFromArgs is used to build an exact index lookup // based on arguments FromArgs(args ...interface{}) ([]byte, error) } +// SingleIndexer is an interface used for defining indexes +// generating a single entry per object +type SingleIndexer interface { + // FromObject is used to extract an index value from an + // object or to indicate that the index value is missing. + FromObject(raw interface{}) (bool, []byte, error) +} + +// MultiIndexer is an interface used for defining indexes +// generating multiple entries per object +type MultiIndexer interface { + // FromObject is used to extract index values from an + // object or to indicate that the index value is missing. + FromObject(raw interface{}) (bool, [][]byte, error) +} + // PrefixIndexer can optionally be implemented for any // indexes that support prefix based iteration. This may // not apply to all indexes. @@ -88,6 +100,79 @@ func (s *StringFieldIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) { return val, nil } +// StringSliceFieldIndex is used to extract a field from an object +// using reflection and builds an index on that field. +type StringSliceFieldIndex struct { + Field string + Lowercase bool +} + +func (s *StringSliceFieldIndex) FromObject(obj interface{}) (bool, [][]byte, error) { + v := reflect.ValueOf(obj) + v = reflect.Indirect(v) // Dereference the pointer if any + + fv := v.FieldByName(s.Field) + if !fv.IsValid() { + return false, nil, + fmt.Errorf("field '%s' for %#v is invalid", s.Field, obj) + } + + if fv.Kind() != reflect.Slice || fv.Type().Elem().Kind() != reflect.String { + return false, nil, fmt.Errorf("field '%s' is not a string slice", s.Field) + } + + length := fv.Len() + vals := make([][]byte, 0, length) + for i := 0; i < fv.Len(); i++ { + val := fv.Index(i).String() + if val == "" { + continue + } + + if s.Lowercase { + val = strings.ToLower(val) + } + + // Add the null character as a terminator + val += "\x00" + vals = append(vals, []byte(val)) + } + if len(vals) == 0 { + return false, nil, nil + } + return true, vals, nil +} + +func (s *StringSliceFieldIndex) FromArgs(args ...interface{}) ([]byte, error) { + if len(args) != 1 { + return nil, fmt.Errorf("must provide only a single argument") + } + arg, ok := args[0].(string) + if !ok { + return nil, fmt.Errorf("argument must be a string: %#v", args[0]) + } + if s.Lowercase { + arg = strings.ToLower(arg) + } + // Add the null character as a terminator + arg += "\x00" + return []byte(arg), nil +} + +func (s *StringSliceFieldIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) { + val, err := s.FromArgs(args...) + if err != nil { + return nil, err + } + + // Strip the null terminator, the rest is a prefix + n := len(val) + if n > 0 { + return val[:n-1], nil + } + return val, nil +} + // UUIDFieldIndex is used to extract a field from an object // using reflection and builds an index on that field by treating // it as a UUID. This is an optimization to using a StringFieldIndex @@ -270,7 +355,11 @@ type CompoundIndex struct { func (c *CompoundIndex) FromObject(raw interface{}) (bool, []byte, error) { var out []byte - for i, idx := range c.Indexes { + for i, idxRaw := range c.Indexes { + idx, ok := idxRaw.(SingleIndexer) + if !ok { + return false, nil, fmt.Errorf("sub-index %d error: %s", i, "sub-index must be a SingleIndexer") + } ok, val, err := idx.FromObject(raw) if err != nil { return false, nil, fmt.Errorf("sub-index %d error: %v", i, err) @@ -291,6 +380,10 @@ func (c *CompoundIndex) FromArgs(args ...interface{}) ([]byte, error) { if len(args) != len(c.Indexes) { return nil, fmt.Errorf("less arguments than index fields") } + return c.PrefixFromArgs(args...) +} + +func (c *CompoundIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) { var out []byte for i, arg := range args { val, err := c.Indexes[i].FromArgs(arg) @@ -301,30 +394,3 @@ func (c *CompoundIndex) FromArgs(args ...interface{}) ([]byte, error) { } return out, nil } - -func (c *CompoundIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) { - if len(args) > len(c.Indexes) { - return nil, fmt.Errorf("more arguments than index fields") - } - var out []byte - for i, arg := range args { - if i+1 < len(args) { - val, err := c.Indexes[i].FromArgs(arg) - if err != nil { - return nil, fmt.Errorf("sub-index %d error: %v", i, err) - } - out = append(out, val...) - } else { - prefixIndexer, ok := c.Indexes[i].(PrefixIndexer) - if !ok { - return nil, fmt.Errorf("sub-index %d does not support prefix scanning", i) - } - val, err := prefixIndexer.PrefixFromArgs(arg) - if err != nil { - return nil, fmt.Errorf("sub-index %d error: %v", i, err) - } - out = append(out, val...) - } - } - return out, nil -} diff --git a/vendor/github.com/hashicorp/go-memdb/schema.go b/vendor/github.com/hashicorp/go-memdb/schema.go index 2b8ffb4760..26d0fcb99f 100644 --- a/vendor/github.com/hashicorp/go-memdb/schema.go +++ b/vendor/github.com/hashicorp/go-memdb/schema.go @@ -46,6 +46,9 @@ func (s *TableSchema) Validate() error { if !s.Indexes["id"].Unique { return fmt.Errorf("id index must be unique") } + if _, ok := s.Indexes["id"].Indexer.(SingleIndexer); !ok { + return fmt.Errorf("id index must be a SingleIndexer") + } for name, index := range s.Indexes { if name != index.Name { return fmt.Errorf("index name mis-match for '%s'", name) @@ -72,5 +75,11 @@ func (s *IndexSchema) Validate() error { if s.Indexer == nil { return fmt.Errorf("missing index function for '%s'", s.Name) } + switch s.Indexer.(type) { + case SingleIndexer: + case MultiIndexer: + default: + return fmt.Errorf("indexer for '%s' must be a SingleIndexer or MultiIndexer", s.Name) + } return nil } diff --git a/vendor/github.com/hashicorp/go-memdb/txn.go b/vendor/github.com/hashicorp/go-memdb/txn.go index 6228677dac..ffc66ba4f9 100644 --- a/vendor/github.com/hashicorp/go-memdb/txn.go +++ b/vendor/github.com/hashicorp/go-memdb/txn.go @@ -148,7 +148,8 @@ func (txn *Txn) Insert(table string, obj interface{}) error { // Get the primary ID of the object idSchema := tableSchema.Indexes[id] - ok, idVal, err := idSchema.Indexer.FromObject(obj) + idIndexer := idSchema.Indexer.(SingleIndexer) + ok, idVal, err := idIndexer.FromObject(obj) if err != nil { return fmt.Errorf("failed to build primary index: %v", err) } @@ -167,7 +168,19 @@ func (txn *Txn) Insert(table string, obj interface{}) error { indexTxn := txn.writableIndex(table, name) // Determine the new index value - ok, val, err := indexSchema.Indexer.FromObject(obj) + var ( + ok bool + vals [][]byte + err error + ) + switch indexer := indexSchema.Indexer.(type) { + case SingleIndexer: + var val []byte + ok, val, err = indexer.FromObject(obj) + vals = [][]byte{val} + case MultiIndexer: + ok, vals, err = indexer.FromObject(obj) + } if err != nil { return fmt.Errorf("failed to build index '%s': %v", name, err) } @@ -176,28 +189,44 @@ func (txn *Txn) Insert(table string, obj interface{}) error { // This is done by appending the primary key which must // be unique anyways. if ok && !indexSchema.Unique { - val = append(val, idVal...) + for i := range vals { + vals[i] = append(vals[i], idVal...) + } } // Handle the update by deleting from the index first if update { - okExist, valExist, err := indexSchema.Indexer.FromObject(existing) + var ( + okExist bool + valsExist [][]byte + err error + ) + switch indexer := indexSchema.Indexer.(type) { + case SingleIndexer: + var valExist []byte + okExist, valExist, err = indexer.FromObject(existing) + valsExist = [][]byte{valExist} + case MultiIndexer: + okExist, valsExist, err = indexer.FromObject(existing) + } if err != nil { return fmt.Errorf("failed to build index '%s': %v", name, err) } if okExist { - // Handle non-unique index by computing a unique index. - // This is done by appending the primary key which must - // be unique anyways. - if !indexSchema.Unique { - valExist = append(valExist, idVal...) - } + for i, valExist := range valsExist { + // Handle non-unique index by computing a unique index. + // This is done by appending the primary key which must + // be unique anyways. + if !indexSchema.Unique { + valExist = append(valExist, idVal...) + } - // If we are writing to the same index with the same value, - // we can avoid the delete as the insert will overwrite the - // value anyways. - if !bytes.Equal(valExist, val) { - indexTxn.Delete(valExist) + // If we are writing to the same index with the same value, + // we can avoid the delete as the insert will overwrite the + // value anyways. + if i >= len(vals) || !bytes.Equal(valExist, vals[i]) { + indexTxn.Delete(valExist) + } } } } @@ -213,7 +242,9 @@ func (txn *Txn) Insert(table string, obj interface{}) error { } // Update the value of the index - indexTxn.Insert(val, obj) + for _, val := range vals { + indexTxn.Insert(val, obj) + } } return nil } @@ -233,7 +264,8 @@ func (txn *Txn) Delete(table string, obj interface{}) error { // Get the primary ID of the object idSchema := tableSchema.Indexes[id] - ok, idVal, err := idSchema.Indexer.FromObject(obj) + idIndexer := idSchema.Indexer.(SingleIndexer) + ok, idVal, err := idIndexer.FromObject(obj) if err != nil { return fmt.Errorf("failed to build primary index: %v", err) } @@ -253,7 +285,19 @@ func (txn *Txn) Delete(table string, obj interface{}) error { indexTxn := txn.writableIndex(table, name) // Handle the update by deleting from the index first - ok, val, err := indexSchema.Indexer.FromObject(existing) + var ( + ok bool + vals [][]byte + err error + ) + switch indexer := indexSchema.Indexer.(type) { + case SingleIndexer: + var val []byte + ok, val, err = indexer.FromObject(existing) + vals = [][]byte{val} + case MultiIndexer: + ok, vals, err = indexer.FromObject(existing) + } if err != nil { return fmt.Errorf("failed to build index '%s': %v", name, err) } @@ -261,10 +305,12 @@ func (txn *Txn) Delete(table string, obj interface{}) error { // Handle non-unique index by computing a unique index. // This is done by appending the primary key which must // be unique anyways. - if !indexSchema.Unique { - val = append(val, idVal...) + for _, val := range vals { + if !indexSchema.Unique { + val = append(val, idVal...) + } + indexTxn.Delete(val) } - indexTxn.Delete(val) } } return nil @@ -334,39 +380,6 @@ func (txn *Txn) First(table, index string, args ...interface{}) (interface{}, er return value, nil } -// LongestPrefix is used to fetch the longest prefix match for the given -// constraints on the index. Note that this will not work with the memdb -// StringFieldIndex because it adds null terminators which prevent the -// algorithm from correctly finding a match (it will get to right before the -// null and fail to find a leaf node). This should only be used where the prefix -// given is capable of matching indexed entries directly, which typically only -// applies to a custom indexer. See the unit test for an example. -func (txn *Txn) LongestPrefix(table, index string, args ...interface{}) (interface{}, error) { - // Enforce that this only works on prefix indexes. - if !strings.HasSuffix(index, "_prefix") { - return nil, fmt.Errorf("must use '%s_prefix' on index", index) - } - - // Get the index value. - indexSchema, val, err := txn.getIndexValue(table, index, args...) - if err != nil { - return nil, err - } - - // This algorithm only makes sense against a unique index, otherwise the - // index keys will have the IDs appended to them. - if !indexSchema.Unique { - return nil, fmt.Errorf("index '%s' is not unique", index) - } - - // Find the longest prefix match with the given index. - indexTxn := txn.readableIndex(table, indexSchema.Name) - if _, value, ok := indexTxn.Root().LongestPrefix(val); ok { - return value, nil - } - return nil, nil -} - // getIndexValue is used to get the IndexSchema and the value // used to scan the index given the parameters. This handles prefix based // scans when the index has the "_prefix" suffix. The index must support