kotovalexarian-likes-github
/
moby--moby
mirror of https://github.com/moby/moby.git
1
0
Fork 0
Code Releases Activity

Merge pull request #43624 from rumpl/feat-memdb-prefix 

Use hashicorp/go-memdb instead of truncindex
This commit is contained in:
Sebastiaan van Stijn 2022-07-08 11:09:20 +02:00 committed by GitHub
parent 755dd793b6 70dc392bfa
commit 5daceee6ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 375 additions and 1608 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
container/view.go
View File

@ -17,6 +17,7 @@ const (
memdbContainersTable = "containers"
memdbNamesTable = "names"
memdbIDIndex = "id"
memdbIDIndexPrefix = "id_prefix"
memdbContainerIDIndex = "containerid"
)
@ -27,6 +28,24 @@ var (
ErrNameNotReserved = errors.New("name is not reserved")
)
var (
// ErrEmptyPrefix is an error returned if the prefix was empty.
ErrEmptyPrefix = errors.New("Prefix can't be empty")
// ErrNotExist is returned when ID or its prefix not found in index.
ErrNotExist = errors.New("ID does not exist")
)
// ErrAmbiguousPrefix is returned if the prefix was ambiguous
// (multiple ids for the prefix).
type ErrAmbiguousPrefix struct {
prefix string
}
func (e ErrAmbiguousPrefix) Error() string {
return fmt.Sprintf("Multiple IDs found with provided prefix: %s", e.prefix)
}
// Snapshot is a read only view for Containers. It holds all information necessary to serve container queries in a
// versioned ACID in-memory store.
type Snapshot struct {
@ -64,6 +83,8 @@ type ViewDB interface {
Save(*Container) error
Delete(*Container) error
GetByPrefix(string) (string, error)
ReserveName(name, containerID string) error
ReleaseName(name string) error
}
@ -131,6 +152,38 @@ func NewViewDB() (ViewDB, error) {
return &memDB{store: store}, nil
}
func (db *memDB) GetByPrefix(s string) (string, error) {
if s == "" {
return "", ErrEmptyPrefix
}
txn := db.store.Txn(false)
iter, err := txn.Get(memdbContainersTable, memdbIDIndexPrefix, s)
if err != nil {
return "", err
}
var (
id string
)
for {
item := iter.Next()
if item == nil {
break
}
if id != "" {
return "", ErrAmbiguousPrefix{prefix: s}
}
id = item.(*Container).ID
}
if id != "" {
return id, nil
}
return "", ErrNotExist
}
// Snapshot provides a consistent read-only View of the database
func (db *memDB) Snapshot() View {
return &memdbView{
@ -441,6 +494,20 @@ func (e *containerByIDIndexer) FromArgs(args ...interface{}) ([]byte, error) {
return []byte(arg), nil
}
func (e *containerByIDIndexer) PrefixFromArgs(args ...interface{}) ([]byte, error) {
val, err := e.FromArgs(args...)
if err != nil {
return nil, err
}
// Strip the null terminator, the rest is a prefix
n := len(val)
if n > 0 {
return val[:n-1], nil
}
return val, nil
}
// namesByNameIndexer is used to index container name associations by name.
type namesByNameIndexer struct{}

299
container/view_test.go
View File

@ -1,12 +1,14 @@
package container // import "github.com/docker/docker/container"
import (
"math/rand"
"os"
"path/filepath"
"testing"
"github.com/docker/docker/api/types"
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/pkg/stringid"
"github.com/google/uuid"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
@ -183,3 +185,300 @@ func TestViewWithHealthCheck(t *testing.T) {
t.Fatalf("expected Health=starting. Got: %+v", s)
}
}
func TestTruncIndex(t *testing.T) {
db, err := NewViewDB()
if err != nil {
t.Fatal(err)
}
// Get on an empty index
if _, err := db.GetByPrefix("foobar"); err == nil {
t.Fatal("Get on an empty index should return an error")
}
id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
// Add an id
if err := db.Save(&Container{ID: id}); err != nil {
t.Fatal(err)
}
type testacase struct {
name string
input string
expectedResult string
expectError bool
}
for _, tc := range []testacase{
{
name: "Get a non-existing id",
input: "abracadabra",
expectError: true,
},
{
name: "Get an empty id",
input: "",
expectedResult: "",
expectError: true,
},
{
name: "Get the exact id",
input: id,
expectedResult: id,
expectError: false,
},
{
name: "The first letter should match",
input: id[:1],
expectedResult: id,
expectError: false,
},
{
name: "The first half should match",
input: id[:len(id)/2],
expectedResult: id,
expectError: false,
},
{
name: "The second half should NOT match",
input: id[len(id)/2:],
expectError: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
assertIndexGet(t, db, tc.input, tc.expectedResult, tc.expectError)
})
}
id2 := id[:6] + "blabla"
// Add an id
if err := db.Save(&Container{ID: id2}); err != nil {
t.Fatal(err)
}
for _, tc := range []testacase{
{
name: "id should work",
input: id,
expectedResult: id,
expectError: false,
},
{
name: "id2 should work",
input: id2,
expectedResult: id2,
expectError: false,
},
{
name: "6 characters should conflict",
input: id[:6],
expectedResult: "",
expectError: true,
},
{
name: "4 characters should conflict",
input: id[:4],
expectedResult: "",
expectError: true,
},
{
name: "1 character should conflict",
input: id[:6],
expectedResult: "",
expectError: true,
},
{
name: "7 characters of id should not conflict",
input: id[:7],
expectedResult: id,
expectError: false,
},
{
name: "7 characters of id2 should not conflict",
input: id2[:7],
expectedResult: id2,
expectError: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
assertIndexGet(t, db, tc.input, tc.expectedResult, tc.expectError)
})
}
// Deleting id2 should remove conflicts
if err := db.Delete(&Container{ID: id2}); err != nil {
t.Fatal(err)
}
for _, tc := range []testacase{
{
name: "id2 should no longer work",
input: id2,
expectedResult: "",
expectError: true,
},
{
name: "7 characters id2 should no longer work",
input: id2[:7],
expectedResult: "",
expectError: true,
},
{
name: "11 characters id2 should no longer work",
input: id2[:11],
expectedResult: "",
expectError: true,
},
{
name: "conflicts between id[:6] and id2 should be gone",
input: id[:6],
expectedResult: id,
expectError: false,
},
{
name: "conflicts between id[:4] and id2 should be gone",
input: id[:4],
expectedResult: id,
expectError: false,
},
{
name: "conflicts between id[:1] and id2 should be gone",
input: id[:1],
expectedResult: id,
expectError: false,
},
{
name: "non-conflicting 7 character substrings should still not conflict",
input: id[:7],
expectedResult: id,
expectError: false,
},
{
name: "non-conflicting 15 character substrings should still not conflict",
input: id[:15],
expectedResult: id,
expectError: false,
},
{
name: "non-conflicting substrings should still not conflict",
input: id,
expectedResult: id,
expectError: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
assertIndexGet(t, db, tc.input, tc.expectedResult, tc.expectError)
})
}
}
func assertIndexGet(t *testing.T, snapshot ViewDB, input, expectedResult string, expectError bool) {
if result, err := snapshot.GetByPrefix(input); err != nil && !expectError {
t.Fatalf("Unexpected error getting '%s': %s", input, err)
} else if err == nil && expectError {
t.Fatalf("Getting '%s' should return an error, not '%s'", input, result)
} else if result != expectedResult {
t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
}
}
func BenchmarkDBAdd100(b *testing.B) {
var testSet []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, err := NewViewDB()
if err != nil {
b.Fatal(err)
}
for _, id := range testSet {
if err := db.Save(&Container{ID: id}); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkDBGetByPrefix100(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
db, err := NewViewDB()
if err != nil {
b.Fatal(err)
}
for _, id := range testSet {
if err := db.Save(&Container{ID: id}); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := db.GetByPrefix(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkDBGetByPrefix250(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 250; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
db, err := NewViewDB()
if err != nil {
b.Fatal(err)
}
for _, id := range testSet {
if err := db.Save(&Container{ID: id}); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := db.GetByPrefix(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkDBGetByPrefix500(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 500; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
db, err := NewViewDB()
if err != nil {
b.Fatal(err)
}
for _, id := range testSet {
if err := db.Save(&Container{ID: id}); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := db.GetByPrefix(id); err != nil {
b.Fatal(res, err)
}
}
}
}

6
daemon/container.go
View File

@ -16,7 +16,6 @@ import (
"github.com/docker/docker/oci/caps"
"github.com/docker/docker/opts"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/runconfig"
volumemounts "github.com/docker/docker/volume/mounts"
"github.com/docker/go-connections/nat"
@ -49,10 +48,10 @@ func (daemon *Daemon) GetContainer(prefixOrName string) (*container.Container, e
return containerByName, nil
}
containerID, indexError := daemon.idIndex.Get(prefixOrName)
containerID, indexError := daemon.containersReplica.GetByPrefix(prefixOrName)
if indexError != nil {
// When truncindex defines an error type, use that instead
if indexError == truncindex.ErrNotExist {
if indexError == container.ErrNotExist {
return nil, containerNotFound(prefixOrName)
}
return nil, errdefs.System(indexError)
@ -119,7 +118,6 @@ func (daemon *Daemon) Register(c *container.Container) error {
defer c.Unlock()
daemon.containers.Add(c.ID, c)
daemon.idIndex.Add(c.ID)
return c.CheckpointTo(daemon.containersReplica)
}

3
daemon/daemon.go
View File

@ -49,7 +49,6 @@ import (
"github.com/docker/docker/pkg/plugingetter"
"github.com/docker/docker/pkg/sysinfo"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/plugin"
pluginexec "github.com/docker/docker/plugin/executor/containerd"
refstore "github.com/docker/docker/reference"
@ -81,7 +80,6 @@ type Daemon struct {
containersReplica container.ViewDB
execCommands *exec.Store
imageService *images.ImageService
idIndex *truncindex.TruncIndex
configStore *config.Config
statsCollector *stats.Collector
defaultLogConfig containertypes.LogConfig
@ -1037,7 +1035,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
return nil, err
}
d.execCommands = exec.NewStore()
d.idIndex = truncindex.NewTruncIndex([]string{})
d.statsCollector = d.newStatsCollector(1 * time.Second)
d.EventsService = events.New()

15
daemon/daemon_test.go
View File

@ -11,7 +11,6 @@ import (
"github.com/docker/docker/errdefs"
"github.com/docker/docker/libnetwork"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/truncindex"
volumesservice "github.com/docker/docker/volume/service"
"github.com/docker/go-connections/nat"
"github.com/pkg/errors"
@ -56,22 +55,20 @@ func TestGetContainer(t *testing.T) {
store.Add(c4.ID, c4)
store.Add(c5.ID, c5)
index := truncindex.NewTruncIndex([]string{})
index.Add(c1.ID)
index.Add(c2.ID)
index.Add(c3.ID)
index.Add(c4.ID)
index.Add(c5.ID)
containersReplica, err := container.NewViewDB()
if err != nil {
t.Fatalf("could not create ViewDB: %v", err)
}
containersReplica.Save(c1)
containersReplica.Save(c2)
containersReplica.Save(c3)
containersReplica.Save(c4)
containersReplica.Save(c5)
daemon := &Daemon{
containers: store,
containersReplica: containersReplica,
idIndex: index,
}
daemon.reserveName(c1.ID, c1.Name)

1
daemon/delete.go
View File

@ -146,7 +146,6 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty
linkNames := daemon.linkIndex.delete(container)
selinux.ReleaseLabel(container.ProcessLabel)
daemon.idIndex.Delete(container.ID)
daemon.containers.Delete(container.ID)
daemon.containersReplica.Delete(container)
if err := daemon.removeMountPoints(container, config.RemoveVolume); err != nil {

2
daemon/list.go
View File

@ -130,7 +130,7 @@ func (daemon *Daemon) filterByNameIDMatches(view container.View, ctx *listContex
matches := make(map[string]bool)
// find ID matches; errors represent "not found" and can be ignored
for _, id := range ids {
if fullID, err := daemon.idIndex.Get(id); err == nil {
if fullID, err := daemon.containersReplica.GetByPrefix(id); err == nil {
matches[fullID] = true
}
}

139
pkg/truncindex/truncindex.go
View File

@ -1,139 +0,0 @@
// Package truncindex provides a general 'index tree', used by Docker
// in order to be able to reference containers by only a few unambiguous
// characters of their id.
package truncindex // import "github.com/docker/docker/pkg/truncindex"
import (
"errors"
"fmt"
"strings"
"sync"
"github.com/tchap/go-patricia/patricia"
)
var (
// ErrEmptyPrefix is an error returned if the prefix was empty.
ErrEmptyPrefix = errors.New("Prefix can't be empty")
// ErrIllegalChar is returned when a space is in the ID
ErrIllegalChar = errors.New("illegal character: ' '")
// ErrNotExist is returned when ID or its prefix not found in index.
ErrNotExist = errors.New("ID does not exist")
)
// ErrAmbiguousPrefix is returned if the prefix was ambiguous
// (multiple ids for the prefix).
type ErrAmbiguousPrefix struct {
prefix string
}
func (e ErrAmbiguousPrefix) Error() string {
return fmt.Sprintf("Multiple IDs found with provided prefix: %s", e.prefix)
}
// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
type TruncIndex struct {
sync.RWMutex
trie *patricia.Trie
ids map[string]struct{}
}
// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs.
func NewTruncIndex(ids []string) (idx *TruncIndex) {
idx = &TruncIndex{
ids: make(map[string]struct{}),
// Change patricia max prefix per node length,
// because our len(ID) always 64
trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)),
}
for _, id := range ids {
idx.addID(id)
}
return
}
func (idx *TruncIndex) addID(id string) error {
if strings.Contains(id, " ") {
return ErrIllegalChar
}
if id == "" {
return ErrEmptyPrefix
}
if _, exists := idx.ids[id]; exists {
return fmt.Errorf("id already exists: '%s'", id)
}
idx.ids[id] = struct{}{}
if inserted := idx.trie.Insert(patricia.Prefix(id), struct{}{}); !inserted {
return fmt.Errorf("failed to insert id: %s", id)
}
return nil
}
// Add adds a new ID to the TruncIndex.
func (idx *TruncIndex) Add(id string) error {
idx.Lock()
defer idx.Unlock()
return idx.addID(id)
}
// Delete removes an ID from the TruncIndex. If there are multiple IDs
// with the given prefix, an error is thrown.
func (idx *TruncIndex) Delete(id string) error {
idx.Lock()
defer idx.Unlock()
if _, exists := idx.ids[id]; !exists || id == "" {
return fmt.Errorf("no such id: '%s'", id)
}
delete(idx.ids, id)
if deleted := idx.trie.Delete(patricia.Prefix(id)); !deleted {
return fmt.Errorf("no such id: '%s'", id)
}
return nil
}
// Get retrieves an ID from the TruncIndex. If there are multiple IDs
// with the given prefix, an error is thrown.
func (idx *TruncIndex) Get(s string) (string, error) {
if s == "" {
return "", ErrEmptyPrefix
}
var (
id string
)
subTreeVisitFunc := func(prefix patricia.Prefix, item patricia.Item) error {
if id != "" {
// we haven't found the ID if there are two or more IDs
id = ""
return ErrAmbiguousPrefix{prefix: s}
}
id = string(prefix)
return nil
}
idx.RLock()
defer idx.RUnlock()
if err := idx.trie.VisitSubtree(patricia.Prefix(s), subTreeVisitFunc); err != nil {
return "", err
}
if id != "" {
return id, nil
}
return "", ErrNotExist
}
// Iterate iterates over all stored IDs and passes each of them to the given
// handler. Take care that the handler method does not call any public
// method on truncindex as the internal locking is not reentrant/recursive
// and will result in deadlock.
func (idx *TruncIndex) Iterate(handler func(id string)) {
idx.Lock()
defer idx.Unlock()
idx.trie.Visit(func(prefix patricia.Prefix, item patricia.Item) error {
handler(string(prefix))
return nil
})
}

453
pkg/truncindex/truncindex_test.go
View File

@ -1,453 +0,0 @@
package truncindex // import "github.com/docker/docker/pkg/truncindex"
import (
"math/rand"
"testing"
"time"
"github.com/docker/docker/pkg/stringid"
)
// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
func TestTruncIndex(t *testing.T) {
var ids []string
index := NewTruncIndex(ids)
// Get on an empty index
if _, err := index.Get("foobar"); err == nil {
t.Fatal("Get on an empty index should return an error")
}
// Spaces should be illegal in an id
if err := index.Add("I have a space"); err == nil {
t.Fatalf("Adding an id with ' ' should return an error")
}
id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
// Add an id
if err := index.Add(id); err != nil {
t.Fatal(err)
}
// Add an empty id (should fail)
if err := index.Add(""); err == nil {
t.Fatalf("Adding an empty id should return an error")
}
// Get a non-existing id
assertIndexGet(t, index, "abracadabra", "", true)
// Get an empty id
assertIndexGet(t, index, "", "", true)
// Get the exact id
assertIndexGet(t, index, id, id, false)
// The first letter should match
assertIndexGet(t, index, id[:1], id, false)
// The first half should match
assertIndexGet(t, index, id[:len(id)/2], id, false)
// The second half should NOT match
assertIndexGet(t, index, id[len(id)/2:], "", true)
id2 := id[:6] + "blabla"
// Add an id
if err := index.Add(id2); err != nil {
t.Fatal(err)
}
// Both exact IDs should work
assertIndexGet(t, index, id, id, false)
assertIndexGet(t, index, id2, id2, false)
// 6 characters or less should conflict
assertIndexGet(t, index, id[:6], "", true)
assertIndexGet(t, index, id[:4], "", true)
assertIndexGet(t, index, id[:1], "", true)
// An ambiguous id prefix should return an error
if _, err := index.Get(id[:4]); err == nil {
t.Fatal("An ambiguous id prefix should return an error")
}
// 7 characters should NOT conflict
assertIndexGet(t, index, id[:7], id, false)
assertIndexGet(t, index, id2[:7], id2, false)
// Deleting a non-existing id should return an error
if err := index.Delete("non-existing"); err == nil {
t.Fatalf("Deleting a non-existing id should return an error")
}
// Deleting an empty id should return an error
if err := index.Delete(""); err == nil {
t.Fatal("Deleting an empty id should return an error")
}
// Deleting id2 should remove conflicts
if err := index.Delete(id2); err != nil {
t.Fatal(err)
}
// id2 should no longer work
assertIndexGet(t, index, id2, "", true)
assertIndexGet(t, index, id2[:7], "", true)
assertIndexGet(t, index, id2[:11], "", true)
// conflicts between id and id2 should be gone
assertIndexGet(t, index, id[:6], id, false)
assertIndexGet(t, index, id[:4], id, false)
assertIndexGet(t, index, id[:1], id, false)
// non-conflicting substrings should still not conflict
assertIndexGet(t, index, id[:7], id, false)
assertIndexGet(t, index, id[:15], id, false)
assertIndexGet(t, index, id, id, false)
assertIndexIterate(t)
assertIndexIterateDoNotPanic(t)
}
func assertIndexIterate(t *testing.T) {
ids := []string{
"19b36c2c326ccc11e726eee6ee78a0baf166ef96",
"28b36c2c326ccc11e726eee6ee78a0baf166ef96",
"37b36c2c326ccc11e726eee6ee78a0baf166ef96",
"46b36c2c326ccc11e726eee6ee78a0baf166ef96",
}
index := NewTruncIndex(ids)
index.Iterate(func(targetId string) {
for _, id := range ids {
if targetId == id {
return
}
}
t.Fatalf("An unknown ID '%s'", targetId)
})
}
func assertIndexIterateDoNotPanic(t *testing.T) {
ids := []string{
"19b36c2c326ccc11e726eee6ee78a0baf166ef96",
"28b36c2c326ccc11e726eee6ee78a0baf166ef96",
}
index := NewTruncIndex(ids)
iterationStarted := make(chan bool, 1)
go func() {
<-iterationStarted
index.Delete("19b36c2c326ccc11e726eee6ee78a0baf166ef96")
}()
index.Iterate(func(targetId string) {
if targetId == "19b36c2c326ccc11e726eee6ee78a0baf166ef96" {
iterationStarted <- true
time.Sleep(100 * time.Millisecond)
}
})
}
func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
if result, err := index.Get(input); err != nil && !expectError {
t.Fatalf("Unexpected error getting '%s': %s", input, err)
} else if err == nil && expectError {
t.Fatalf("Getting '%s' should return an error, not '%s'", input, result)
} else if result != expectedResult {
t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
}
}
func BenchmarkTruncIndexAdd100(b *testing.B) {
var testSet []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexAdd250(b *testing.B) {
var testSet []string
for i := 0; i < 250; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexAdd500(b *testing.B) {
var testSet []string
for i := 0; i < 500; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexGet100(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkTruncIndexGet250(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 250; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkTruncIndexGet500(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 500; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkTruncIndexDelete100(b *testing.B) {
var testSet []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
b.StartTimer()
for _, id := range testSet {
if err := index.Delete(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexDelete250(b *testing.B) {
var testSet []string
for i := 0; i < 250; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
b.StartTimer()
for _, id := range testSet {
if err := index.Delete(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexDelete500(b *testing.B) {
var testSet []string
for i := 0; i < 500; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
b.StartTimer()
for _, id := range testSet {
if err := index.Delete(id); err != nil {
b.Fatal(err)
}
}
}
}
func BenchmarkTruncIndexNew100(b *testing.B) {
var testSet []string
for i := 0; i < 100; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewTruncIndex(testSet)
}
}
func BenchmarkTruncIndexNew250(b *testing.B) {
var testSet []string
for i := 0; i < 250; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewTruncIndex(testSet)
}
}
func BenchmarkTruncIndexNew500(b *testing.B) {
var testSet []string
for i := 0; i < 500; i++ {
testSet = append(testSet, stringid.GenerateRandomID())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewTruncIndex(testSet)
}
}
func BenchmarkTruncIndexAddGet100(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 500; i++ {
id := stringid.GenerateRandomID()
testSet = append(testSet, id)
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkTruncIndexAddGet250(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 500; i++ {
id := stringid.GenerateRandomID()
testSet = append(testSet, id)
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}
func BenchmarkTruncIndexAddGet500(b *testing.B) {
var testSet []string
var testKeys []string
for i := 0; i < 500; i++ {
id := stringid.GenerateRandomID()
testSet = append(testSet, id)
l := rand.Intn(12) + 12
testKeys = append(testKeys, id[:l])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
index := NewTruncIndex([]string{})
for _, id := range testSet {
if err := index.Add(id); err != nil {
b.Fatal(err)
}
}
for _, id := range testKeys {
if res, err := index.Get(id); err != nil {
b.Fatal(res, err)
}
}
}
}

1
vendor.mod
View File

@ -71,7 +71,6 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/tchap/go-patricia v2.3.0+incompatible
github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274
github.com/tonistiigi/go-archvariant v1.0.0
github.com/vbatts/tar-split v0.11.2

2
vendor.sum
View File

@ -997,8 +997,6 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs=
github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/thecodeteam/gosync v0.1.0/go.mod h1:43QHsngcnWc8GE1aCmi7PEypslflHjCzXFleuWKEb00=
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=

3
vendor/github.com/tchap/go-patricia/AUTHORS generated vendored
View File

@ -1,3 +0,0 @@
This is the complete list of go-patricia copyright holders:
Ondřej Kupka <ondra.cap@gmail.com>

20
vendor/github.com/tchap/go-patricia/LICENSE generated vendored
View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2014 The AUTHORS
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

363
vendor/github.com/tchap/go-patricia/patricia/children.go generated vendored
View File

@ -1,363 +0,0 @@
// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.
package patricia
import (
"fmt"
"io"
"sort"
)
type childList interface {
length() int
head() *Trie
add(child *Trie) childList
remove(b byte)
replace(b byte, child *Trie)
next(b byte) *Trie
walk(prefix *Prefix, visitor VisitorFunc) error
print(w io.Writer, indent int)
clone() childList
total() int
}
type tries []*Trie
func (t tries) Len() int {
return len(t)
}
func (t tries) Less(i, j int) bool {
strings := sort.StringSlice{string(t[i].prefix), string(t[j].prefix)}
return strings.Less(0, 1)
}
func (t tries) Swap(i, j int) {
t[i], t[j] = t[j], t[i]
}
type sparseChildList struct {
children tries
}
func newSparseChildList(maxChildrenPerSparseNode int) childList {
return &sparseChildList{
children: make(tries, 0, maxChildrenPerSparseNode),
}
}
func (list *sparseChildList) length() int {
return len(list.children)
}
func (list *sparseChildList) head() *Trie {
return list.children[0]
}
func (list *sparseChildList) add(child *Trie) childList {
// Search for an empty spot and insert the child if possible.
if len(list.children) != cap(list.children) {
list.children = append(list.children, child)
return list
}
// Otherwise we have to transform to the dense list type.
return newDenseChildList(list, child)
}
func (list *sparseChildList) remove(b byte) {
for i, node := range list.children {
if node.prefix[0] == b {
list.children[i] = list.children[len(list.children)-1]
list.children[len(list.children)-1] = nil
list.children = list.children[:len(list.children)-1]
return
}
}
// This is not supposed to be reached.
panic("removing non-existent child")
}
func (list *sparseChildList) replace(b byte, child *Trie) {
// Make a consistency check.
if p0 := child.prefix[0]; p0 != b {
panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
}
// Seek the child and replace it.
for i, node := range list.children {
if node.prefix[0] == b {
list.children[i] = child
return
}
}
}
func (list *sparseChildList) next(b byte) *Trie {
for _, child := range list.children {
if child.prefix[0] == b {
return child
}
}
return nil
}
func (list *sparseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
sort.Sort(list.children)
for _, child := range list.children {
*prefix = append(*prefix, child.prefix...)
if child.item != nil {
err := visitor(*prefix, child.item)
if err != nil {
if err == SkipSubtree {
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
continue
}
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
return err
}
}
err := child.children.walk(prefix, visitor)
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
if err != nil {
return err
}
}
return nil
}
func (list *sparseChildList) total() int {
tot := 0
for _, child := range list.children {
if child != nil {
tot = tot + child.total()
}
}
return tot
}
func (list *sparseChildList) clone() childList {
clones := make(tries, len(list.children), cap(list.children))
for i, child := range list.children {
clones[i] = child.Clone()
}
return &sparseChildList{
children: clones,
}
}
func (list *sparseChildList) print(w io.Writer, indent int) {
for _, child := range list.children {
if child != nil {
child.print(w, indent)
}
}
}
type denseChildList struct {
min int
max int
numChildren int
headIndex int
children []*Trie
}
func newDenseChildList(list *sparseChildList, child *Trie) childList {
var (
min int = 255
max int = 0
)
for _, child := range list.children {
b := int(child.prefix[0])
if b < min {
min = b
}
if b > max {
max = b
}
}
b := int(child.prefix[0])
if b < min {
min = b
}
if b > max {
max = b
}
children := make([]*Trie, max-min+1)
for _, child := range list.children {
children[int(child.prefix[0])-min] = child
}
children[int(child.prefix[0])-min] = child
return &denseChildList{
min: min,
max: max,
numChildren: list.length() + 1,
headIndex: 0,
children: children,
}
}
func (list *denseChildList) length() int {
return list.numChildren
}
func (list *denseChildList) head() *Trie {
return list.children[list.headIndex]
}
func (list *denseChildList) add(child *Trie) childList {
b := int(child.prefix[0])
var i int
switch {
case list.min <= b && b <= list.max:
if list.children[b-list.min] != nil {
panic("dense child list collision detected")
}
i = b - list.min
list.children[i] = child
case b < list.min:
children := make([]*Trie, list.max-b+1)
i = 0
children[i] = child
copy(children[list.min-b:], list.children)
list.children = children
list.min = b
default: // b > list.max
children := make([]*Trie, b-list.min+1)
i = b - list.min
children[i] = child
copy(children, list.children)
list.children = children
list.max = b
}
list.numChildren++
if i < list.headIndex {
list.headIndex = i
}
return list
}
func (list *denseChildList) remove(b byte) {
i := int(b) - list.min
if list.children[i] == nil {
// This is not supposed to be reached.
panic("removing non-existent child")
}
list.numChildren--
list.children[i] = nil
// Update head index.
if i == list.headIndex {
for ; i < len(list.children); i++ {
if list.children[i] != nil {
list.headIndex = i
return
}
}
}
}
func (list *denseChildList) replace(b byte, child *Trie) {
// Make a consistency check.
if p0 := child.prefix[0]; p0 != b {
panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
}
// Replace the child.
list.children[int(b)-list.min] = child
}
func (list *denseChildList) next(b byte) *Trie {
i := int(b)
if i < list.min || list.max < i {
return nil
}
return list.children[i-list.min]
}
func (list *denseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
for _, child := range list.children {
if child == nil {
continue
}
*prefix = append(*prefix, child.prefix...)
if child.item != nil {
if err := visitor(*prefix, child.item); err != nil {
if err == SkipSubtree {
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
continue
}
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
return err
}
}
err := child.children.walk(prefix, visitor)
*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
if err != nil {
return err
}
}
return nil
}
func (list *denseChildList) print(w io.Writer, indent int) {
for _, child := range list.children {
if child != nil {
child.print(w, indent)
}
}
}
func (list *denseChildList) clone() childList {
clones := make(tries, cap(list.children))
if list.numChildren != 0 {
clonedCount := 0
for i := list.headIndex; i < len(list.children); i++ {
child := list.children[i]
if child != nil {
clones[i] = child.Clone()
clonedCount++
if clonedCount == list.numChildren {
break
}
}
}
}
return &denseChildList{
min: list.min,
max: list.max,
numChildren: list.numChildren,
headIndex: list.headIndex,
children: clones,
}
}
func (list *denseChildList) total() int {
tot := 0
for _, child := range list.children {
if child != nil {
tot = tot + child.total()
}
}
return tot
}

606
vendor/github.com/tchap/go-patricia/patricia/patricia.go generated vendored
View File

@ -1,606 +0,0 @@
// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.
package patricia
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
)
//------------------------------------------------------------------------------
// Trie
//------------------------------------------------------------------------------
const (
DefaultMaxPrefixPerNode = 10
DefaultMaxChildrenPerSparseNode = 8
)
type (
Prefix []byte
Item interface{}
VisitorFunc func(prefix Prefix, item Item) error
)
// Trie is a generic patricia trie that allows fast retrieval of items by prefix.
// and other funky stuff.
//
// Trie is not thread-safe.
type Trie struct {
prefix Prefix
item Item
maxPrefixPerNode int
maxChildrenPerSparseNode int
children childList
}
// Public API ------------------------------------------------------------------
type Option func(*Trie)
// Trie constructor.
func NewTrie(options ...Option) *Trie {
trie := &Trie{}
for _, opt := range options {
opt(trie)
}
if trie.maxPrefixPerNode <= 0 {
trie.maxPrefixPerNode = DefaultMaxPrefixPerNode
}
if trie.maxChildrenPerSparseNode <= 0 {
trie.maxChildrenPerSparseNode = DefaultMaxChildrenPerSparseNode
}
trie.children = newSparseChildList(trie.maxChildrenPerSparseNode)
return trie
}
func MaxPrefixPerNode(value int) Option {
return func(trie *Trie) {
trie.maxPrefixPerNode = value
}
}
func MaxChildrenPerSparseNode(value int) Option {
return func(trie *Trie) {
trie.maxChildrenPerSparseNode = value
}
}
// Clone makes a copy of an existing trie.
// Items stored in both tries become shared, obviously.
func (trie *Trie) Clone() *Trie {
return &Trie{
prefix: append(Prefix(nil), trie.prefix...),
item: trie.item,
maxPrefixPerNode: trie.maxPrefixPerNode,
maxChildrenPerSparseNode: trie.maxChildrenPerSparseNode,
children: trie.children.clone(),
}
}
// Item returns the item stored in the root of this trie.
func (trie *Trie) Item() Item {
return trie.item
}
// Insert inserts a new item into the trie using the given prefix. Insert does
// not replace existing items. It returns false if an item was already in place.
func (trie *Trie) Insert(key Prefix, item Item) (inserted bool) {
return trie.put(key, item, false)
}
// Set works much like Insert, but it always sets the item, possibly replacing
// the item previously inserted.
func (trie *Trie) Set(key Prefix, item Item) {
trie.put(key, item, true)
}
// Get returns the item located at key.
//
// This method is a bit dangerous, because Get can as well end up in an internal
// node that is not really representing any user-defined value. So when nil is
// a valid value being used, it is not possible to tell if the value was inserted
// into the tree by the user or not. A possible workaround for this is not to use
// nil interface as a valid value, even using zero value of any type is enough
// to prevent this bad behaviour.
func (trie *Trie) Get(key Prefix) (item Item) {
_, node, found, leftover := trie.findSubtree(key)
if !found || len(leftover) != 0 {
return nil
}
return node.item
}
// Match returns what Get(prefix) != nil would return. The same warning as for
// Get applies here as well.
func (trie *Trie) Match(prefix Prefix) (matchedExactly bool) {
return trie.Get(prefix) != nil
}
// MatchSubtree returns true when there is a subtree representing extensions
// to key, that is if there are any keys in the tree which have key as prefix.
func (trie *Trie) MatchSubtree(key Prefix) (matched bool) {
_, _, matched, _ = trie.findSubtree(key)
return
}
// Visit calls visitor on every node containing a non-nil item
// in alphabetical order.
//
// If an error is returned from visitor, the function stops visiting the tree
// and returns that error, unless it is a special error - SkipSubtree. In that
// case Visit skips the subtree represented by the current node and continues
// elsewhere.
func (trie *Trie) Visit(visitor VisitorFunc) error {
return trie.walk(nil, visitor)
}
func (trie *Trie) size() int {
n := 0
trie.walk(nil, func(prefix Prefix, item Item) error {
n++
return nil
})
return n
}
func (trie *Trie) total() int {
return 1 + trie.children.total()
}
// VisitSubtree works much like Visit, but it only visits nodes matching prefix.
func (trie *Trie) VisitSubtree(prefix Prefix, visitor VisitorFunc) error {
// Nil prefix not allowed.
if prefix == nil {
panic(ErrNilPrefix)
}
// Empty trie must be handled explicitly.
if trie.prefix == nil {
return nil
}
// Locate the relevant subtree.
_, root, found, leftover := trie.findSubtree(prefix)
if !found {
return nil
}
prefix = append(prefix, leftover...)
// Visit it.
return root.walk(prefix, visitor)
}
// VisitPrefixes visits only nodes that represent prefixes of key.
// To say the obvious, returning SkipSubtree from visitor makes no sense here.
func (trie *Trie) VisitPrefixes(key Prefix, visitor VisitorFunc) error {
// Nil key not allowed.
if key == nil {
panic(ErrNilPrefix)
}
// Empty trie must be handled explicitly.
if trie.prefix == nil {
return nil
}
// Walk the path matching key prefixes.
node := trie
prefix := key
offset := 0
for {
// Compute what part of prefix matches.
common := node.longestCommonPrefixLength(key)
key = key[common:]
offset += common
// Partial match means that there is no subtree matching prefix.
if common < len(node.prefix) {
return nil
}
// Call the visitor.
if item := node.item; item != nil {
if err := visitor(prefix[:offset], item); err != nil {
return err
}
}
if len(key) == 0 {
// This node represents key, we are finished.
return nil
}
// There is some key suffix left, move to the children.
child := node.children.next(key[0])
if child == nil {
// There is nowhere to continue, return.
return nil
}
node = child
}
}
// Delete deletes the item represented by the given prefix.
//
// True is returned if the matching node was found and deleted.
func (trie *Trie) Delete(key Prefix) (deleted bool) {
// Nil prefix not allowed.
if key == nil {
panic(ErrNilPrefix)
}
// Empty trie must be handled explicitly.
if trie.prefix == nil {
return false
}
// Find the relevant node.
path, found, _ := trie.findSubtreePath(key)
if !found {
return false
}
node := path[len(path)-1]
var parent *Trie
if len(path) != 1 {
parent = path[len(path)-2]
}
// If the item is already set to nil, there is nothing to do.
if node.item == nil {
return false
}
// Delete the item.
node.item = nil
// Initialise i before goto.
// Will be used later in a loop.
i := len(path) - 1
// In case there are some child nodes, we cannot drop the whole subtree.
// We can try to compact nodes, though.
if node.children.length() != 0 {
goto Compact
}
// In case we are at the root, just reset it and we are done.
if parent == nil {
node.reset()
return true
}
// We can drop a subtree.
// Find the first ancestor that has its value set or it has 2 or more child nodes.
// That will be the node where to drop the subtree at.
for ; i >= 0; i-- {
if current := path[i]; current.item != nil || current.children.length() >= 2 {
break
}
}
// Handle the case when there is no such node.
// In other words, we can reset the whole tree.
if i == -1 {
path[0].reset()
return true
}
// We can just remove the subtree here.
node = path[i]
if i == 0 {
parent = nil
} else {
parent = path[i-1]
}
// i+1 is always a valid index since i is never pointing to the last node.
// The loop above skips at least the last node since we are sure that the item
// is set to nil and it has no children, othewise we would be compacting instead.
node.children.remove(path[i+1].prefix[0])
Compact:
// The node is set to the first non-empty ancestor,
// so try to compact since that might be possible now.
if compacted := node.compact(); compacted != node {
if parent == nil {
*node = *compacted
} else {
parent.children.replace(node.prefix[0], compacted)
*parent = *parent.compact()
}
}
return true
}
// DeleteSubtree finds the subtree exactly matching prefix and deletes it.
//
// True is returned if the subtree was found and deleted.
func (trie *Trie) DeleteSubtree(prefix Prefix) (deleted bool) {
// Nil prefix not allowed.
if prefix == nil {
panic(ErrNilPrefix)
}
// Empty trie must be handled explicitly.
if trie.prefix == nil {
return false
}
// Locate the relevant subtree.
parent, root, found, _ := trie.findSubtree(prefix)
if !found {
return false
}
// If we are in the root of the trie, reset the trie.
if parent == nil {
root.reset()
return true
}
// Otherwise remove the root node from its parent.
parent.children.remove(root.prefix[0])
return true
}
// Internal helper methods -----------------------------------------------------
func (trie *Trie) empty() bool {
return trie.item == nil && trie.children.length() == 0
}
func (trie *Trie) reset() {
trie.prefix = nil
trie.children = newSparseChildList(trie.maxPrefixPerNode)
}
func (trie *Trie) put(key Prefix, item Item, replace bool) (inserted bool) {
// Nil prefix not allowed.
if key == nil {
panic(ErrNilPrefix)
}
var (
common int
node *Trie = trie
child *Trie
)
if node.prefix == nil {
if len(key) <= trie.maxPrefixPerNode {
node.prefix = key
goto InsertItem
}
node.prefix = key[:trie.maxPrefixPerNode]
key = key[trie.maxPrefixPerNode:]
goto AppendChild
}
for {
// Compute the longest common prefix length.
common = node.longestCommonPrefixLength(key)
key = key[common:]
// Only a part matches, split.
if common < len(node.prefix) {
goto SplitPrefix
}
// common == len(node.prefix) since never (common > len(node.prefix))
// common == len(former key) <-> 0 == len(key)
// -> former key == node.prefix
if len(key) == 0 {
goto InsertItem
}
// Check children for matching prefix.
child = node.children.next(key[0])
if child == nil {
goto AppendChild
}
node = child
}
SplitPrefix:
// Split the prefix if necessary.
child = new(Trie)
*child = *node
*node = *NewTrie()
node.prefix = child.prefix[:common]
child.prefix = child.prefix[common:]
child = child.compact()
node.children = node.children.add(child)
AppendChild:
// Keep appending children until whole prefix is inserted.
// This loop starts with empty node.prefix that needs to be filled.
for len(key) != 0 {
child := NewTrie()
if len(key) <= trie.maxPrefixPerNode {
child.prefix = key
node.children = node.children.add(child)
node = child
goto InsertItem
} else {
child.prefix = key[:trie.maxPrefixPerNode]
key = key[trie.maxPrefixPerNode:]
node.children = node.children.add(child)
node = child
}
}
InsertItem:
// Try to insert the item if possible.
if replace || node.item == nil {
node.item = item
return true
}
return false
}
func (trie *Trie) compact() *Trie {
// Only a node with a single child can be compacted.
if trie.children.length() != 1 {
return trie
}
child := trie.children.head()
// If any item is set, we cannot compact since we want to retain
// the ability to do searching by key. This makes compaction less usable,
// but that simply cannot be avoided.
if trie.item != nil || child.item != nil {
return trie
}
// Make sure the combined prefixes fit into a single node.
if len(trie.prefix)+len(child.prefix) > trie.maxPrefixPerNode {
return trie
}
// Concatenate the prefixes, move the items.
child.prefix = append(trie.prefix, child.prefix...)
if trie.item != nil {
child.item = trie.item
}
return child
}
func (trie *Trie) findSubtree(prefix Prefix) (parent *Trie, root *Trie, found bool, leftover Prefix) {
// Find the subtree matching prefix.
root = trie
for {
// Compute what part of prefix matches.
common := root.longestCommonPrefixLength(prefix)
prefix = prefix[common:]
// We used up the whole prefix, subtree found.
if len(prefix) == 0 {
found = true
leftover = root.prefix[common:]
return
}
// Partial match means that there is no subtree matching prefix.
if common < len(root.prefix) {
leftover = root.prefix[common:]
return
}
// There is some prefix left, move to the children.
child := root.children.next(prefix[0])
if child == nil {
// There is nowhere to continue, there is no subtree matching prefix.
return
}
parent = root
root = child
}
}
func (trie *Trie) findSubtreePath(prefix Prefix) (path []*Trie, found bool, leftover Prefix) {
// Find the subtree matching prefix.
root := trie
var subtreePath []*Trie
for {
// Append the current root to the path.
subtreePath = append(subtreePath, root)
// Compute what part of prefix matches.
common := root.longestCommonPrefixLength(prefix)
prefix = prefix[common:]
// We used up the whole prefix, subtree found.
if len(prefix) == 0 {
path = subtreePath
found = true
leftover = root.prefix[common:]
return
}
// Partial match means that there is no subtree matching prefix.
if common < len(root.prefix) {
leftover = root.prefix[common:]
return
}
// There is some prefix left, move to the children.
child := root.children.next(prefix[0])
if child == nil {
// There is nowhere to continue, there is no subtree matching prefix.
return
}
root = child
}
}
func (trie *Trie) walk(actualRootPrefix Prefix, visitor VisitorFunc) error {
var prefix Prefix
// Allocate a bit more space for prefix at the beginning.
if actualRootPrefix == nil {
prefix = make(Prefix, 32+len(trie.prefix))
copy(prefix, trie.prefix)
prefix = prefix[:len(trie.prefix)]
} else {
prefix = make(Prefix, 32+len(actualRootPrefix))
copy(prefix, actualRootPrefix)
prefix = prefix[:len(actualRootPrefix)]
}
// Visit the root first. Not that this works for empty trie as well since
// in that case item == nil && len(children) == 0.
if trie.item != nil {
if err := visitor(prefix, trie.item); err != nil {
if err == SkipSubtree {
return nil
}
return err
}
}
// Then continue to the children.
return trie.children.walk(&prefix, visitor)
}
func (trie *Trie) longestCommonPrefixLength(prefix Prefix) (i int) {
for ; i < len(prefix) && i < len(trie.prefix) && prefix[i] == trie.prefix[i]; i++ {
}
return
}
func (trie *Trie) dump() string {
writer := &bytes.Buffer{}
trie.print(writer, 0)
return writer.String()
}
func (trie *Trie) print(writer io.Writer, indent int) {
fmt.Fprintf(writer, "%s%s %v\n", strings.Repeat(" ", indent), string(trie.prefix), trie.item)
trie.children.print(writer, indent+2)
}
// Errors ----------------------------------------------------------------------
var (
SkipSubtree = errors.New("Skip this subtree")
ErrNilPrefix = errors.New("Nil prefix passed into a method call")
)

3
vendor/modules.txt vendored
View File

@ -746,9 +746,6 @@ github.com/spf13/cobra
# github.com/spf13/pflag v1.0.5
## explicit; go 1.12
github.com/spf13/pflag
# github.com/tchap/go-patricia v2.3.0+incompatible
## explicit
github.com/tchap/go-patricia/patricia
# github.com/tinylib/msgp v1.1.0
## explicit
github.com/tinylib/msgp/msgp