Merge pull request #6648 from LK4D4/truncindex_bench

Truncindex benchmarks

daemon/daemon.go

@@ -31,6 +31,7 @@ import (
 	"github.com/dotcloud/docker/pkg/namesgenerator"
 	"github.com/dotcloud/docker/pkg/networkfs/resolvconf"
 	"github.com/dotcloud/docker/pkg/sysinfo"
+	"github.com/dotcloud/docker/pkg/truncindex"
 	"github.com/dotcloud/docker/runconfig"
 	"github.com/dotcloud/docker/utils"
 )
@@ -87,7 +88,7 @@ type Daemon struct {
 	containers     *contStore
 	graph          *graph.Graph
 	repositories   *graph.TagStore
-	idIndex        *utils.TruncIndex
+	idIndex        *truncindex.TruncIndex
 	sysInfo        *sysinfo.SysInfo
 	volumes        *graph.Graph
 	srv            Server
@@ -870,7 +871,7 @@ func NewDaemonFromDirectory(config *daemonconfig.Config, eng *engine.Engine) (*D
 		containers:     &contStore{s: make(map[string]*Container)},
 		graph:          g,
 		repositories:   repositories,
-		idIndex:        utils.NewTruncIndex([]string{}),
+		idIndex:        truncindex.NewTruncIndex([]string{}),
 		sysInfo:        sysInfo,
 		volumes:        volumes,
 		config:         config,

graph/graph.go

@@ -16,6 +16,7 @@ import (
 	"github.com/dotcloud/docker/daemon/graphdriver"
 	"github.com/dotcloud/docker/dockerversion"
 	"github.com/dotcloud/docker/image"
+	"github.com/dotcloud/docker/pkg/truncindex"
 	"github.com/dotcloud/docker/runconfig"
 	"github.com/dotcloud/docker/utils"
 )
@@ -23,7 +24,7 @@ import (
 // A Graph is a store for versioned filesystem images and the relationship between them.
 type Graph struct {
 	Root    string
-	idIndex *utils.TruncIndex
+	idIndex *truncindex.TruncIndex
 	driver  graphdriver.Driver
 }
 
@@ -41,7 +42,7 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
 
 	graph := &Graph{
 		Root:    abspath,
-		idIndex: utils.NewTruncIndex([]string{}),
+		idIndex: truncindex.NewTruncIndex([]string{}),
 		driver:  driver,
 	}
 	if err := graph.restore(); err != nil {
@@ -62,7 +63,7 @@ func (graph *Graph) restore() error {
 			ids = append(ids, id)
 		}
 	}
-	graph.idIndex = utils.NewTruncIndex(ids)
+	graph.idIndex = truncindex.NewTruncIndex(ids)
 	utils.Debugf("Restored %d elements", len(dir))
 	return nil
 }

pkg/truncindex/truncindex.go

@@ -0,0 +1,102 @@
+package truncindex
+
+import (
+	"fmt"
+	"index/suffixarray"
+	"strings"
+	"sync"
+)
+
+// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
+// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
+type TruncIndex struct {
+	sync.RWMutex
+	index *suffixarray.Index
+	ids   map[string]bool
+	bytes []byte
+}
+
+func NewTruncIndex(ids []string) (idx *TruncIndex) {
+	idx = &TruncIndex{
+		ids:   make(map[string]bool),
+		bytes: []byte{' '},
+	}
+	for _, id := range ids {
+		idx.ids[id] = true
+		idx.bytes = append(idx.bytes, []byte(id+" ")...)
+	}
+	idx.index = suffixarray.New(idx.bytes)
+	return
+}
+
+func (idx *TruncIndex) addId(id string) error {
+	if strings.Contains(id, " ") {
+		return fmt.Errorf("Illegal character: ' '")
+	}
+	if _, exists := idx.ids[id]; exists {
+		return fmt.Errorf("Id already exists: %s", id)
+	}
+	idx.ids[id] = true
+	idx.bytes = append(idx.bytes, []byte(id+" ")...)
+	return nil
+}
+
+func (idx *TruncIndex) Add(id string) error {
+	idx.Lock()
+	defer idx.Unlock()
+	if err := idx.addId(id); err != nil {
+		return err
+	}
+	idx.index = suffixarray.New(idx.bytes)
+	return nil
+}
+
+func (idx *TruncIndex) AddWithoutSuffixarrayUpdate(id string) error {
+	idx.Lock()
+	defer idx.Unlock()
+	return idx.addId(id)
+}
+
+func (idx *TruncIndex) UpdateSuffixarray() {
+	idx.Lock()
+	defer idx.Unlock()
+	idx.index = suffixarray.New(idx.bytes)
+}
+
+func (idx *TruncIndex) Delete(id string) error {
+	idx.Lock()
+	defer idx.Unlock()
+	if _, exists := idx.ids[id]; !exists {
+		return fmt.Errorf("No such id: %s", id)
+	}
+	before, after, err := idx.lookup(id)
+	if err != nil {
+		return err
+	}
+	delete(idx.ids, id)
+	idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
+	idx.index = suffixarray.New(idx.bytes)
+	return nil
+}
+
+func (idx *TruncIndex) lookup(s string) (int, int, error) {
+	offsets := idx.index.Lookup([]byte(" "+s), -1)
+	//log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
+	if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
+		return -1, -1, fmt.Errorf("No such id: %s", s)
+	}
+	offsetBefore := offsets[0] + 1
+	offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
+	return offsetBefore, offsetAfter, nil
+}
+
+func (idx *TruncIndex) Get(s string) (string, error) {
+	idx.RLock()
+	defer idx.RUnlock()
+	before, after, err := idx.lookup(s)
+	//log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
+	if err != nil {
+		return "", err
+	}
+	return string(idx.bytes[before:after]), err
+}

pkg/truncindex/truncindex_test.go

@@ -0,0 +1,244 @@
+package truncindex
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/dotcloud/docker/utils"
+)
+
+// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
+func TestTruncIndex(t *testing.T) {
+	ids := []string{}
+	index := NewTruncIndex(ids)
+	// Get on an empty index
+	if _, err := index.Get("foobar"); err == nil {
+		t.Fatal("Get on an empty index should return an error")
+	}
+
+	// Spaces should be illegal in an id
+	if err := index.Add("I have a space"); err == nil {
+		t.Fatalf("Adding an id with ' ' should return an error")
+	}
+
+	id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
+	// Add an id
+	if err := index.Add(id); err != nil {
+		t.Fatal(err)
+	}
+	// Get a non-existing id
+	assertIndexGet(t, index, "abracadabra", "", true)
+	// Get the exact id
+	assertIndexGet(t, index, id, id, false)
+	// The first letter should match
+	assertIndexGet(t, index, id[:1], id, false)
+	// The first half should match
+	assertIndexGet(t, index, id[:len(id)/2], id, false)
+	// The second half should NOT match
+	assertIndexGet(t, index, id[len(id)/2:], "", true)
+
+	id2 := id[:6] + "blabla"
+	// Add an id
+	if err := index.Add(id2); err != nil {
+		t.Fatal(err)
+	}
+	// Both exact IDs should work
+	assertIndexGet(t, index, id, id, false)
+	assertIndexGet(t, index, id2, id2, false)
+
+	// 6 characters or less should conflict
+	assertIndexGet(t, index, id[:6], "", true)
+	assertIndexGet(t, index, id[:4], "", true)
+	assertIndexGet(t, index, id[:1], "", true)
+
+	// 7 characters should NOT conflict
+	assertIndexGet(t, index, id[:7], id, false)
+	assertIndexGet(t, index, id2[:7], id2, false)
+
+	// Deleting a non-existing id should return an error
+	if err := index.Delete("non-existing"); err == nil {
+		t.Fatalf("Deleting a non-existing id should return an error")
+	}
+
+	// Deleting id2 should remove conflicts
+	if err := index.Delete(id2); err != nil {
+		t.Fatal(err)
+	}
+	// id2 should no longer work
+	assertIndexGet(t, index, id2, "", true)
+	assertIndexGet(t, index, id2[:7], "", true)
+	assertIndexGet(t, index, id2[:11], "", true)
+
+	// conflicts between id and id2 should be gone
+	assertIndexGet(t, index, id[:6], id, false)
+	assertIndexGet(t, index, id[:4], id, false)
+	assertIndexGet(t, index, id[:1], id, false)
+
+	// non-conflicting substrings should still not conflict
+	assertIndexGet(t, index, id[:7], id, false)
+	assertIndexGet(t, index, id[:15], id, false)
+	assertIndexGet(t, index, id, id, false)
+}
+
+func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
+	if result, err := index.Get(input); err != nil && !expectError {
+		t.Fatalf("Unexpected error getting '%s': %s", input, err)
+	} else if err == nil && expectError {
+		t.Fatalf("Getting '%s' should return an error", input)
+	} else if result != expectedResult {
+		t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
+	}
+}
+
+func BenchmarkTruncIndexAdd100(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAdd250(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAdd500(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet100(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet250(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet500(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexNew100(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}
+
+func BenchmarkTruncIndexNew250(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}
+
+func BenchmarkTruncIndexNew500(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, utils.GenerateRandomID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}

utils/utils.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"index/suffixarray"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -397,100 +396,6 @@ func GetTotalUsedFds() int {
 	return -1
 }
 
-// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
-// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
-type TruncIndex struct {
-	sync.RWMutex
-	index *suffixarray.Index
-	ids   map[string]bool
-	bytes []byte
-}
-
-func NewTruncIndex(ids []string) (idx *TruncIndex) {
-	idx = &TruncIndex{
-		ids:   make(map[string]bool),
-		bytes: []byte{' '},
-	}
-	for _, id := range ids {
-		idx.ids[id] = true
-		idx.bytes = append(idx.bytes, []byte(id+" ")...)
-	}
-	idx.index = suffixarray.New(idx.bytes)
-	return
-}
-
-func (idx *TruncIndex) addId(id string) error {
-	if strings.Contains(id, " ") {
-		return fmt.Errorf("Illegal character: ' '")
-	}
-	if _, exists := idx.ids[id]; exists {
-		return fmt.Errorf("Id already exists: %s", id)
-	}
-	idx.ids[id] = true
-	idx.bytes = append(idx.bytes, []byte(id+" ")...)
-	return nil
-}
-
-func (idx *TruncIndex) Add(id string) error {
-	idx.Lock()
-	defer idx.Unlock()
-	if err := idx.addId(id); err != nil {
-		return err
-	}
-	idx.index = suffixarray.New(idx.bytes)
-	return nil
-}
-
-func (idx *TruncIndex) AddWithoutSuffixarrayUpdate(id string) error {
-	idx.Lock()
-	defer idx.Unlock()
-	return idx.addId(id)
-}
-
-func (idx *TruncIndex) UpdateSuffixarray() {
-	idx.Lock()
-	defer idx.Unlock()
-	idx.index = suffixarray.New(idx.bytes)
-}
-
-func (idx *TruncIndex) Delete(id string) error {
-	idx.Lock()
-	defer idx.Unlock()
-	if _, exists := idx.ids[id]; !exists {
-		return fmt.Errorf("No such id: %s", id)
-	}
-	before, after, err := idx.lookup(id)
-	if err != nil {
-		return err
-	}
-	delete(idx.ids, id)
-	idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
-	idx.index = suffixarray.New(idx.bytes)
-	return nil
-}
-
-func (idx *TruncIndex) lookup(s string) (int, int, error) {
-	offsets := idx.index.Lookup([]byte(" "+s), -1)
-	//log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
-	if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
-		return -1, -1, fmt.Errorf("No such id: %s", s)
-	}
-	offsetBefore := offsets[0] + 1
-	offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
-	return offsetBefore, offsetAfter, nil
-}
-
-func (idx *TruncIndex) Get(s string) (string, error) {
-	idx.RLock()
-	defer idx.RUnlock()
-	before, after, err := idx.lookup(s)
-	//log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
-	if err != nil {
-		return "", err
-	}
-	return string(idx.bytes[before:after]), err
-}
-
 // TruncateID returns a shorthand version of a string identifier for convenience.
 // A collision with other shorthands is very unlikely, but possible.
 // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller

utils/utils_test.go

@@ -135,108 +135,6 @@ func TestRaceWriteBroadcaster(t *testing.T) {
 	<-c
 }
 
-// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
-func TestTruncIndex(t *testing.T) {
-	ids := []string{}
-	index := NewTruncIndex(ids)
-	// Get on an empty index
-	if _, err := index.Get("foobar"); err == nil {
-		t.Fatal("Get on an empty index should return an error")
-	}
-
-	// Spaces should be illegal in an id
-	if err := index.Add("I have a space"); err == nil {
-		t.Fatalf("Adding an id with ' ' should return an error")
-	}
-
-	id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
-	// Add an id
-	if err := index.Add(id); err != nil {
-		t.Fatal(err)
-	}
-	// Get a non-existing id
-	assertIndexGet(t, index, "abracadabra", "", true)
-	// Get the exact id
-	assertIndexGet(t, index, id, id, false)
-	// The first letter should match
-	assertIndexGet(t, index, id[:1], id, false)
-	// The first half should match
-	assertIndexGet(t, index, id[:len(id)/2], id, false)
-	// The second half should NOT match
-	assertIndexGet(t, index, id[len(id)/2:], "", true)
-
-	id2 := id[:6] + "blabla"
-	// Add an id
-	if err := index.Add(id2); err != nil {
-		t.Fatal(err)
-	}
-	// Both exact IDs should work
-	assertIndexGet(t, index, id, id, false)
-	assertIndexGet(t, index, id2, id2, false)
-
-	// 6 characters or less should conflict
-	assertIndexGet(t, index, id[:6], "", true)
-	assertIndexGet(t, index, id[:4], "", true)
-	assertIndexGet(t, index, id[:1], "", true)
-
-	// 7 characters should NOT conflict
-	assertIndexGet(t, index, id[:7], id, false)
-	assertIndexGet(t, index, id2[:7], id2, false)
-
-	// Deleting a non-existing id should return an error
-	if err := index.Delete("non-existing"); err == nil {
-		t.Fatalf("Deleting a non-existing id should return an error")
-	}
-
-	// Deleting id2 should remove conflicts
-	if err := index.Delete(id2); err != nil {
-		t.Fatal(err)
-	}
-	// id2 should no longer work
-	assertIndexGet(t, index, id2, "", true)
-	assertIndexGet(t, index, id2[:7], "", true)
-	assertIndexGet(t, index, id2[:11], "", true)
-
-	// conflicts between id and id2 should be gone
-	assertIndexGet(t, index, id[:6], id, false)
-	assertIndexGet(t, index, id[:4], id, false)
-	assertIndexGet(t, index, id[:1], id, false)
-
-	// non-conflicting substrings should still not conflict
-	assertIndexGet(t, index, id[:7], id, false)
-	assertIndexGet(t, index, id[:15], id, false)
-	assertIndexGet(t, index, id, id, false)
-}
-
-func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
-	if result, err := index.Get(input); err != nil && !expectError {
-		t.Fatalf("Unexpected error getting '%s': %s", input, err)
-	} else if err == nil && expectError {
-		t.Fatalf("Getting '%s' should return an error", input)
-	} else if result != expectedResult {
-		t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
-	}
-}
-
-func BenchmarkTruncIndexAdd(b *testing.B) {
-	ids := []string{"banana", "bananaa", "bananab"}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		index := NewTruncIndex([]string{})
-		for _, id := range ids {
-			index.Add(id)
-		}
-	}
-}
-
-func BenchmarkTruncIndexNew(b *testing.B) {
-	ids := []string{"banana", "bananaa", "bananab"}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		NewTruncIndex(ids)
-	}
-}
-
 func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) {
 	if r := CompareKernelVersion(a, b); r != result {
 		t.Fatalf("Unexpected kernel version comparison result. Found %d, expected %d", r, result)