From 5cfbdceafe87df8bab85c0561a1483d3191eb775 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Thu, 15 Sep 2016 10:53:48 -0700 Subject: [PATCH] Vendoring boltdb @fff57c100 Signed-off-by: Alessandro Boch --- hack/vendor.sh | 2 +- vendor/src/github.com/boltdb/bolt/README.md | 29 ++++++++++------- vendor/src/github.com/boltdb/bolt/bolt_386.go | 3 ++ .../src/github.com/boltdb/bolt/bolt_amd64.go | 3 ++ vendor/src/github.com/boltdb/bolt/bolt_arm.go | 21 ++++++++++++ .../src/github.com/boltdb/bolt/bolt_arm64.go | 3 ++ .../github.com/boltdb/bolt/bolt_ppc64le.go | 3 ++ .../src/github.com/boltdb/bolt/bolt_s390x.go | 3 ++ vendor/src/github.com/boltdb/bolt/bucket.go | 32 ++++++++++++++++++- vendor/src/github.com/boltdb/bolt/freelist.go | 20 ++++++++---- vendor/src/github.com/boltdb/bolt/node.go | 5 +++ vendor/src/github.com/boltdb/bolt/page.go | 6 ++++ 12 files changed, 110 insertions(+), 20 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 5efd4a444b..d4e4d7d997 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -88,7 +88,7 @@ clone git github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1 clone git github.com/coreos/etcd 3a49cbb769ebd8d1dd25abb1e83386e9883a5707 clone git github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 clone git github.com/hashicorp/consul v0.5.2 -clone git github.com/boltdb/bolt v1.2.1 +clone git github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages diff --git a/vendor/src/github.com/boltdb/bolt/README.md b/vendor/src/github.com/boltdb/bolt/README.md index 3bff3cce4b..2a69d95e78 100644 --- a/vendor/src/github.com/boltdb/bolt/README.md +++ b/vendor/src/github.com/boltdb/bolt/README.md @@ -1,4 +1,4 @@ -Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg) +Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) ==== Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] @@ -15,11 +15,11 @@ and setting values. That's it. ## Project Status -Bolt is stable and the API is fixed. Full unit test coverage and randomized -black box testing are used to ensure database consistency and thread safety. -Bolt is currently in high-load production environments serving databases as -large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed -services every day. +Bolt is stable, the API is fixed, and the file format is fixed. Full unit +test coverage and randomized black box testing are used to ensure database +consistency and thread safety. Bolt is currently in high-load production +environments serving databases as large as 1TB. Many companies such as +Shopify and Heroku use Bolt-backed services every day. ## Table of Contents @@ -209,7 +209,7 @@ and then safely close your transaction if an error is returned. This is the recommended way to use Bolt transactions. However, sometimes you may want to manually start and end your transactions. -You can use the `Tx.Begin()` function directly but **please** be sure to close +You can use the `DB.Begin()` function directly but **please** be sure to close the transaction. ```go @@ -313,7 +313,7 @@ func (s *Store) CreateUser(u *User) error { // Generate ID for the user. // This returns an error only if the Tx is closed or not writeable. // That can't happen in an Update() call so I ignore the error check. - id, _ = b.NextSequence() + id, _ := b.NextSequence() u.ID = int(id) // Marshal user data into bytes. @@ -448,6 +448,10 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Please note that keys and values in `ForEach()` are only valid while +the transaction is open. If you need to use a key or value outside of +the transaction, you must use `copy()` to copy it to another byte +slice. ### Nested buckets @@ -557,7 +561,7 @@ if err != nil { Bolt is able to run on mobile devices by leveraging the binding feature of the [gomobile](https://github.com/golang/mobile) tool. Create a struct that will contain your database logic and a reference to a `*bolt.DB` with a initializing -contstructor that takes in a filepath where the database file will be stored. +constructor that takes in a filepath where the database file will be stored. Neither Android nor iOS require extra permissions or cleanup from using this method. ```go @@ -807,6 +811,7 @@ them via pull request. Below is a list of public, open source projects that use Bolt: +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. * [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. * [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. @@ -825,7 +830,6 @@ Below is a list of public, open source projects that use Bolt: * [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. * [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database. * [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. @@ -842,9 +846,12 @@ Below is a list of public, open source projects that use Bolt: * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. -* [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. * [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/src/github.com/boltdb/bolt/bolt_386.go b/vendor/src/github.com/boltdb/bolt/bolt_386.go index e659bfb91f..820d533c15 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_386.go @@ -5,3 +5,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_amd64.go b/vendor/src/github.com/boltdb/bolt/bolt_amd64.go index cca6b7eb70..98fafdb47d 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_amd64.go @@ -5,3 +5,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_arm.go b/vendor/src/github.com/boltdb/bolt/bolt_arm.go index e659bfb91f..7e5cb4b941 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_arm.go @@ -1,7 +1,28 @@ package bolt +import "unsafe" + // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned bool + +func init() { + // Simple check to see whether this arch handles unaligned load/stores + // correctly. + + // ARM9 and older devices require load/stores to be from/to aligned + // addresses. If not, the lower 2 bits are cleared and that address is + // read in a jumbled up order. + + // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html + + raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11} + val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2)) + + brokenUnaligned = val != 0x11222211 +} diff --git a/vendor/src/github.com/boltdb/bolt/bolt_arm64.go b/vendor/src/github.com/boltdb/bolt/bolt_arm64.go index 6d2309352e..b26d84f91b 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_arm64.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go index 8351e129f6..8c143bc5d1 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_s390x.go b/vendor/src/github.com/boltdb/bolt/bolt_s390x.go index f4dd26bbba..d7c39af925 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_s390x.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bucket.go b/vendor/src/github.com/boltdb/bolt/bucket.go index d2f8c524e4..511ce72d33 100644 --- a/vendor/src/github.com/boltdb/bolt/bucket.go +++ b/vendor/src/github.com/boltdb/bolt/bucket.go @@ -130,9 +130,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket { func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) + // If unaligned load/stores are broken on this arch and value is + // unaligned simply clone to an aligned byte array. + unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0 + + if unaligned { + value = cloneBytes(value) + } + // If this is a writable transaction then we need to copy the bucket entry. // Read-only transactions can point directly at the mmap entry. - if b.tx.writable { + if b.tx.writable && !unaligned { child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) } else { @@ -329,6 +337,28 @@ func (b *Bucket) Delete(key []byte) error { return nil } +// Sequence returns the current integer for the bucket without incrementing it. +func (b *Bucket) Sequence() uint64 { return b.bucket.sequence } + +// SetSequence updates the sequence number for the bucket. +func (b *Bucket) SetSequence(v uint64) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Materialize the root node if it hasn't been already so that the + // bucket will be saved during commit. + if b.rootNode == nil { + _ = b.node(b.root, nil) + } + + // Increment and return the sequence. + b.bucket.sequence = v + return nil +} + // NextSequence returns an autoincrementing integer for the bucket. func (b *Bucket) NextSequence() (uint64, error) { if b.tx.db == nil { diff --git a/vendor/src/github.com/boltdb/bolt/freelist.go b/vendor/src/github.com/boltdb/bolt/freelist.go index 0161948fcf..d32f6cd937 100644 --- a/vendor/src/github.com/boltdb/bolt/freelist.go +++ b/vendor/src/github.com/boltdb/bolt/freelist.go @@ -166,12 +166,16 @@ func (f *freelist) read(p *page) { } // Copy the list of page ids from the freelist. - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + f.ids = make([]pgid, len(ids)) + copy(f.ids, ids) - // Make sure they're sorted. - sort.Sort(pgids(f.ids)) + // Make sure they're sorted. + sort.Sort(pgids(f.ids)) + } // Rebuild the page cache. f.reindex() @@ -189,7 +193,9 @@ func (f *freelist) write(p *page) error { // The page.count can only hold up to 64k elements so if we overflow that // number then we handle it by putting the size in the first element. - if len(ids) < 0xFFFF { + if len(ids) == 0 { + p.count = uint16(len(ids)) + } else if len(ids) < 0xFFFF { p.count = uint16(len(ids)) copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids) } else { @@ -230,7 +236,7 @@ func (f *freelist) reload(p *page) { // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { - f.cache = make(map[pgid]bool) + f.cache = make(map[pgid]bool, len(f.ids)) for _, id := range f.ids { f.cache[id] = true } diff --git a/vendor/src/github.com/boltdb/bolt/node.go b/vendor/src/github.com/boltdb/bolt/node.go index e9d64af81e..159318b229 100644 --- a/vendor/src/github.com/boltdb/bolt/node.go +++ b/vendor/src/github.com/boltdb/bolt/node.go @@ -201,6 +201,11 @@ func (n *node) write(p *page) { } p.count = uint16(len(n.inodes)) + // Stop here if there are no items to write. + if p.count == 0 { + return + } + // Loop over each item and write it to the page. b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] for i, item := range n.inodes { diff --git a/vendor/src/github.com/boltdb/bolt/page.go b/vendor/src/github.com/boltdb/bolt/page.go index 4a555286a3..7651a6bf7d 100644 --- a/vendor/src/github.com/boltdb/bolt/page.go +++ b/vendor/src/github.com/boltdb/bolt/page.go @@ -62,6 +62,9 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -72,6 +75,9 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] }