1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Godeps update

Signed-off-by: Madhu Venugopal <madhu@docker.com>
This commit is contained in:
Madhu Venugopal 2015-09-30 08:16:58 -07:00
parent 00772afa41
commit 1b393486b5
137 changed files with 5656 additions and 3384 deletions

View file

@ -1,6 +1,6 @@
{
"ImportPath": "github.com/docker/libnetwork",
"GoVersion": "go1.4.2",
"GoVersion": "go1.4.1",
"Packages": [
"./..."
],
@ -19,6 +19,11 @@
"ImportPath": "github.com/armon/go-metrics",
"Rev": "eb0af217e5e9747e41dd5303755356b62d28e3ec"
},
{
"ImportPath": "github.com/boltdb/bolt",
"Comment": "v1.0-117-g0f053fa",
"Rev": "0f053fabc06119583d61937a0a06ef0ba0f1b301"
},
{
"ImportPath": "github.com/codegangsta/cli",
"Comment": "1.2.0-143-ga65b733",
@ -39,70 +44,105 @@
"Comment": "v3",
"Rev": "be94bc700879ae8217780e9d141789a2defa302b"
},
{
"ImportPath": "github.com/deckarep/golang-set",
"Comment": "v1-26-gef32fa3",
"Rev": "ef32fa3046d9f249d399f98ebaf9be944430fd1d"
},
{
"ImportPath": "github.com/docker/docker/pkg/discovery",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/homedir",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/ioutils",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/listenbuffer",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/mflag",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/mount",
"Comment": "v1.4.1-4127-gb81f2ee",
"Rev": "b81f2ee5f20d094c13893f565ce716595c539d22"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/parsers",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
},
{
"ImportPath": "github.com/docker/docker/pkg/plugins",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/proxy",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/random",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/reexec",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/signal",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/sockets",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/stringid",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/symlink",
"Comment": "v1.4.1-4127-gb81f2ee",
"Rev": "b81f2ee5f20d094c13893f565ce716595c539d22"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/system",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/term",
"Comment": "v1.4.1-4106-g637023a",
"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/tlsconfig",
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/docker/pkg/units",
"Comment": "v1.4.1-4127-gb81f2ee",
"Rev": "b81f2ee5f20d094c13893f565ce716595c539d22"
},
{
"ImportPath": "github.com/docker/libcontainer/user",
"Comment": "v1.4.0-495-g3e66118",
"Rev": "3e661186ba24f259d3860f067df052c7f6904bee"
"Comment": "v1.4.1-6485-g949270b",
"Rev": "949270ba7ca3c93f25a13abf68d5e0e4c05be08a"
},
{
"ImportPath": "github.com/docker/libkv",
@ -167,11 +207,6 @@
{
"ImportPath": "github.com/vishvananda/netns",
"Rev": "493029407eeb434d0c2d44e02ea072ff2488d322"
},
{
"ImportPath": "github.com/boltdb/bolt",
"Comment": "v1.0-117-g0f053fa",
"Rev": "0f053fabc06119583d61937a0a06ef0ba0f1b301"
}
]
}

View file

@ -0,0 +1,41 @@
---
page_title: Docker discovery
page_description: discovery
page_keywords: docker, clustering, discovery
---
# Discovery
Docker comes with multiple Discovery backends.
## Backends
### Using etcd
Point your Docker Engine instances to a common etcd instance. You can specify
the address Docker uses to advertise the node using the `--discovery-address`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --discovery-address=<node_ip:2376> --discovery-backend etcd://<etcd_ip>/<path>
```
### Using consul
Point your Docker Engine instances to a common Consul instance. You can specify
the address Docker uses to advertise the node using the `--discovery-address`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --discovery-address=<node_ip:2376> --discovery-backend consul://<consul_ip>/<path>
```
### Using zookeeper
Point your Docker Engine instances to a common Zookeeper instance. You can specify
the address Docker uses to advertise the node using the `--discovery-address`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --discovery-address=<node_ip:2376> --discovery-backend zk://<zk_addr1>,<zk_addr2>>/<path>
```

View file

@ -0,0 +1,53 @@
package discovery
import (
"fmt"
"strings"
"time"
log "github.com/Sirupsen/logrus"
)
var (
// Backends is a global map of discovery backends indexed by their
// associated scheme.
backends map[string]Backend
)
func init() {
backends = make(map[string]Backend)
}
// Register makes a discovery backend available by the provided scheme.
// If Register is called twice with the same scheme an error is returned.
func Register(scheme string, d Backend) error {
if _, exists := backends[scheme]; exists {
return fmt.Errorf("scheme already registered %s", scheme)
}
log.WithField("name", scheme).Debug("Registering discovery service")
backends[scheme] = d
return nil
}
func parse(rawurl string) (string, string) {
parts := strings.SplitN(rawurl, "://", 2)
// nodes:port,node2:port => nodes://node1:port,node2:port
if len(parts) == 1 {
return "nodes", parts[0]
}
return parts[0], parts[1]
}
// New returns a new Discovery given a URL, heartbeat and ttl settings.
// Returns an error if the URL scheme is not supported.
func New(rawurl string, heartbeat time.Duration, ttl time.Duration) (Backend, error) {
scheme, uri := parse(rawurl)
if backend, exists := backends[scheme]; exists {
log.WithFields(log.Fields{"name": scheme, "uri": uri}).Debug("Initializing discovery service")
err := backend.Initialize(uri, heartbeat, ttl)
return backend, err
}
return nil, ErrNotSupported
}

View file

@ -0,0 +1,35 @@
package discovery
import (
"errors"
"time"
)
var (
// ErrNotSupported is returned when a discovery service is not supported.
ErrNotSupported = errors.New("discovery service not supported")
// ErrNotImplemented is returned when discovery feature is not implemented
// by discovery backend.
ErrNotImplemented = errors.New("not implemented in this discovery service")
)
// Watcher provides watching over a cluster for nodes joining and leaving.
type Watcher interface {
// Watch the discovery for entry changes.
// Returns a channel that will receive changes or an error.
// Providing a non-nil stopCh can be used to stop watching.
Watch(stopCh <-chan struct{}) (<-chan Entries, <-chan error)
}
// Backend is implemented by discovery backends which manage cluster entries.
type Backend interface {
// Watcher must be provided by every backend.
Watcher
// Initialize the discovery with URIs, a heartbeat and a ttl.
Initialize(string, time.Duration, time.Duration) error
// Register to the discovery.
Register(string) error
}

View file

@ -0,0 +1,120 @@
package discovery
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNewEntry(t *testing.T) {
entry, err := NewEntry("127.0.0.1:2375")
assert.NoError(t, err)
assert.True(t, entry.Equals(&Entry{Host: "127.0.0.1", Port: "2375"}))
assert.Equal(t, entry.String(), "127.0.0.1:2375")
_, err = NewEntry("127.0.0.1")
assert.Error(t, err)
}
func TestParse(t *testing.T) {
scheme, uri := parse("127.0.0.1:2375")
assert.Equal(t, scheme, "nodes")
assert.Equal(t, uri, "127.0.0.1:2375")
scheme, uri = parse("localhost:2375")
assert.Equal(t, scheme, "nodes")
assert.Equal(t, uri, "localhost:2375")
scheme, uri = parse("scheme://127.0.0.1:2375")
assert.Equal(t, scheme, "scheme")
assert.Equal(t, uri, "127.0.0.1:2375")
scheme, uri = parse("scheme://localhost:2375")
assert.Equal(t, scheme, "scheme")
assert.Equal(t, uri, "localhost:2375")
scheme, uri = parse("")
assert.Equal(t, scheme, "nodes")
assert.Equal(t, uri, "")
}
func TestCreateEntries(t *testing.T) {
entries, err := CreateEntries(nil)
assert.Equal(t, entries, Entries{})
assert.NoError(t, err)
entries, err = CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", ""})
assert.NoError(t, err)
expected := Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
}
assert.True(t, entries.Equals(expected))
_, err = CreateEntries([]string{"127.0.0.1", "127.0.0.2"})
assert.Error(t, err)
}
func TestContainsEntry(t *testing.T) {
entries, err := CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", ""})
assert.NoError(t, err)
assert.True(t, entries.Contains(&Entry{Host: "127.0.0.1", Port: "2375"}))
assert.False(t, entries.Contains(&Entry{Host: "127.0.0.3", Port: "2375"}))
}
func TestEntriesEquality(t *testing.T) {
entries := Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
}
// Same
assert.True(t, entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
}))
// Different size
assert.False(t, entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
&Entry{Host: "127.0.0.3", Port: "2375"},
}))
// Different content
assert.False(t, entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.42", Port: "2375"},
}))
}
func TestEntriesDiff(t *testing.T) {
entry1 := &Entry{Host: "1.1.1.1", Port: "1111"}
entry2 := &Entry{Host: "2.2.2.2", Port: "2222"}
entry3 := &Entry{Host: "3.3.3.3", Port: "3333"}
entries := Entries{entry1, entry2}
// No diff
added, removed := entries.Diff(Entries{entry2, entry1})
assert.Empty(t, added)
assert.Empty(t, removed)
// Add
added, removed = entries.Diff(Entries{entry2, entry3, entry1})
assert.Len(t, added, 1)
assert.True(t, added.Contains(entry3))
assert.Empty(t, removed)
// Remove
added, removed = entries.Diff(Entries{entry2})
assert.Empty(t, added)
assert.Len(t, removed, 1)
assert.True(t, removed.Contains(entry1))
// Add and remove
added, removed = entries.Diff(Entries{entry1, entry3})
assert.Len(t, added, 1)
assert.True(t, added.Contains(entry3))
assert.Len(t, removed, 1)
assert.True(t, removed.Contains(entry2))
}

View file

@ -0,0 +1,97 @@
package discovery
import (
"fmt"
"net"
)
// NewEntry creates a new entry.
func NewEntry(url string) (*Entry, error) {
host, port, err := net.SplitHostPort(url)
if err != nil {
return nil, err
}
return &Entry{host, port}, nil
}
// An Entry represents a host.
type Entry struct {
Host string
Port string
}
// Equals returns true if cmp contains the same data.
func (e *Entry) Equals(cmp *Entry) bool {
return e.Host == cmp.Host && e.Port == cmp.Port
}
// String returns the string form of an entry.
func (e *Entry) String() string {
return fmt.Sprintf("%s:%s", e.Host, e.Port)
}
// Entries is a list of *Entry with some helpers.
type Entries []*Entry
// Equals returns true if cmp contains the same data.
func (e Entries) Equals(cmp Entries) bool {
// Check if the file has really changed.
if len(e) != len(cmp) {
return false
}
for i := range e {
if !e[i].Equals(cmp[i]) {
return false
}
}
return true
}
// Contains returns true if the Entries contain a given Entry.
func (e Entries) Contains(entry *Entry) bool {
for _, curr := range e {
if curr.Equals(entry) {
return true
}
}
return false
}
// Diff compares two entries and returns the added and removed entries.
func (e Entries) Diff(cmp Entries) (Entries, Entries) {
added := Entries{}
for _, entry := range cmp {
if !e.Contains(entry) {
added = append(added, entry)
}
}
removed := Entries{}
for _, entry := range e {
if !cmp.Contains(entry) {
removed = append(removed, entry)
}
}
return added, removed
}
// CreateEntries returns an array of entries based on the given addresses.
func CreateEntries(addrs []string) (Entries, error) {
entries := Entries{}
if addrs == nil {
return entries, nil
}
for _, addr := range addrs {
if len(addr) == 0 {
continue
}
entry, err := NewEntry(addr)
if err != nil {
return nil, err
}
entries = append(entries, entry)
}
return entries, nil
}

View file

@ -0,0 +1,109 @@
package file
import (
"fmt"
"io/ioutil"
"strings"
"time"
"github.com/docker/docker/pkg/discovery"
)
// Discovery is exported
type Discovery struct {
heartbeat time.Duration
path string
}
func init() {
Init()
}
// Init is exported
func Init() {
discovery.Register("file", &Discovery{})
}
// Initialize is exported
func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration) error {
s.path = path
s.heartbeat = heartbeat
return nil
}
func parseFileContent(content []byte) []string {
var result []string
for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") {
line = strings.TrimSpace(line)
// Ignoring line starts with #
if strings.HasPrefix(line, "#") {
continue
}
// Inlined # comment also ignored.
if strings.Contains(line, "#") {
line = line[0:strings.Index(line, "#")]
// Trim additional spaces caused by above stripping.
line = strings.TrimSpace(line)
}
for _, ip := range discovery.Generate(line) {
result = append(result, ip)
}
}
return result
}
func (s *Discovery) fetch() (discovery.Entries, error) {
fileContent, err := ioutil.ReadFile(s.path)
if err != nil {
return nil, fmt.Errorf("failed to read '%s': %v", s.path, err)
}
return discovery.CreateEntries(parseFileContent(fileContent))
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
errCh := make(chan error)
ticker := time.NewTicker(s.heartbeat)
go func() {
defer close(errCh)
defer close(ch)
// Send the initial entries if available.
currentEntries, err := s.fetch()
if err != nil {
errCh <- err
} else {
ch <- currentEntries
}
// Periodically send updates.
for {
select {
case <-ticker.C:
newEntries, err := s.fetch()
if err != nil {
errCh <- err
continue
}
// Check if the file has really changed.
if !newEntries.Equals(currentEntries) {
ch <- newEntries
}
currentEntries = newEntries
case <-stopCh:
ticker.Stop()
return
}
}
}()
return ch, errCh
}
// Register is exported
func (s *Discovery) Register(addr string) error {
return discovery.ErrNotImplemented
}

View file

@ -0,0 +1,106 @@
package file
import (
"io/ioutil"
"os"
"testing"
"github.com/docker/docker/pkg/discovery"
"github.com/stretchr/testify/assert"
)
func TestInitialize(t *testing.T) {
d := &Discovery{}
d.Initialize("/path/to/file", 1000, 0)
assert.Equal(t, d.path, "/path/to/file")
}
func TestNew(t *testing.T) {
d, err := discovery.New("file:///path/to/file", 0, 0)
assert.NoError(t, err)
assert.Equal(t, d.(*Discovery).path, "/path/to/file")
}
func TestContent(t *testing.T) {
data := `
1.1.1.[1:2]:1111
2.2.2.[2:4]:2222
`
ips := parseFileContent([]byte(data))
assert.Len(t, ips, 5)
assert.Equal(t, ips[0], "1.1.1.1:1111")
assert.Equal(t, ips[1], "1.1.1.2:1111")
assert.Equal(t, ips[2], "2.2.2.2:2222")
assert.Equal(t, ips[3], "2.2.2.3:2222")
assert.Equal(t, ips[4], "2.2.2.4:2222")
}
func TestRegister(t *testing.T) {
discovery := &Discovery{path: "/path/to/file"}
assert.Error(t, discovery.Register("0.0.0.0"))
}
func TestParsingContentsWithComments(t *testing.T) {
data := `
### test ###
1.1.1.1:1111 # inline comment
# 2.2.2.2:2222
### empty line with comment
3.3.3.3:3333
### test ###
`
ips := parseFileContent([]byte(data))
assert.Len(t, ips, 2)
assert.Equal(t, "1.1.1.1:1111", ips[0])
assert.Equal(t, "3.3.3.3:3333", ips[1])
}
func TestWatch(t *testing.T) {
data := `
1.1.1.1:1111
2.2.2.2:2222
`
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
// Create a temporary file and remove it.
tmp, err := ioutil.TempFile(os.TempDir(), "discovery-file-test")
assert.NoError(t, err)
assert.NoError(t, tmp.Close())
assert.NoError(t, os.Remove(tmp.Name()))
// Set up file discovery.
d := &Discovery{}
d.Initialize(tmp.Name(), 1000, 0)
stopCh := make(chan struct{})
ch, errCh := d.Watch(stopCh)
// Make sure it fires errors since the file doesn't exist.
assert.Error(t, <-errCh)
// We have to drain the error channel otherwise Watch will get stuck.
go func() {
for range errCh {
}
}()
// Write the file and make sure we get the expected value back.
assert.NoError(t, ioutil.WriteFile(tmp.Name(), []byte(data), 0600))
assert.Equal(t, expected, <-ch)
// Add a new entry and look it up.
expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"})
f, err := os.OpenFile(tmp.Name(), os.O_APPEND|os.O_WRONLY, 0600)
assert.NoError(t, err)
assert.NotNil(t, f)
_, err = f.WriteString("\n3.3.3.3:3333\n")
assert.NoError(t, err)
f.Close()
assert.Equal(t, expected, <-ch)
// Stop and make sure it closes all channels.
close(stopCh)
assert.Nil(t, <-ch)
assert.Nil(t, <-errCh)
}

View file

@ -0,0 +1,35 @@
package discovery
import (
"fmt"
"regexp"
"strconv"
)
// Generate takes care of IP generation
func Generate(pattern string) []string {
re, _ := regexp.Compile(`\[(.+):(.+)\]`)
submatch := re.FindStringSubmatch(pattern)
if submatch == nil {
return []string{pattern}
}
from, err := strconv.Atoi(submatch[1])
if err != nil {
return []string{pattern}
}
to, err := strconv.Atoi(submatch[2])
if err != nil {
return []string{pattern}
}
template := re.ReplaceAllString(pattern, "%d")
var result []string
for val := from; val <= to; val++ {
entry := fmt.Sprintf(template, val)
result = append(result, entry)
}
return result
}

View file

@ -0,0 +1,55 @@
package discovery
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGeneratorNotGenerate(t *testing.T) {
ips := Generate("127.0.0.1")
assert.Equal(t, len(ips), 1)
assert.Equal(t, ips[0], "127.0.0.1")
}
func TestGeneratorWithPortNotGenerate(t *testing.T) {
ips := Generate("127.0.0.1:8080")
assert.Equal(t, len(ips), 1)
assert.Equal(t, ips[0], "127.0.0.1:8080")
}
func TestGeneratorMatchFailedNotGenerate(t *testing.T) {
ips := Generate("127.0.0.[1]")
assert.Equal(t, len(ips), 1)
assert.Equal(t, ips[0], "127.0.0.[1]")
}
func TestGeneratorWithPort(t *testing.T) {
ips := Generate("127.0.0.[1:11]:2375")
assert.Equal(t, len(ips), 11)
assert.Equal(t, ips[0], "127.0.0.1:2375")
assert.Equal(t, ips[1], "127.0.0.2:2375")
assert.Equal(t, ips[2], "127.0.0.3:2375")
assert.Equal(t, ips[3], "127.0.0.4:2375")
assert.Equal(t, ips[4], "127.0.0.5:2375")
assert.Equal(t, ips[5], "127.0.0.6:2375")
assert.Equal(t, ips[6], "127.0.0.7:2375")
assert.Equal(t, ips[7], "127.0.0.8:2375")
assert.Equal(t, ips[8], "127.0.0.9:2375")
assert.Equal(t, ips[9], "127.0.0.10:2375")
assert.Equal(t, ips[10], "127.0.0.11:2375")
}
func TestGenerateWithMalformedInputAtRangeStart(t *testing.T) {
malformedInput := "127.0.0.[x:11]:2375"
ips := Generate(malformedInput)
assert.Equal(t, len(ips), 1)
assert.Equal(t, ips[0], malformedInput)
}
func TestGenerateWithMalformedInputAtRangeEnd(t *testing.T) {
malformedInput := "127.0.0.[1:x]:2375"
ips := Generate(malformedInput)
assert.Equal(t, len(ips), 1)
assert.Equal(t, ips[0], malformedInput)
}

View file

@ -0,0 +1,148 @@
package kv
import (
"fmt"
"path"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/discovery"
"github.com/docker/libkv"
"github.com/docker/libkv/store"
"github.com/docker/libkv/store/consul"
"github.com/docker/libkv/store/etcd"
"github.com/docker/libkv/store/zookeeper"
)
const (
discoveryPath = "docker/nodes"
)
// Discovery is exported
type Discovery struct {
backend store.Backend
store store.Store
heartbeat time.Duration
ttl time.Duration
prefix string
path string
}
func init() {
Init()
}
// Init is exported
func Init() {
// Register to libkv
zookeeper.Register()
consul.Register()
etcd.Register()
// Register to internal discovery service
discovery.Register("zk", &Discovery{backend: store.ZK})
discovery.Register("consul", &Discovery{backend: store.CONSUL})
discovery.Register("etcd", &Discovery{backend: store.ETCD})
}
// Initialize is exported
func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration) error {
var (
parts = strings.SplitN(uris, "/", 2)
addrs = strings.Split(parts[0], ",")
err error
)
// A custom prefix to the path can be optionally used.
if len(parts) == 2 {
s.prefix = parts[1]
}
s.heartbeat = heartbeat
s.ttl = ttl
s.path = path.Join(s.prefix, discoveryPath)
// Creates a new store, will ignore options given
// if not supported by the chosen store
s.store, err = libkv.NewStore(s.backend, addrs, nil)
return err
}
// Watch the store until either there's a store error or we receive a stop request.
// Returns false if we shouldn't attempt watching the store anymore (stop request received).
func (s *Discovery) watchOnce(stopCh <-chan struct{}, watchCh <-chan []*store.KVPair, discoveryCh chan discovery.Entries, errCh chan error) bool {
for {
select {
case pairs := <-watchCh:
if pairs == nil {
return true
}
log.WithField("discovery", s.backend).Debugf("Watch triggered with %d nodes", len(pairs))
// Convert `KVPair` into `discovery.Entry`.
addrs := make([]string, len(pairs))
for _, pair := range pairs {
addrs = append(addrs, string(pair.Value))
}
entries, err := discovery.CreateEntries(addrs)
if err != nil {
errCh <- err
} else {
discoveryCh <- entries
}
case <-stopCh:
// We were requested to stop watching.
return false
}
}
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
errCh := make(chan error)
go func() {
defer close(ch)
defer close(errCh)
// Forever: Create a store watch, watch until we get an error and then try again.
// Will only stop if we receive a stopCh request.
for {
// Set up a watch.
watchCh, err := s.store.WatchTree(s.path, stopCh)
if err != nil {
errCh <- err
} else {
if !s.watchOnce(stopCh, watchCh, ch, errCh) {
return
}
}
// If we get here it means the store watch channel was closed. This
// is unexpected so let's retry later.
errCh <- fmt.Errorf("Unexpected watch error")
time.Sleep(s.heartbeat)
}
}()
return ch, errCh
}
// Register is exported
func (s *Discovery) Register(addr string) error {
opts := &store.WriteOptions{TTL: s.ttl}
return s.store.Put(path.Join(s.path, addr), []byte(addr), opts)
}
// Store returns the underlying store used by KV discovery.
func (s *Discovery) Store() store.Store {
return s.store
}
// Prefix returns the store prefix
func (s *Discovery) Prefix() string {
return s.prefix
}

View file

@ -0,0 +1,119 @@
package kv
import (
"errors"
"path"
"testing"
"time"
"github.com/docker/docker/pkg/discovery"
"github.com/docker/libkv/store"
libkvmock "github.com/docker/libkv/store/mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func TestInitialize(t *testing.T) {
storeMock, err := libkvmock.New([]string{"127.0.0.1"}, nil)
assert.NotNil(t, storeMock)
assert.NoError(t, err)
d := &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1", 0, 0)
d.store = storeMock
s := d.store.(*libkvmock.Mock)
assert.Len(t, s.Endpoints, 1)
assert.Equal(t, s.Endpoints[0], "127.0.0.1")
assert.Equal(t, d.path, discoveryPath)
storeMock, err = libkvmock.New([]string{"127.0.0.1:1234"}, nil)
assert.NotNil(t, storeMock)
assert.NoError(t, err)
d = &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234/path", 0, 0)
d.store = storeMock
s = d.store.(*libkvmock.Mock)
assert.Len(t, s.Endpoints, 1)
assert.Equal(t, s.Endpoints[0], "127.0.0.1:1234")
assert.Equal(t, d.path, "path/"+discoveryPath)
storeMock, err = libkvmock.New([]string{"127.0.0.1:1234", "127.0.0.2:1234", "127.0.0.3:1234"}, nil)
assert.NotNil(t, storeMock)
assert.NoError(t, err)
d = &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234,127.0.0.2:1234,127.0.0.3:1234/path", 0, 0)
d.store = storeMock
s = d.store.(*libkvmock.Mock)
if assert.Len(t, s.Endpoints, 3) {
assert.Equal(t, s.Endpoints[0], "127.0.0.1:1234")
assert.Equal(t, s.Endpoints[1], "127.0.0.2:1234")
assert.Equal(t, s.Endpoints[2], "127.0.0.3:1234")
}
assert.Equal(t, d.path, "path/"+discoveryPath)
}
func TestWatch(t *testing.T) {
storeMock, err := libkvmock.New([]string{"127.0.0.1:1234"}, nil)
assert.NotNil(t, storeMock)
assert.NoError(t, err)
d := &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234/path", 0, 0)
d.store = storeMock
s := d.store.(*libkvmock.Mock)
mockCh := make(chan []*store.KVPair)
// The first watch will fail.
s.On("WatchTree", "path/"+discoveryPath, mock.Anything).Return(mockCh, errors.New("test error")).Once()
// The second one will succeed.
s.On("WatchTree", "path/"+discoveryPath, mock.Anything).Return(mockCh, nil).Once()
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
kvs := []*store.KVPair{
{Key: path.Join("path", discoveryPath, "1.1.1.1"), Value: []byte("1.1.1.1:1111")},
{Key: path.Join("path", discoveryPath, "2.2.2.2"), Value: []byte("2.2.2.2:2222")},
}
stopCh := make(chan struct{})
ch, errCh := d.Watch(stopCh)
// It should fire an error since the first WatchTree call failed.
assert.EqualError(t, <-errCh, "test error")
// We have to drain the error channel otherwise Watch will get stuck.
go func() {
for range errCh {
}
}()
// Push the entries into the store channel and make sure discovery emits.
mockCh <- kvs
assert.Equal(t, <-ch, expected)
// Add a new entry.
expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"})
kvs = append(kvs, &store.KVPair{Key: path.Join("path", discoveryPath, "3.3.3.3"), Value: []byte("3.3.3.3:3333")})
mockCh <- kvs
assert.Equal(t, <-ch, expected)
// Make sure that if an error occurs it retries.
// This third call to WatchTree will be checked later by AssertExpectations.
s.On("WatchTree", "path/"+discoveryPath, mock.Anything).Return(mockCh, nil)
close(mockCh)
// Give it enough time to call WatchTree.
time.Sleep(3)
// Stop and make sure it closes all channels.
close(stopCh)
assert.Nil(t, <-ch)
assert.Nil(t, <-errCh)
s.AssertExpectations(t)
}

View file

@ -0,0 +1,54 @@
package nodes
import (
"fmt"
"strings"
"time"
"github.com/docker/docker/pkg/discovery"
)
// Discovery is exported
type Discovery struct {
entries discovery.Entries
}
func init() {
Init()
}
// Init is exported
func Init() {
discovery.Register("nodes", &Discovery{})
}
// Initialize is exported
func (s *Discovery) Initialize(uris string, _ time.Duration, _ time.Duration) error {
for _, input := range strings.Split(uris, ",") {
for _, ip := range discovery.Generate(input) {
entry, err := discovery.NewEntry(ip)
if err != nil {
return fmt.Errorf("%s, please check you are using the correct discovery (missing token:// ?)", err.Error())
}
s.entries = append(s.entries, entry)
}
}
return nil
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
go func() {
defer close(ch)
ch <- s.entries
<-stopCh
}()
return ch, nil
}
// Register is exported
func (s *Discovery) Register(addr string) error {
return discovery.ErrNotImplemented
}

View file

@ -0,0 +1,43 @@
package nodes
import (
"testing"
"github.com/docker/docker/pkg/discovery"
"github.com/stretchr/testify/assert"
)
func TestInitialize(t *testing.T) {
d := &Discovery{}
d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0)
assert.Equal(t, len(d.entries), 2)
assert.Equal(t, d.entries[0].String(), "1.1.1.1:1111")
assert.Equal(t, d.entries[1].String(), "2.2.2.2:2222")
}
func TestInitializeWithPattern(t *testing.T) {
d := &Discovery{}
d.Initialize("1.1.1.[1:2]:1111,2.2.2.[2:4]:2222", 0, 0)
assert.Equal(t, len(d.entries), 5)
assert.Equal(t, d.entries[0].String(), "1.1.1.1:1111")
assert.Equal(t, d.entries[1].String(), "1.1.1.2:1111")
assert.Equal(t, d.entries[2].String(), "2.2.2.2:2222")
assert.Equal(t, d.entries[3].String(), "2.2.2.3:2222")
assert.Equal(t, d.entries[4].String(), "2.2.2.4:2222")
}
func TestWatch(t *testing.T) {
d := &Discovery{}
d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0)
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
ch, _ := d.Watch(nil)
assert.True(t, expected.Equals(<-ch))
}
func TestRegister(t *testing.T) {
d := &Discovery{}
assert.Error(t, d.Register("0.0.0.0"))
}

View file

@ -4,7 +4,7 @@ import (
"os"
"runtime"
"github.com/docker/libcontainer/user"
"github.com/opencontainers/runc/libcontainer/user"
)
// Key returns the env var name for the user's home dir based on

View file

@ -0,0 +1,89 @@
package ioutils
const maxCap = 1e6
// BytesPipe is io.ReadWriter which works similarly to pipe(queue).
// All written data could be read only once. Also BytesPipe is allocating
// and releasing new byte slices to adjust to current needs, so there won't be
// overgrown buffer after high load peak.
// BytesPipe isn't goroutine-safe, caller must synchronize it if needed.
type BytesPipe struct {
buf [][]byte // slice of byte-slices of buffered data
lastRead int // index in the first slice to a read point
bufLen int // length of data buffered over the slices
}
// NewBytesPipe creates new BytesPipe, initialized by specified slice.
// If buf is nil, then it will be initialized with slice which cap is 64.
// buf will be adjusted in a way that len(buf) == 0, cap(buf) == cap(buf).
func NewBytesPipe(buf []byte) *BytesPipe {
if cap(buf) == 0 {
buf = make([]byte, 0, 64)
}
return &BytesPipe{
buf: [][]byte{buf[:0]},
}
}
// Write writes p to BytesPipe.
// It can allocate new []byte slices in a process of writing.
func (bp *BytesPipe) Write(p []byte) (n int, err error) {
for {
// write data to the last buffer
b := bp.buf[len(bp.buf)-1]
// copy data to the current empty allocated area
n := copy(b[len(b):cap(b)], p)
// increment buffered data length
bp.bufLen += n
// include written data in last buffer
bp.buf[len(bp.buf)-1] = b[:len(b)+n]
// if there was enough room to write all then break
if len(p) == n {
break
}
// more data: write to the next slice
p = p[n:]
// allocate slice that has twice the size of the last unless maximum reached
nextCap := 2 * cap(bp.buf[len(bp.buf)-1])
if maxCap < nextCap {
nextCap = maxCap
}
// add new byte slice to the buffers slice and continue writing
bp.buf = append(bp.buf, make([]byte, 0, nextCap))
}
return
}
func (bp *BytesPipe) len() int {
return bp.bufLen - bp.lastRead
}
// Read reads bytes from BytesPipe.
// Data could be read only once.
func (bp *BytesPipe) Read(p []byte) (n int, err error) {
for {
read := copy(p, bp.buf[0][bp.lastRead:])
n += read
bp.lastRead += read
if bp.len() == 0 {
// we have read everything. reset to the beginning.
bp.lastRead = 0
bp.bufLen -= len(bp.buf[0])
bp.buf[0] = bp.buf[0][:0]
break
}
// break if everything was read
if len(p) == read {
break
}
// more buffered data and more asked. read from next slice.
p = p[read:]
bp.lastRead = 0
bp.bufLen -= len(bp.buf[0])
bp.buf[0] = nil // throw away old slice
bp.buf = bp.buf[1:] // switch to next
}
return
}

View file

@ -0,0 +1,141 @@
package ioutils
import (
"crypto/sha1"
"encoding/hex"
"testing"
)
func TestBytesPipeRead(t *testing.T) {
buf := NewBytesPipe(nil)
buf.Write([]byte("12"))
buf.Write([]byte("34"))
buf.Write([]byte("56"))
buf.Write([]byte("78"))
buf.Write([]byte("90"))
rd := make([]byte, 4)
n, err := buf.Read(rd)
if err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 4)
}
if string(rd) != "1234" {
t.Fatalf("Read %s, but must be %s", rd, "1234")
}
n, err = buf.Read(rd)
if err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 4)
}
if string(rd) != "5678" {
t.Fatalf("Read %s, but must be %s", rd, "5679")
}
n, err = buf.Read(rd)
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 2)
}
if string(rd[:n]) != "90" {
t.Fatalf("Read %s, but must be %s", rd, "90")
}
}
func TestBytesPipeWrite(t *testing.T) {
buf := NewBytesPipe(nil)
buf.Write([]byte("12"))
buf.Write([]byte("34"))
buf.Write([]byte("56"))
buf.Write([]byte("78"))
buf.Write([]byte("90"))
if string(buf.buf[0]) != "1234567890" {
t.Fatalf("Buffer %s, must be %s", buf.buf, "1234567890")
}
}
// Write and read in different speeds/chunk sizes and check valid data is read.
func TestBytesPipeWriteRandomChunks(t *testing.T) {
cases := []struct{ iterations, writesPerLoop, readsPerLoop int }{
{100, 10, 1},
{1000, 10, 5},
{1000, 100, 0},
{1000, 5, 6},
{10000, 50, 25},
}
testMessage := []byte("this is a random string for testing")
// random slice sizes to read and write
writeChunks := []int{25, 35, 15, 20}
readChunks := []int{5, 45, 20, 25}
for _, c := range cases {
// first pass: write directly to hash
hash := sha1.New()
for i := 0; i < c.iterations*c.writesPerLoop; i++ {
if _, err := hash.Write(testMessage[:writeChunks[i%len(writeChunks)]]); err != nil {
t.Fatal(err)
}
}
expected := hex.EncodeToString(hash.Sum(nil))
// write/read through buffer
buf := NewBytesPipe(nil)
hash.Reset()
for i := 0; i < c.iterations; i++ {
for w := 0; w < c.writesPerLoop; w++ {
buf.Write(testMessage[:writeChunks[(i*c.writesPerLoop+w)%len(writeChunks)]])
}
for r := 0; r < c.readsPerLoop; r++ {
p := make([]byte, readChunks[(i*c.readsPerLoop+r)%len(readChunks)])
n, _ := buf.Read(p)
hash.Write(p[:n])
}
}
// read rest of the data from buffer
for i := 0; ; i++ {
p := make([]byte, readChunks[(c.iterations*c.readsPerLoop+i)%len(readChunks)])
n, _ := buf.Read(p)
if n == 0 {
break
}
hash.Write(p[:n])
}
actual := hex.EncodeToString(hash.Sum(nil))
if expected != actual {
t.Fatalf("BytesPipe returned invalid data. Expected checksum %v, got %v", expected, actual)
}
}
}
func BenchmarkBytesPipeWrite(b *testing.B) {
for i := 0; i < b.N; i++ {
buf := NewBytesPipe(nil)
for j := 0; j < 1000; j++ {
buf.Write([]byte("pretty short line, because why not?"))
}
}
}
func BenchmarkBytesPipeRead(b *testing.B) {
rd := make([]byte, 1024)
for i := 0; i < b.N; i++ {
b.StopTimer()
buf := NewBytesPipe(nil)
for j := 0; j < 1000; j++ {
buf.Write(make([]byte, 1024))
}
b.StartTimer()
for j := 0; j < 1000; j++ {
if n, _ := buf.Read(rd); n != 1024 {
b.Fatalf("Wrong number of bytes: %d", n)
}
}
}
}

View file

@ -12,3 +12,11 @@ func FprintfIfNotEmpty(w io.Writer, format, value string) (int, error) {
}
return 0, nil
}
// FprintfIfTrue prints the boolean value if it's true
func FprintfIfTrue(w io.Writer, format string, ok bool) (int, error) {
if ok {
return fmt.Fprintf(w, format, ok)
}
return 0, nil
}

View file

@ -0,0 +1,226 @@
package ioutils
import (
"bytes"
"fmt"
"io"
"os"
)
type pos struct {
idx int
offset int64
}
type multiReadSeeker struct {
readers []io.ReadSeeker
pos *pos
posIdx map[io.ReadSeeker]int
}
func (r *multiReadSeeker) Seek(offset int64, whence int) (int64, error) {
var tmpOffset int64
switch whence {
case os.SEEK_SET:
for i, rdr := range r.readers {
// get size of the current reader
s, err := rdr.Seek(0, os.SEEK_END)
if err != nil {
return -1, err
}
if offset > tmpOffset+s {
if i == len(r.readers)-1 {
rdrOffset := s + (offset - tmpOffset)
if _, err := rdr.Seek(rdrOffset, os.SEEK_SET); err != nil {
return -1, err
}
r.pos = &pos{i, rdrOffset}
return offset, nil
}
tmpOffset += s
continue
}
rdrOffset := offset - tmpOffset
idx := i
rdr.Seek(rdrOffset, os.SEEK_SET)
// make sure all following readers are at 0
for _, rdr := range r.readers[i+1:] {
rdr.Seek(0, os.SEEK_SET)
}
if rdrOffset == s && i != len(r.readers)-1 {
idx++
rdrOffset = 0
}
r.pos = &pos{idx, rdrOffset}
return offset, nil
}
case os.SEEK_END:
for _, rdr := range r.readers {
s, err := rdr.Seek(0, os.SEEK_END)
if err != nil {
return -1, err
}
tmpOffset += s
}
r.Seek(tmpOffset+offset, os.SEEK_SET)
return tmpOffset + offset, nil
case os.SEEK_CUR:
if r.pos == nil {
return r.Seek(offset, os.SEEK_SET)
}
// Just return the current offset
if offset == 0 {
return r.getCurOffset()
}
curOffset, err := r.getCurOffset()
if err != nil {
return -1, err
}
rdr, rdrOffset, err := r.getReaderForOffset(curOffset + offset)
if err != nil {
return -1, err
}
r.pos = &pos{r.posIdx[rdr], rdrOffset}
return curOffset + offset, nil
default:
return -1, fmt.Errorf("Invalid whence: %d", whence)
}
return -1, fmt.Errorf("Error seeking for whence: %d, offset: %d", whence, offset)
}
func (r *multiReadSeeker) getReaderForOffset(offset int64) (io.ReadSeeker, int64, error) {
var rdr io.ReadSeeker
var rdrOffset int64
for i, rdr := range r.readers {
offsetTo, err := r.getOffsetToReader(rdr)
if err != nil {
return nil, -1, err
}
if offsetTo > offset {
rdr = r.readers[i-1]
rdrOffset = offsetTo - offset
break
}
if rdr == r.readers[len(r.readers)-1] {
rdrOffset = offsetTo + offset
break
}
}
return rdr, rdrOffset, nil
}
func (r *multiReadSeeker) getCurOffset() (int64, error) {
var totalSize int64
for _, rdr := range r.readers[:r.pos.idx+1] {
if r.posIdx[rdr] == r.pos.idx {
totalSize += r.pos.offset
break
}
size, err := getReadSeekerSize(rdr)
if err != nil {
return -1, fmt.Errorf("error getting seeker size: %v", err)
}
totalSize += size
}
return totalSize, nil
}
func (r *multiReadSeeker) getOffsetToReader(rdr io.ReadSeeker) (int64, error) {
var offset int64
for _, r := range r.readers {
if r == rdr {
break
}
size, err := getReadSeekerSize(rdr)
if err != nil {
return -1, err
}
offset += size
}
return offset, nil
}
func (r *multiReadSeeker) Read(b []byte) (int, error) {
if r.pos == nil {
r.pos = &pos{0, 0}
}
bCap := int64(cap(b))
buf := bytes.NewBuffer(nil)
var rdr io.ReadSeeker
for _, rdr = range r.readers[r.pos.idx:] {
readBytes, err := io.CopyN(buf, rdr, bCap)
if err != nil && err != io.EOF {
return -1, err
}
bCap -= readBytes
if bCap == 0 {
break
}
}
rdrPos, err := rdr.Seek(0, os.SEEK_CUR)
if err != nil {
return -1, err
}
r.pos = &pos{r.posIdx[rdr], rdrPos}
return buf.Read(b)
}
func getReadSeekerSize(rdr io.ReadSeeker) (int64, error) {
// save the current position
pos, err := rdr.Seek(0, os.SEEK_CUR)
if err != nil {
return -1, err
}
// get the size
size, err := rdr.Seek(0, os.SEEK_END)
if err != nil {
return -1, err
}
// reset the position
if _, err := rdr.Seek(pos, os.SEEK_SET); err != nil {
return -1, err
}
return size, nil
}
// MultiReadSeeker returns a ReadSeeker that's the logical concatenation of the provided
// input readseekers. After calling this method the initial position is set to the
// beginning of the first ReadSeeker. At the end of a ReadSeeker, Read always advances
// to the beginning of the next ReadSeeker and returns EOF at the end of the last ReadSeeker.
// Seek can be used over the sum of lengths of all readseekers.
//
// When a MultiReadSeeker is used, no Read and Seek operations should be made on
// its ReadSeeker components. Also, users should make no assumption on the state
// of individual readseekers while the MultiReadSeeker is used.
func MultiReadSeeker(readers ...io.ReadSeeker) io.ReadSeeker {
if len(readers) == 1 {
return readers[0]
}
idx := make(map[io.ReadSeeker]int)
for i, rdr := range readers {
idx[rdr] = i
}
return &multiReadSeeker{
readers: readers,
posIdx: idx,
}
}

View file

@ -0,0 +1,149 @@
package ioutils
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"testing"
)
func TestMultiReadSeekerReadAll(t *testing.T) {
str := "hello world"
s1 := strings.NewReader(str + " 1")
s2 := strings.NewReader(str + " 2")
s3 := strings.NewReader(str + " 3")
mr := MultiReadSeeker(s1, s2, s3)
expectedSize := int64(s1.Len() + s2.Len() + s3.Len())
b, err := ioutil.ReadAll(mr)
if err != nil {
t.Fatal(err)
}
expected := "hello world 1hello world 2hello world 3"
if string(b) != expected {
t.Fatalf("ReadAll failed, got: %q, expected %q", string(b), expected)
}
size, err := mr.Seek(0, os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if size != expectedSize {
t.Fatalf("reader size does not match, got %d, expected %d", size, expectedSize)
}
// Reset the position and read again
pos, err := mr.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if pos != 0 {
t.Fatalf("expected position to be set to 0, got %d", pos)
}
b, err = ioutil.ReadAll(mr)
if err != nil {
t.Fatal(err)
}
if string(b) != expected {
t.Fatalf("ReadAll failed, got: %q, expected %q", string(b), expected)
}
}
func TestMultiReadSeekerReadEach(t *testing.T) {
str := "hello world"
s1 := strings.NewReader(str + " 1")
s2 := strings.NewReader(str + " 2")
s3 := strings.NewReader(str + " 3")
mr := MultiReadSeeker(s1, s2, s3)
var totalBytes int64
for i, s := range []*strings.Reader{s1, s2, s3} {
sLen := int64(s.Len())
buf := make([]byte, s.Len())
expected := []byte(fmt.Sprintf("%s %d", str, i+1))
if _, err := mr.Read(buf); err != nil && err != io.EOF {
t.Fatal(err)
}
if !bytes.Equal(buf, expected) {
t.Fatalf("expected %q to be %q", string(buf), string(expected))
}
pos, err := mr.Seek(0, os.SEEK_CUR)
if err != nil {
t.Fatalf("iteration: %d, error: %v", i+1, err)
}
// check that the total bytes read is the current position of the seeker
totalBytes += sLen
if pos != totalBytes {
t.Fatalf("expected current position to be: %d, got: %d, iteration: %d", totalBytes, pos, i+1)
}
// This tests not only that SEEK_SET and SEEK_CUR give the same values, but that the next iteration is in the expected position as well
newPos, err := mr.Seek(pos, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
if newPos != pos {
t.Fatalf("expected to get same position when calling SEEK_SET with value from SEEK_CUR, cur: %d, set: %d", pos, newPos)
}
}
}
func TestMultiReadSeekerReadSpanningChunks(t *testing.T) {
str := "hello world"
s1 := strings.NewReader(str + " 1")
s2 := strings.NewReader(str + " 2")
s3 := strings.NewReader(str + " 3")
mr := MultiReadSeeker(s1, s2, s3)
buf := make([]byte, s1.Len()+3)
_, err := mr.Read(buf)
if err != nil {
t.Fatal(err)
}
// expected is the contents of s1 + 3 bytes from s2, ie, the `hel` at the end of this string
expected := "hello world 1hel"
if string(buf) != expected {
t.Fatalf("expected %s to be %s", string(buf), expected)
}
}
func TestMultiReadSeekerNegativeSeek(t *testing.T) {
str := "hello world"
s1 := strings.NewReader(str + " 1")
s2 := strings.NewReader(str + " 2")
s3 := strings.NewReader(str + " 3")
mr := MultiReadSeeker(s1, s2, s3)
s1Len := s1.Len()
s2Len := s2.Len()
s3Len := s3.Len()
s, err := mr.Seek(int64(-1*s3.Len()), os.SEEK_END)
if err != nil {
t.Fatal(err)
}
if s != int64(s1Len+s2Len) {
t.Fatalf("expected %d to be %d", s, s1.Len()+s2.Len())
}
buf := make([]byte, s3Len)
if _, err := mr.Read(buf); err != nil && err != io.EOF {
t.Fatal(err)
}
expected := fmt.Sprintf("%s %d", str, 3)
if string(buf) != fmt.Sprintf("%s %d", str, 3) {
t.Fatalf("expected %q to be %q", string(buf), expected)
}
}

View file

@ -1,14 +1,10 @@
package ioutils
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"io"
"math/big"
"sync"
"time"
)
type readCloserWrapper struct {
@ -20,6 +16,7 @@ func (r *readCloserWrapper) Close() error {
return r.closer()
}
// NewReadCloserWrapper returns a new io.ReadCloser.
func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser {
return &readCloserWrapper{
Reader: r,
@ -40,6 +37,7 @@ func (r *readerErrWrapper) Read(p []byte) (int, error) {
return n, err
}
// NewReaderErrWrapper returns a new io.Reader.
func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader {
return &readerErrWrapper{
reader: r,
@ -53,41 +51,27 @@ func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader {
// expanding buffer.
type bufReader struct {
sync.Mutex
buf *bytes.Buffer
reader io.Reader
err error
wait sync.Cond
drainBuf []byte
reuseBuf []byte
maxReuse int64
resetTimeout time.Duration
bufLenResetThreshold int64
maxReadDataReset int64
buf io.ReadWriter
reader io.Reader
err error
wait sync.Cond
drainBuf []byte
}
func NewBufReader(r io.Reader) *bufReader {
var timeout int
if randVal, err := rand.Int(rand.Reader, big.NewInt(120)); err == nil {
timeout = int(randVal.Int64()) + 180
} else {
timeout = 300
}
// NewBufReader returns a new bufReader.
func NewBufReader(r io.Reader) io.ReadCloser {
reader := &bufReader{
buf: &bytes.Buffer{},
drainBuf: make([]byte, 1024),
reuseBuf: make([]byte, 4096),
maxReuse: 1000,
resetTimeout: time.Second * time.Duration(timeout),
bufLenResetThreshold: 100 * 1024,
maxReadDataReset: 10 * 1024 * 1024,
reader: r,
buf: NewBytesPipe(nil),
reader: r,
drainBuf: make([]byte, 1024),
}
reader.wait.L = &reader.Mutex
go reader.drain()
return reader
}
func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer *bytes.Buffer) *bufReader {
// NewBufReaderWithDrainbufAndBuffer returns a BufReader with drainBuffer and buffer.
func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer io.ReadWriter) io.ReadCloser {
reader := &bufReader{
buf: buffer,
drainBuf: drainBuffer,
@ -99,94 +83,22 @@ func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer *
}
func (r *bufReader) drain() {
var (
duration time.Duration
lastReset time.Time
now time.Time
reset bool
bufLen int64
dataSinceReset int64
maxBufLen int64
reuseBufLen int64
reuseCount int64
)
reuseBufLen = int64(len(r.reuseBuf))
lastReset = time.Now()
for {
//Call to scheduler is made to yield from this goroutine.
//This avoids goroutine looping here when n=0,err=nil, fixes code hangs when run with GCC Go.
callSchedulerIfNecessary()
n, err := r.reader.Read(r.drainBuf)
dataSinceReset += int64(n)
r.Lock()
bufLen = int64(r.buf.Len())
if bufLen > maxBufLen {
maxBufLen = bufLen
}
// Avoid unbounded growth of the buffer over time.
// This has been discovered to be the only non-intrusive
// solution to the unbounded growth of the buffer.
// Alternative solutions such as compression, multiple
// buffers, channels and other similar pieces of code
// were reducing throughput, overall Docker performance
// or simply crashed Docker.
// This solution releases the buffer when specific
// conditions are met to avoid the continuous resizing
// of the buffer for long lived containers.
//
// Move data to the front of the buffer if it's
// smaller than what reuseBuf can store
if bufLen > 0 && reuseBufLen >= bufLen {
n, _ := r.buf.Read(r.reuseBuf)
r.buf.Write(r.reuseBuf[0:n])
// Take action if the buffer has been reused too many
// times and if there's data in the buffer.
// The timeout is also used as means to avoid doing
// these operations more often or less often than
// required.
// The various conditions try to detect heavy activity
// in the buffer which might be indicators of heavy
// growth of the buffer.
} else if reuseCount >= r.maxReuse && bufLen > 0 {
now = time.Now()
duration = now.Sub(lastReset)
timeoutReached := duration >= r.resetTimeout
// The timeout has been reached and the
// buffered data couldn't be moved to the front
// of the buffer, so the buffer gets reset.
if timeoutReached && bufLen > reuseBufLen {
reset = true
}
// The amount of buffered data is too high now,
// reset the buffer.
if timeoutReached && maxBufLen >= r.bufLenResetThreshold {
reset = true
}
// Reset the buffer if a certain amount of
// data has gone through the buffer since the
// last reset.
if timeoutReached && dataSinceReset >= r.maxReadDataReset {
reset = true
}
// The buffered data is moved to a fresh buffer,
// swap the old buffer with the new one and
// reset all counters.
if reset {
newbuf := &bytes.Buffer{}
newbuf.ReadFrom(r.buf)
r.buf = newbuf
lastReset = now
reset = false
dataSinceReset = 0
maxBufLen = 0
reuseCount = 0
}
}
if err != nil {
r.err = err
} else {
r.buf.Write(r.drainBuf[0:n])
if n == 0 {
// nothing written, no need to signal
r.Unlock()
continue
}
r.buf.Write(r.drainBuf[:n])
}
reuseCount++
r.wait.Signal()
r.Unlock()
if err != nil {
@ -210,6 +122,7 @@ func (r *bufReader) Read(p []byte) (n int, err error) {
}
}
// Close closes the bufReader
func (r *bufReader) Close() error {
closer, ok := r.reader.(io.ReadCloser)
if !ok {
@ -218,6 +131,7 @@ func (r *bufReader) Close() error {
return closer.Close()
}
// HashData returns the sha256 sum of src.
func HashData(src io.Reader) (string, error) {
h := sha256.New()
if _, err := io.Copy(h, src); err != nil {
@ -225,3 +139,32 @@ func HashData(src io.Reader) (string, error) {
}
return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
}
// OnEOFReader wraps a io.ReadCloser and a function
// the function will run at the end of file or close the file.
type OnEOFReader struct {
Rc io.ReadCloser
Fn func()
}
func (r *OnEOFReader) Read(p []byte) (n int, err error) {
n, err = r.Rc.Read(p)
if err == io.EOF {
r.runFunc()
}
return
}
// Close closes the file and run the function.
func (r *OnEOFReader) Close() error {
err := r.Rc.Close()
r.runFunc()
return err
}
func (r *OnEOFReader) runFunc() {
if fn := r.Fn; fn != nil {
fn()
r.Fn = nil
}
}

View file

@ -7,6 +7,7 @@ import (
"io/ioutil"
"strings"
"testing"
"time"
)
// Implement io.Reader
@ -45,7 +46,7 @@ func TestReaderErrWrapperReadOnError(t *testing.T) {
func TestReaderErrWrapperRead(t *testing.T) {
reader := strings.NewReader("a string reader.")
wrapper := NewReaderErrWrapper(reader, func() {
t.Fatalf("readErrWrapper should not have called the anonymous function on failure")
t.Fatalf("readErrWrapper should not have called the anonymous function")
})
// Read 20 byte (should be ok with the string above)
num, err := wrapper.Read(make([]byte, 20))
@ -61,8 +62,8 @@ func TestNewBufReaderWithDrainbufAndBuffer(t *testing.T) {
reader, writer := io.Pipe()
drainBuffer := make([]byte, 1024)
buffer := bytes.Buffer{}
bufreader := NewBufReaderWithDrainbufAndBuffer(reader, drainBuffer, &buffer)
buffer := NewBytesPipe(nil)
bufreader := NewBufReaderWithDrainbufAndBuffer(reader, drainBuffer, buffer)
// Write everything down to a Pipe
// Usually, a pipe should block but because of the buffered reader,
@ -76,7 +77,11 @@ func TestNewBufReaderWithDrainbufAndBuffer(t *testing.T) {
// Drain the reader *after* everything has been written, just to verify
// it is indeed buffering
<-done
select {
case <-done:
case <-time.After(1 * time.Second):
t.Fatal("timeout")
}
output, err := ioutil.ReadAll(bufreader)
if err != nil {
@ -124,13 +129,16 @@ func TestBufReaderCloseWithNonReaderCloser(t *testing.T) {
}
// implements io.ReadCloser
type simpleReaderCloser struct{}
type simpleReaderCloser struct {
err error
}
func (r *simpleReaderCloser) Read(p []byte) (n int, err error) {
return 0, nil
return 0, r.err
}
func (r *simpleReaderCloser) Close() error {
r.err = io.EOF
return nil
}

View file

@ -0,0 +1,6 @@
// +build !gccgo
package ioutils
func callSchedulerIfNecessary() {
}

View file

@ -0,0 +1,13 @@
// +build gccgo
package ioutils
import (
"runtime"
)
func callSchedulerIfNecessary() {
//allow or force Go scheduler to switch context, without explicitly
//forcing this will make it hang when using gccgo implementation
runtime.Gosched()
}

View file

@ -6,6 +6,7 @@ import (
"sync"
)
// WriteFlusher wraps the Write and Flush operation.
type WriteFlusher struct {
sync.Mutex
w io.Writer
@ -30,12 +31,15 @@ func (wf *WriteFlusher) Flush() {
wf.flusher.Flush()
}
// Flushed returns the state of flushed.
// If it's flushed, return true, or else it return false.
func (wf *WriteFlusher) Flushed() bool {
wf.Lock()
defer wf.Unlock()
return wf.flushed
}
// NewWriteFlusher returns a new WriteFlusher.
func NewWriteFlusher(w io.Writer) *WriteFlusher {
var flusher http.Flusher
if f, ok := w.(http.Flusher); ok {

View file

@ -2,6 +2,7 @@ package ioutils
import "io"
// NopWriter represents a type which write operation is nop.
type NopWriter struct{}
func (*NopWriter) Write(buf []byte) (int, error) {
@ -14,12 +15,15 @@ type nopWriteCloser struct {
func (w *nopWriteCloser) Close() error { return nil }
// NopWriteCloser returns a nopWriteCloser.
func NopWriteCloser(w io.Writer) io.WriteCloser {
return &nopWriteCloser{w}
}
// NopFlusher represents a type which flush opetatin is nop.
type NopFlusher struct{}
// Flush is a nop operation.
func (f *NopFlusher) Flush() {}
type writeCloserWrapper struct {
@ -31,6 +35,7 @@ func (r *writeCloserWrapper) Close() error {
return r.closer()
}
// NewWriteCloserWrapper returns a new io.WriteCloser.
func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser {
return &writeCloserWrapper{
Writer: r,
@ -38,7 +43,7 @@ func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser {
}
}
// Wrap a concrete io.Writer and hold a count of the number
// WriteCounter wraps a concrete io.Writer and hold a count of the number
// of bytes written to the writer during a "session".
// This can be convenient when write return is masked
// (e.g., json.Encoder.Encode())
@ -47,6 +52,7 @@ type WriteCounter struct {
Writer io.Writer
}
// NewWriteCounter returns a new WriteCounter.
func NewWriteCounter(w io.Writer) *WriteCounter {
return &WriteCounter{
Writer: w,

View file

@ -0,0 +1,27 @@
# listenbuffer
listenbuffer uses the kernel's listening backlog functionality to queue
connections, allowing applications to start listening immediately and handle
connections later. This is signaled by closing the activation channel passed to
the constructor.
The maximum amount of queued connections depends on the configuration of your
kernel (typically called SOMAXXCON) and cannot be configured in Go with the
net package. See `src/net/sock_platform.go` in the Go tree or consult your
kernel's manual.
activator := make(chan struct{})
buffer, err := NewListenBuffer("tcp", "localhost:4000", activator)
if err != nil {
panic(err)
}
// will block until activator has been closed or is sent an event
client, err := buffer.Accept()
Somewhere else in your application once it's been booted:
close(activator)
`buffer.Accept()` will return the first client in the kernel listening queue, or
continue to block until a client connects or an error occurs.

View file

@ -0,0 +1,76 @@
/*
Package listenbuffer uses the kernel's listening backlog functionality to queue
connections, allowing applications to start listening immediately and handle
connections later. This is signaled by closing the activation channel passed to
the constructor.
The maximum amount of queued connections depends on the configuration of your
kernel (typically called SOMAXXCON) and cannot be configured in Go with the
net package. See `src/net/sock_platform.go` in the Go tree or consult your
kernel's manual.
activator := make(chan struct{})
buffer, err := NewListenBuffer("tcp", "localhost:4000", activator)
if err != nil {
panic(err)
}
// will block until activator has been closed or is sent an event
client, err := buffer.Accept()
Somewhere else in your application once it's been booted:
close(activator)
`buffer.Accept()` will return the first client in the kernel listening queue, or
continue to block until a client connects or an error occurs.
*/
package listenbuffer
import "net"
// NewListenBuffer returns a net.Listener listening on addr with the protocol
// passed. The channel passed is used to activate the listenbuffer when the
// caller is ready to accept connections.
func NewListenBuffer(proto, addr string, activate <-chan struct{}) (net.Listener, error) {
wrapped, err := net.Listen(proto, addr)
if err != nil {
return nil, err
}
return &defaultListener{
wrapped: wrapped,
activate: activate,
}, nil
}
// defaultListener is the buffered wrapper around the net.Listener
type defaultListener struct {
wrapped net.Listener // The net.Listener wrapped by listenbuffer
ready bool // Whether the listenbuffer has been activated
activate <-chan struct{} // Channel to control activation of the listenbuffer
}
// Close closes the wrapped socket.
func (l *defaultListener) Close() error {
return l.wrapped.Close()
}
// Addr returns the listening address of the wrapped socket.
func (l *defaultListener) Addr() net.Addr {
return l.wrapped.Addr()
}
// Accept returns a client connection on the wrapped socket if the listen buffer
// has been activated. To active the listenbuffer the activation channel passed
// to NewListenBuffer must have been closed or sent an event.
func (l *defaultListener) Accept() (net.Conn, error) {
// if the listen has been told it is ready then we can go ahead and
// start returning connections
if l.ready {
return l.wrapped.Accept()
}
<-l.activate
l.ready = true
return l.Accept()
}

View file

@ -0,0 +1,41 @@
package listenbuffer
import (
"io/ioutil"
"net"
"testing"
)
func TestListenBufferAllowsAcceptingWhenActivated(t *testing.T) {
lock := make(chan struct{})
buffer, err := NewListenBuffer("tcp", "", lock)
if err != nil {
t.Fatal("Unable to create listen buffer: ", err)
}
go func() {
conn, err := net.Dial("tcp", buffer.Addr().String())
if err != nil {
t.Fatal("Client failed to establish connection to server: ", err)
}
conn.Write([]byte("ping"))
conn.Close()
}()
close(lock)
client, err := buffer.Accept()
if err != nil {
t.Fatal("Failed to accept client: ", err)
}
response, err := ioutil.ReadAll(client)
if err != nil {
t.Fatal("Failed to read from client: ", err)
}
if string(response) != "ping" {
t.Fatal("Expected to receive ping from client, received: ", string(response))
}
}

View file

@ -2,83 +2,82 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package flag implements command-line flag parsing.
// Package mflag implements command-line flag parsing.
//
// Usage:
//
// Define flags using flag.String(), Bool(), Int(), etc.
//
// This declares an integer flag, -f or --flagname, stored in the pointer ip, with type *int.
// import "flag /github.com/docker/docker/pkg/mflag"
// var ip = flag.Int([]string{"f", "-flagname"}, 1234, "help message for flagname")
// If you like, you can bind the flag to a variable using the Var() functions.
// var flagvar int
// func init() {
// // -flaghidden will work, but will be hidden from the usage
// flag.IntVar(&flagvar, []string{"f", "#flaghidden", "-flagname"}, 1234, "help message for flagname")
// }
// Or you can create custom flags that satisfy the Value interface (with
// pointer receivers) and couple them to flag parsing by
// flag.Var(&flagVal, []string{"name"}, "help message for flagname")
// For such flags, the default value is just the initial value of the variable.
//
// You can also add "deprecated" flags, they are still usable, but are not shown
// in the usage and will display a warning when you try to use them. `#` before
// an option means this option is deprecated, if there is an following option
// without `#` ahead, then that's the replacement, if not, it will just be removed:
// var ip = flag.Int([]string{"#f", "#flagname", "-flagname"}, 1234, "help message for flagname")
// this will display: `Warning: '-f' is deprecated, it will be replaced by '--flagname' soon. See usage.` or
// this will display: `Warning: '-flagname' is deprecated, it will be replaced by '--flagname' soon. See usage.`
// var ip = flag.Int([]string{"f", "#flagname"}, 1234, "help message for flagname")
// will display: `Warning: '-flagname' is deprecated, it will be removed soon. See usage.`
// so you can only use `-f`.
//
// You can also group one letter flags, bif you declare
// var v = flag.Bool([]string{"v", "-verbose"}, false, "help message for verbose")
// var s = flag.Bool([]string{"s", "-slow"}, false, "help message for slow")
// you will be able to use the -vs or -sv
//
// After all flags are defined, call
// flag.Parse()
// to parse the command line into the defined flags.
//
// Flags may then be used directly. If you're using the flags themselves,
// they are all pointers; if you bind to variables, they're values.
// fmt.Println("ip has value ", *ip)
// fmt.Println("flagvar has value ", flagvar)
//
// After parsing, the arguments after the flag are available as the
// slice flag.Args() or individually as flag.Arg(i).
// The arguments are indexed from 0 through flag.NArg()-1.
//
// Command line flag syntax:
// -flag
// -flag=x
// -flag="x"
// -flag='x'
// -flag x // non-boolean flags only
// One or two minus signs may be used; they are equivalent.
// The last form is not permitted for boolean flags because the
// meaning of the command
// cmd -x *
// will change if there is a file called 0, false, etc. You must
// use the -flag=false form to turn off a boolean flag.
//
// Flag parsing stops just before the first non-flag argument
// ("-" is a non-flag argument) or after the terminator "--".
//
// Integer flags accept 1234, 0664, 0x1234 and may be negative.
// Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False.
// Duration flags accept any input valid for time.ParseDuration.
//
// The default set of command-line flags is controlled by
// top-level functions. The FlagSet type allows one to define
// independent sets of flags, such as to implement subcommands
// in a command-line interface. The methods of FlagSet are
// analogous to the top-level functions for the command-line
// flag set.
Usage:
Define flags using flag.String(), Bool(), Int(), etc.
This declares an integer flag, -f or --flagname, stored in the pointer ip, with type *int.
import "flag /github.com/docker/docker/pkg/mflag"
var ip = flag.Int([]string{"f", "-flagname"}, 1234, "help message for flagname")
If you like, you can bind the flag to a variable using the Var() functions.
var flagvar int
func init() {
// -flaghidden will work, but will be hidden from the usage
flag.IntVar(&flagvar, []string{"f", "#flaghidden", "-flagname"}, 1234, "help message for flagname")
}
Or you can create custom flags that satisfy the Value interface (with
pointer receivers) and couple them to flag parsing by
flag.Var(&flagVal, []string{"name"}, "help message for flagname")
For such flags, the default value is just the initial value of the variable.
You can also add "deprecated" flags, they are still usable, but are not shown
in the usage and will display a warning when you try to use them. `#` before
an option means this option is deprecated, if there is an following option
without `#` ahead, then that's the replacement, if not, it will just be removed:
var ip = flag.Int([]string{"#f", "#flagname", "-flagname"}, 1234, "help message for flagname")
this will display: `Warning: '-f' is deprecated, it will be replaced by '--flagname' soon. See usage.` or
this will display: `Warning: '-flagname' is deprecated, it will be replaced by '--flagname' soon. See usage.`
var ip = flag.Int([]string{"f", "#flagname"}, 1234, "help message for flagname")
will display: `Warning: '-flagname' is deprecated, it will be removed soon. See usage.`
so you can only use `-f`.
You can also group one letter flags, bif you declare
var v = flag.Bool([]string{"v", "-verbose"}, false, "help message for verbose")
var s = flag.Bool([]string{"s", "-slow"}, false, "help message for slow")
you will be able to use the -vs or -sv
After all flags are defined, call
flag.Parse()
to parse the command line into the defined flags.
Flags may then be used directly. If you're using the flags themselves,
they are all pointers; if you bind to variables, they're values.
fmt.Println("ip has value ", *ip)
fmt.Println("flagvar has value ", flagvar)
After parsing, the arguments after the flag are available as the
slice flag.Args() or individually as flag.Arg(i).
The arguments are indexed from 0 through flag.NArg()-1.
Command line flag syntax:
-flag
-flag=x
-flag="x"
-flag='x'
-flag x // non-boolean flags only
One or two minus signs may be used; they are equivalent.
The last form is not permitted for boolean flags because the
meaning of the command
cmd -x *
will change if there is a file called 0, false, etc. You must
use the -flag=false form to turn off a boolean flag.
Flag parsing stops just before the first non-flag argument
("-" is a non-flag argument) or after the terminator "--".
Integer flags accept 1234, 0664, 0x1234 and may be negative.
Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False.
Duration flags accept any input valid for time.ParseDuration.
The default set of command-line flags is controlled by
top-level functions. The FlagSet type allows one to define
independent sets of flags, such as to implement subcommands
in a command-line interface. The methods of FlagSet are
analogous to the top-level functions for the command-line
flag set.
*/
package mflag
import (
@ -277,6 +276,7 @@ type Getter interface {
// ErrorHandling defines how to handle flag parsing errors.
type ErrorHandling int
// ErrorHandling strategies available when a flag parsing error occurs
const (
ContinueOnError ErrorHandling = iota
ExitOnError
@ -358,28 +358,28 @@ func sortFlags(flags map[string]*Flag) []*Flag {
}
// Name returns the name of the FlagSet.
func (f *FlagSet) Name() string {
return f.name
func (fs *FlagSet) Name() string {
return fs.name
}
// Out returns the destination for usage and error messages.
func (f *FlagSet) Out() io.Writer {
if f.output == nil {
func (fs *FlagSet) Out() io.Writer {
if fs.output == nil {
return os.Stderr
}
return f.output
return fs.output
}
// SetOutput sets the destination for usage and error messages.
// If output is nil, os.Stderr is used.
func (f *FlagSet) SetOutput(output io.Writer) {
f.output = output
func (fs *FlagSet) SetOutput(output io.Writer) {
fs.output = output
}
// VisitAll visits the flags in lexicographical order, calling fn for each.
// It visits all flags, even those not set.
func (f *FlagSet) VisitAll(fn func(*Flag)) {
for _, flag := range sortFlags(f.formal) {
func (fs *FlagSet) VisitAll(fn func(*Flag)) {
for _, flag := range sortFlags(fs.formal) {
fn(flag)
}
}
@ -392,8 +392,8 @@ func VisitAll(fn func(*Flag)) {
// Visit visits the flags in lexicographical order, calling fn for each.
// It visits only those flags that have been set.
func (f *FlagSet) Visit(fn func(*Flag)) {
for _, flag := range sortFlags(f.actual) {
func (fs *FlagSet) Visit(fn func(*Flag)) {
for _, flag := range sortFlags(fs.actual) {
fn(flag)
}
}
@ -405,13 +405,13 @@ func Visit(fn func(*Flag)) {
}
// Lookup returns the Flag structure of the named flag, returning nil if none exists.
func (f *FlagSet) Lookup(name string) *Flag {
return f.formal[name]
func (fs *FlagSet) Lookup(name string) *Flag {
return fs.formal[name]
}
// Indicates whether the specified flag was specified at all on the cmd line
func (f *FlagSet) IsSet(name string) bool {
return f.actual[name] != nil
// IsSet indicates whether the specified flag is set in the given FlagSet
func (fs *FlagSet) IsSet(name string) bool {
return fs.actual[name] != nil
}
// Lookup returns the Flag structure of the named command-line flag,
@ -420,7 +420,7 @@ func Lookup(name string) *Flag {
return CommandLine.formal[name]
}
// Indicates whether the specified flag was specified at all on the cmd line
// IsSet indicates whether the specified flag was specified at all on the cmd line.
func IsSet(name string) bool {
return CommandLine.IsSet(name)
}
@ -443,15 +443,15 @@ type nArgRequirement struct {
// The first parameter can be Exact, Max, or Min to respectively specify the exact,
// the maximum, or the minimal number of arguments required.
// The actual check is done in FlagSet.CheckArgs().
func (f *FlagSet) Require(nArgRequirementType nArgRequirementType, nArg int) {
f.nArgRequirements = append(f.nArgRequirements, nArgRequirement{nArgRequirementType, nArg})
func (fs *FlagSet) Require(nArgRequirementType nArgRequirementType, nArg int) {
fs.nArgRequirements = append(fs.nArgRequirements, nArgRequirement{nArgRequirementType, nArg})
}
// CheckArgs uses the requirements set by FlagSet.Require() to validate
// the number of arguments. If the requirements are not met,
// an error message string is returned.
func (f *FlagSet) CheckArgs() (message string) {
for _, req := range f.nArgRequirements {
func (fs *FlagSet) CheckArgs() (message string) {
for _, req := range fs.nArgRequirements {
var arguments string
if req.N == 1 {
arguments = "1 argument"
@ -460,20 +460,20 @@ func (f *FlagSet) CheckArgs() (message string) {
}
str := func(kind string) string {
return fmt.Sprintf("%q requires %s%s", f.name, kind, arguments)
return fmt.Sprintf("%q requires %s%s", fs.name, kind, arguments)
}
switch req.Type {
case Exact:
if f.NArg() != req.N {
if fs.NArg() != req.N {
return str("")
}
case Max:
if f.NArg() > req.N {
if fs.NArg() > req.N {
return str("a maximum of ")
}
case Min:
if f.NArg() < req.N {
if fs.NArg() < req.N {
return str("a minimum of ")
}
}
@ -482,18 +482,18 @@ func (f *FlagSet) CheckArgs() (message string) {
}
// Set sets the value of the named flag.
func (f *FlagSet) Set(name, value string) error {
flag, ok := f.formal[name]
func (fs *FlagSet) Set(name, value string) error {
flag, ok := fs.formal[name]
if !ok {
return fmt.Errorf("no such flag -%v", name)
}
if err := flag.Value.Set(value); err != nil {
return err
}
if f.actual == nil {
f.actual = make(map[string]*Flag)
if fs.actual == nil {
fs.actual = make(map[string]*Flag)
}
f.actual[name] = flag
fs.actual[name] = flag
return nil
}
@ -504,8 +504,8 @@ func Set(name, value string) error {
// PrintDefaults prints, to standard error unless configured
// otherwise, the default values of all defined flags in the set.
func (f *FlagSet) PrintDefaults() {
writer := tabwriter.NewWriter(f.Out(), 20, 1, 3, ' ', 0)
func (fs *FlagSet) PrintDefaults() {
writer := tabwriter.NewWriter(fs.Out(), 20, 1, 3, ' ', 0)
home := homedir.Get()
// Don't substitute when HOME is /
@ -514,11 +514,11 @@ func (f *FlagSet) PrintDefaults() {
}
// Add a blank line between cmd description and list of options
if f.FlagCount() > 0 {
if fs.FlagCount() > 0 {
fmt.Fprintln(writer, "")
}
f.VisitAll(func(flag *Flag) {
fs.VisitAll(func(flag *Flag) {
format := " -%s=%s"
names := []string{}
for _, name := range flag.Names {
@ -526,7 +526,7 @@ func (f *FlagSet) PrintDefaults() {
names = append(names, name)
}
}
if len(names) > 0 {
if len(names) > 0 && len(flag.Usage) > 0 {
val := flag.DefValue
if home != "" && strings.HasPrefix(val, home) {
@ -551,13 +551,13 @@ func PrintDefaults() {
}
// defaultUsage is the default function to print a usage message.
func defaultUsage(f *FlagSet) {
if f.name == "" {
fmt.Fprintf(f.Out(), "Usage:\n")
func defaultUsage(fs *FlagSet) {
if fs.name == "" {
fmt.Fprintf(fs.Out(), "Usage:\n")
} else {
fmt.Fprintf(f.Out(), "Usage of %s:\n", f.name)
fmt.Fprintf(fs.Out(), "Usage of %s:\n", fs.name)
}
f.PrintDefaults()
fs.PrintDefaults()
}
// NOTE: Usage is not just defaultUsage(CommandLine)
@ -578,12 +578,12 @@ var ShortUsage = func() {
}
// FlagCount returns the number of flags that have been defined.
func (f *FlagSet) FlagCount() int { return len(sortFlags(f.formal)) }
func (fs *FlagSet) FlagCount() int { return len(sortFlags(fs.formal)) }
// FlagCountUndeprecated returns the number of undeprecated flags that have been defined.
func (f *FlagSet) FlagCountUndeprecated() int {
func (fs *FlagSet) FlagCountUndeprecated() int {
count := 0
for _, flag := range sortFlags(f.formal) {
for _, flag := range sortFlags(fs.formal) {
for _, name := range flag.Names {
if name[0] != '#' {
count++
@ -595,18 +595,18 @@ func (f *FlagSet) FlagCountUndeprecated() int {
}
// NFlag returns the number of flags that have been set.
func (f *FlagSet) NFlag() int { return len(f.actual) }
func (fs *FlagSet) NFlag() int { return len(fs.actual) }
// NFlag returns the number of command-line flags that have been set.
func NFlag() int { return len(CommandLine.actual) }
// Arg returns the i'th argument. Arg(0) is the first remaining argument
// after flags have been processed.
func (f *FlagSet) Arg(i int) string {
if i < 0 || i >= len(f.args) {
func (fs *FlagSet) Arg(i int) string {
if i < 0 || i >= len(fs.args) {
return ""
}
return f.args[i]
return fs.args[i]
}
// Arg returns the i'th command-line argument. Arg(0) is the first remaining argument
@ -616,21 +616,21 @@ func Arg(i int) string {
}
// NArg is the number of arguments remaining after flags have been processed.
func (f *FlagSet) NArg() int { return len(f.args) }
func (fs *FlagSet) NArg() int { return len(fs.args) }
// NArg is the number of arguments remaining after flags have been processed.
func NArg() int { return len(CommandLine.args) }
// Args returns the non-flag arguments.
func (f *FlagSet) Args() []string { return f.args }
func (fs *FlagSet) Args() []string { return fs.args }
// Args returns the non-flag command-line arguments.
func Args() []string { return CommandLine.args }
// BoolVar defines a bool flag with specified name, default value, and usage string.
// The argument p points to a bool variable in which to store the value of the flag.
func (f *FlagSet) BoolVar(p *bool, names []string, value bool, usage string) {
f.Var(newBoolValue(value, p), names, usage)
func (fs *FlagSet) BoolVar(p *bool, names []string, value bool, usage string) {
fs.Var(newBoolValue(value, p), names, usage)
}
// BoolVar defines a bool flag with specified name, default value, and usage string.
@ -641,9 +641,9 @@ func BoolVar(p *bool, names []string, value bool, usage string) {
// Bool defines a bool flag with specified name, default value, and usage string.
// The return value is the address of a bool variable that stores the value of the flag.
func (f *FlagSet) Bool(names []string, value bool, usage string) *bool {
func (fs *FlagSet) Bool(names []string, value bool, usage string) *bool {
p := new(bool)
f.BoolVar(p, names, value, usage)
fs.BoolVar(p, names, value, usage)
return p
}
@ -655,8 +655,8 @@ func Bool(names []string, value bool, usage string) *bool {
// IntVar defines an int flag with specified name, default value, and usage string.
// The argument p points to an int variable in which to store the value of the flag.
func (f *FlagSet) IntVar(p *int, names []string, value int, usage string) {
f.Var(newIntValue(value, p), names, usage)
func (fs *FlagSet) IntVar(p *int, names []string, value int, usage string) {
fs.Var(newIntValue(value, p), names, usage)
}
// IntVar defines an int flag with specified name, default value, and usage string.
@ -667,9 +667,9 @@ func IntVar(p *int, names []string, value int, usage string) {
// Int defines an int flag with specified name, default value, and usage string.
// The return value is the address of an int variable that stores the value of the flag.
func (f *FlagSet) Int(names []string, value int, usage string) *int {
func (fs *FlagSet) Int(names []string, value int, usage string) *int {
p := new(int)
f.IntVar(p, names, value, usage)
fs.IntVar(p, names, value, usage)
return p
}
@ -681,8 +681,8 @@ func Int(names []string, value int, usage string) *int {
// Int64Var defines an int64 flag with specified name, default value, and usage string.
// The argument p points to an int64 variable in which to store the value of the flag.
func (f *FlagSet) Int64Var(p *int64, names []string, value int64, usage string) {
f.Var(newInt64Value(value, p), names, usage)
func (fs *FlagSet) Int64Var(p *int64, names []string, value int64, usage string) {
fs.Var(newInt64Value(value, p), names, usage)
}
// Int64Var defines an int64 flag with specified name, default value, and usage string.
@ -693,9 +693,9 @@ func Int64Var(p *int64, names []string, value int64, usage string) {
// Int64 defines an int64 flag with specified name, default value, and usage string.
// The return value is the address of an int64 variable that stores the value of the flag.
func (f *FlagSet) Int64(names []string, value int64, usage string) *int64 {
func (fs *FlagSet) Int64(names []string, value int64, usage string) *int64 {
p := new(int64)
f.Int64Var(p, names, value, usage)
fs.Int64Var(p, names, value, usage)
return p
}
@ -707,8 +707,8 @@ func Int64(names []string, value int64, usage string) *int64 {
// UintVar defines a uint flag with specified name, default value, and usage string.
// The argument p points to a uint variable in which to store the value of the flag.
func (f *FlagSet) UintVar(p *uint, names []string, value uint, usage string) {
f.Var(newUintValue(value, p), names, usage)
func (fs *FlagSet) UintVar(p *uint, names []string, value uint, usage string) {
fs.Var(newUintValue(value, p), names, usage)
}
// UintVar defines a uint flag with specified name, default value, and usage string.
@ -719,9 +719,9 @@ func UintVar(p *uint, names []string, value uint, usage string) {
// Uint defines a uint flag with specified name, default value, and usage string.
// The return value is the address of a uint variable that stores the value of the flag.
func (f *FlagSet) Uint(names []string, value uint, usage string) *uint {
func (fs *FlagSet) Uint(names []string, value uint, usage string) *uint {
p := new(uint)
f.UintVar(p, names, value, usage)
fs.UintVar(p, names, value, usage)
return p
}
@ -733,8 +733,8 @@ func Uint(names []string, value uint, usage string) *uint {
// Uint64Var defines a uint64 flag with specified name, default value, and usage string.
// The argument p points to a uint64 variable in which to store the value of the flag.
func (f *FlagSet) Uint64Var(p *uint64, names []string, value uint64, usage string) {
f.Var(newUint64Value(value, p), names, usage)
func (fs *FlagSet) Uint64Var(p *uint64, names []string, value uint64, usage string) {
fs.Var(newUint64Value(value, p), names, usage)
}
// Uint64Var defines a uint64 flag with specified name, default value, and usage string.
@ -745,9 +745,9 @@ func Uint64Var(p *uint64, names []string, value uint64, usage string) {
// Uint64 defines a uint64 flag with specified name, default value, and usage string.
// The return value is the address of a uint64 variable that stores the value of the flag.
func (f *FlagSet) Uint64(names []string, value uint64, usage string) *uint64 {
func (fs *FlagSet) Uint64(names []string, value uint64, usage string) *uint64 {
p := new(uint64)
f.Uint64Var(p, names, value, usage)
fs.Uint64Var(p, names, value, usage)
return p
}
@ -759,8 +759,8 @@ func Uint64(names []string, value uint64, usage string) *uint64 {
// StringVar defines a string flag with specified name, default value, and usage string.
// The argument p points to a string variable in which to store the value of the flag.
func (f *FlagSet) StringVar(p *string, names []string, value string, usage string) {
f.Var(newStringValue(value, p), names, usage)
func (fs *FlagSet) StringVar(p *string, names []string, value string, usage string) {
fs.Var(newStringValue(value, p), names, usage)
}
// StringVar defines a string flag with specified name, default value, and usage string.
@ -771,9 +771,9 @@ func StringVar(p *string, names []string, value string, usage string) {
// String defines a string flag with specified name, default value, and usage string.
// The return value is the address of a string variable that stores the value of the flag.
func (f *FlagSet) String(names []string, value string, usage string) *string {
func (fs *FlagSet) String(names []string, value string, usage string) *string {
p := new(string)
f.StringVar(p, names, value, usage)
fs.StringVar(p, names, value, usage)
return p
}
@ -785,8 +785,8 @@ func String(names []string, value string, usage string) *string {
// Float64Var defines a float64 flag with specified name, default value, and usage string.
// The argument p points to a float64 variable in which to store the value of the flag.
func (f *FlagSet) Float64Var(p *float64, names []string, value float64, usage string) {
f.Var(newFloat64Value(value, p), names, usage)
func (fs *FlagSet) Float64Var(p *float64, names []string, value float64, usage string) {
fs.Var(newFloat64Value(value, p), names, usage)
}
// Float64Var defines a float64 flag with specified name, default value, and usage string.
@ -797,9 +797,9 @@ func Float64Var(p *float64, names []string, value float64, usage string) {
// Float64 defines a float64 flag with specified name, default value, and usage string.
// The return value is the address of a float64 variable that stores the value of the flag.
func (f *FlagSet) Float64(names []string, value float64, usage string) *float64 {
func (fs *FlagSet) Float64(names []string, value float64, usage string) *float64 {
p := new(float64)
f.Float64Var(p, names, value, usage)
fs.Float64Var(p, names, value, usage)
return p
}
@ -811,8 +811,8 @@ func Float64(names []string, value float64, usage string) *float64 {
// DurationVar defines a time.Duration flag with specified name, default value, and usage string.
// The argument p points to a time.Duration variable in which to store the value of the flag.
func (f *FlagSet) DurationVar(p *time.Duration, names []string, value time.Duration, usage string) {
f.Var(newDurationValue(value, p), names, usage)
func (fs *FlagSet) DurationVar(p *time.Duration, names []string, value time.Duration, usage string) {
fs.Var(newDurationValue(value, p), names, usage)
}
// DurationVar defines a time.Duration flag with specified name, default value, and usage string.
@ -823,9 +823,9 @@ func DurationVar(p *time.Duration, names []string, value time.Duration, usage st
// Duration defines a time.Duration flag with specified name, default value, and usage string.
// The return value is the address of a time.Duration variable that stores the value of the flag.
func (f *FlagSet) Duration(names []string, value time.Duration, usage string) *time.Duration {
func (fs *FlagSet) Duration(names []string, value time.Duration, usage string) *time.Duration {
p := new(time.Duration)
f.DurationVar(p, names, value, usage)
fs.DurationVar(p, names, value, usage)
return p
}
@ -841,26 +841,26 @@ func Duration(names []string, value time.Duration, usage string) *time.Duration
// caller could create a flag that turns a comma-separated string into a slice
// of strings by giving the slice the methods of Value; in particular, Set would
// decompose the comma-separated string into the slice.
func (f *FlagSet) Var(value Value, names []string, usage string) {
func (fs *FlagSet) Var(value Value, names []string, usage string) {
// Remember the default value as a string; it won't change.
flag := &Flag{names, usage, value, value.String()}
for _, name := range names {
name = strings.TrimPrefix(name, "#")
_, alreadythere := f.formal[name]
_, alreadythere := fs.formal[name]
if alreadythere {
var msg string
if f.name == "" {
if fs.name == "" {
msg = fmt.Sprintf("flag redefined: %s", name)
} else {
msg = fmt.Sprintf("%s flag redefined: %s", f.name, name)
msg = fmt.Sprintf("%s flag redefined: %s", fs.name, name)
}
fmt.Fprintln(f.Out(), msg)
fmt.Fprintln(fs.Out(), msg)
panic(msg) // Happens only if flags are declared with identical names
}
if f.formal == nil {
f.formal = make(map[string]*Flag)
if fs.formal == nil {
fs.formal = make(map[string]*Flag)
}
f.formal[name] = flag
fs.formal[name] = flag
}
}
@ -876,26 +876,26 @@ func Var(value Value, names []string, usage string) {
// failf prints to standard error a formatted error and usage message and
// returns the error.
func (f *FlagSet) failf(format string, a ...interface{}) error {
func (fs *FlagSet) failf(format string, a ...interface{}) error {
err := fmt.Errorf(format, a...)
fmt.Fprintln(f.Out(), err)
if os.Args[0] == f.name {
fmt.Fprintf(f.Out(), "See '%s --help'.\n", os.Args[0])
fmt.Fprintln(fs.Out(), err)
if os.Args[0] == fs.name {
fmt.Fprintf(fs.Out(), "See '%s --help'.\n", os.Args[0])
} else {
fmt.Fprintf(f.Out(), "See '%s %s --help'.\n", os.Args[0], f.name)
fmt.Fprintf(fs.Out(), "See '%s %s --help'.\n", os.Args[0], fs.name)
}
return err
}
// usage calls the Usage method for the flag set, or the usage function if
// the flag set is CommandLine.
func (f *FlagSet) usage() {
if f == CommandLine {
func (fs *FlagSet) usage() {
if fs == CommandLine {
Usage()
} else if f.Usage == nil {
defaultUsage(f)
} else if fs.Usage == nil {
defaultUsage(fs)
} else {
f.Usage()
fs.Usage()
}
}
@ -934,25 +934,25 @@ func trimQuotes(str string) string {
}
// parseOne parses one flag. It reports whether a flag was seen.
func (f *FlagSet) parseOne() (bool, string, error) {
if len(f.args) == 0 {
func (fs *FlagSet) parseOne() (bool, string, error) {
if len(fs.args) == 0 {
return false, "", nil
}
s := f.args[0]
s := fs.args[0]
if len(s) == 0 || s[0] != '-' || len(s) == 1 {
return false, "", nil
}
if s[1] == '-' && len(s) == 2 { // "--" terminates the flags
f.args = f.args[1:]
fs.args = fs.args[1:]
return false, "", nil
}
name := s[1:]
if len(name) == 0 || name[0] == '=' {
return false, "", f.failf("bad flag syntax: %s", s)
return false, "", fs.failf("bad flag syntax: %s", s)
}
// it's a flag. does it have an argument?
f.args = f.args[1:]
fs.args = fs.args[1:]
hasValue := false
value := ""
if i := strings.Index(name, "="); i != -1 {
@ -961,44 +961,44 @@ func (f *FlagSet) parseOne() (bool, string, error) {
name = name[:i]
}
m := f.formal
m := fs.formal
flag, alreadythere := m[name] // BUG
if !alreadythere {
if name == "-help" || name == "help" || name == "h" { // special case for nice help message.
f.usage()
fs.usage()
return false, "", ErrHelp
}
if len(name) > 0 && name[0] == '-' {
return false, "", f.failf("flag provided but not defined: -%s", name)
return false, "", fs.failf("flag provided but not defined: -%s", name)
}
return false, name, ErrRetry
}
if fv, ok := flag.Value.(boolFlag); ok && fv.IsBoolFlag() { // special case: doesn't need an arg
if hasValue {
if err := fv.Set(value); err != nil {
return false, "", f.failf("invalid boolean value %q for -%s: %v", value, name, err)
return false, "", fs.failf("invalid boolean value %q for -%s: %v", value, name, err)
}
} else {
fv.Set("true")
}
} else {
// It must have a value, which might be the next argument.
if !hasValue && len(f.args) > 0 {
if !hasValue && len(fs.args) > 0 {
// value is the next arg
hasValue = true
value, f.args = f.args[0], f.args[1:]
value, fs.args = fs.args[0], fs.args[1:]
}
if !hasValue {
return false, "", f.failf("flag needs an argument: -%s", name)
return false, "", fs.failf("flag needs an argument: -%s", name)
}
if err := flag.Value.Set(value); err != nil {
return false, "", f.failf("invalid value %q for flag -%s: %v", value, name, err)
return false, "", fs.failf("invalid value %q for flag -%s: %v", value, name, err)
}
}
if f.actual == nil {
f.actual = make(map[string]*Flag)
if fs.actual == nil {
fs.actual = make(map[string]*Flag)
}
f.actual[name] = flag
fs.actual[name] = flag
for i, n := range flag.Names {
if n == fmt.Sprintf("#%s", name) {
replacement := ""
@ -1009,9 +1009,9 @@ func (f *FlagSet) parseOne() (bool, string, error) {
}
}
if replacement != "" {
fmt.Fprintf(f.Out(), "Warning: '-%s' is deprecated, it will be replaced by '-%s' soon. See usage.\n", name, replacement)
fmt.Fprintf(fs.Out(), "Warning: '-%s' is deprecated, it will be replaced by '-%s' soon. See usage.\n", name, replacement)
} else {
fmt.Fprintf(f.Out(), "Warning: '-%s' is deprecated, it will be removed soon. See usage.\n", name)
fmt.Fprintf(fs.Out(), "Warning: '-%s' is deprecated, it will be removed soon. See usage.\n", name)
}
}
}
@ -1022,11 +1022,11 @@ func (f *FlagSet) parseOne() (bool, string, error) {
// include the command name. Must be called after all flags in the FlagSet
// are defined and before flags are accessed by the program.
// The return value will be ErrHelp if -help was set but not defined.
func (f *FlagSet) Parse(arguments []string) error {
f.parsed = true
f.args = arguments
func (fs *FlagSet) Parse(arguments []string) error {
fs.parsed = true
fs.args = arguments
for {
seen, name, err := f.parseOne()
seen, name, err := fs.parseOne()
if seen {
continue
}
@ -1037,13 +1037,13 @@ func (f *FlagSet) Parse(arguments []string) error {
if len(name) > 1 {
err = nil
for _, letter := range strings.Split(name, "") {
f.args = append([]string{"-" + letter}, f.args...)
seen2, _, err2 := f.parseOne()
fs.args = append([]string{"-" + letter}, fs.args...)
seen2, _, err2 := fs.parseOne()
if seen2 {
continue
}
if err2 != nil {
err = f.failf("flag provided but not defined: -%s", name)
err = fs.failf("flag provided but not defined: -%s", name)
break
}
}
@ -1051,10 +1051,10 @@ func (f *FlagSet) Parse(arguments []string) error {
continue
}
} else {
err = f.failf("flag provided but not defined: -%s", name)
err = fs.failf("flag provided but not defined: -%s", name)
}
}
switch f.errorHandling {
switch fs.errorHandling {
case ContinueOnError:
return err
case ExitOnError:
@ -1067,46 +1067,48 @@ func (f *FlagSet) Parse(arguments []string) error {
}
// ParseFlags is a utility function that adds a help flag if withHelp is true,
// calls cmd.Parse(args) and prints a relevant error message if there are
// calls fs.Parse(args) and prints a relevant error message if there are
// incorrect number of arguments. It returns error only if error handling is
// set to ContinueOnError and parsing fails. If error handling is set to
// ExitOnError, it's safe to ignore the return value.
func (cmd *FlagSet) ParseFlags(args []string, withHelp bool) error {
func (fs *FlagSet) ParseFlags(args []string, withHelp bool) error {
var help *bool
if withHelp {
help = cmd.Bool([]string{"#help", "-help"}, false, "Print usage")
help = fs.Bool([]string{"#help", "-help"}, false, "Print usage")
}
if err := cmd.Parse(args); err != nil {
if err := fs.Parse(args); err != nil {
return err
}
if help != nil && *help {
cmd.SetOutput(os.Stdout)
cmd.Usage()
fs.SetOutput(os.Stdout)
fs.Usage()
os.Exit(0)
}
if str := cmd.CheckArgs(); str != "" {
cmd.SetOutput(os.Stderr)
cmd.ReportError(str, withHelp)
cmd.ShortUsage()
if str := fs.CheckArgs(); str != "" {
fs.SetOutput(os.Stderr)
fs.ReportError(str, withHelp)
fs.ShortUsage()
os.Exit(1)
}
return nil
}
func (cmd *FlagSet) ReportError(str string, withHelp bool) {
// ReportError is a utility method that prints a user-friendly message
// containing the error that occurred during parsing and a suggestion to get help
func (fs *FlagSet) ReportError(str string, withHelp bool) {
if withHelp {
if os.Args[0] == cmd.Name() {
if os.Args[0] == fs.Name() {
str += ".\nSee '" + os.Args[0] + " --help'"
} else {
str += ".\nSee '" + os.Args[0] + " " + cmd.Name() + " --help'"
str += ".\nSee '" + os.Args[0] + " " + fs.Name() + " --help'"
}
}
fmt.Fprintf(cmd.Out(), "docker: %s.\n", str)
fmt.Fprintf(fs.Out(), "docker: %s.\n", str)
}
// Parsed reports whether f.Parse has been called.
func (f *FlagSet) Parsed() bool {
return f.parsed
// Parsed reports whether fs.Parse has been called.
func (fs *FlagSet) Parsed() bool {
return fs.parsed
}
// Parse parses the command-line flags from os.Args[1:]. Must be called
@ -1139,7 +1141,61 @@ func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet {
// Init sets the name and error handling property for a flag set.
// By default, the zero FlagSet uses an empty name and the
// ContinueOnError error handling policy.
func (f *FlagSet) Init(name string, errorHandling ErrorHandling) {
f.name = name
f.errorHandling = errorHandling
func (fs *FlagSet) Init(name string, errorHandling ErrorHandling) {
fs.name = name
fs.errorHandling = errorHandling
}
type mergeVal struct {
Value
key string
fset *FlagSet
}
func (v mergeVal) Set(s string) error {
return v.fset.Set(v.key, s)
}
func (v mergeVal) IsBoolFlag() bool {
if b, ok := v.Value.(boolFlag); ok {
return b.IsBoolFlag()
}
return false
}
// Merge is an helper function that merges n FlagSets into a single dest FlagSet
// In case of name collision between the flagsets it will apply
// the destination FlagSet's errorHandling behaviour.
func Merge(dest *FlagSet, flagsets ...*FlagSet) error {
for _, fset := range flagsets {
for k, f := range fset.formal {
if _, ok := dest.formal[k]; ok {
var err error
if fset.name == "" {
err = fmt.Errorf("flag redefined: %s", k)
} else {
err = fmt.Errorf("%s flag redefined: %s", fset.name, k)
}
fmt.Fprintln(fset.Out(), err.Error())
// Happens only if flags are declared with identical names
switch dest.errorHandling {
case ContinueOnError:
return err
case ExitOnError:
os.Exit(2)
case PanicOnError:
panic(err)
}
}
newF := *f
newF.Value = mergeVal{f.Value, k, fset}
dest.formal[k] = &newF
}
}
return nil
}
// IsEmpty reports if the FlagSet is actually empty.
func (fs *FlagSet) IsEmpty() bool {
return len(fs.actual) == 0
}

View file

@ -5,7 +5,7 @@ import (
)
// GetMounts retrieves a list of mounts for the current running process.
func GetMounts() ([]*MountInfo, error) {
func GetMounts() ([]*Info, error) {
return parseMountTable()
}

View file

@ -1,10 +1,10 @@
package mount
// MountInfo reveals information about a particular mounted filesystem. This
// Info reveals information about a particular mounted filesystem. This
// struct is populated from the content in the /proc/<pid>/mountinfo file.
type MountInfo struct {
// Id is a unique identifier of the mount (may be reused after umount).
Id int
type Info struct {
// ID is a unique identifier of the mount (may be reused after umount).
ID int
// Parent indicates the ID of the mount parent (or of self for the top of the
// mount tree).

View file

@ -15,7 +15,7 @@ import (
// Parse /proc/self/mountinfo because comparing Dev and ino does not work from
// bind mounts.
func parseMountTable() ([]*MountInfo, error) {
func parseMountTable() ([]*Info, error) {
var rawEntries *C.struct_statfs
count := int(C.getmntinfo(&rawEntries, C.MNT_WAIT))
@ -29,9 +29,9 @@ func parseMountTable() ([]*MountInfo, error) {
header.Len = count
header.Data = uintptr(unsafe.Pointer(rawEntries))
var out []*MountInfo
var out []*Info
for _, entry := range entries {
var mountinfo MountInfo
var mountinfo Info
mountinfo.Mountpoint = C.GoString(&entry.f_mntonname[0])
mountinfo.Source = C.GoString(&entry.f_mntfromname[0])
mountinfo.Fstype = C.GoString(&entry.f_fstypename[0])

View file

@ -30,7 +30,7 @@ const (
// Parse /proc/self/mountinfo because comparing Dev and ino does not work from
// bind mounts
func parseMountTable() ([]*MountInfo, error) {
func parseMountTable() ([]*Info, error) {
f, err := os.Open("/proc/self/mountinfo")
if err != nil {
return nil, err
@ -40,10 +40,10 @@ func parseMountTable() ([]*MountInfo, error) {
return parseInfoFile(f)
}
func parseInfoFile(r io.Reader) ([]*MountInfo, error) {
func parseInfoFile(r io.Reader) ([]*Info, error) {
var (
s = bufio.NewScanner(r)
out = []*MountInfo{}
out = []*Info{}
)
for s.Scan() {
@ -52,13 +52,13 @@ func parseInfoFile(r io.Reader) ([]*MountInfo, error) {
}
var (
p = &MountInfo{}
p = &Info{}
text = s.Text()
optionalFields string
)
if _, err := fmt.Sscanf(text, mountinfoFormat,
&p.Id, &p.Parent, &p.Major, &p.Minor,
&p.ID, &p.Parent, &p.Major, &p.Minor,
&p.Root, &p.Mountpoint, &p.Opts, &optionalFields); err != nil {
return nil, fmt.Errorf("Scanning '%s' failed: %s", text, err)
}
@ -84,7 +84,7 @@ func parseInfoFile(r io.Reader) ([]*MountInfo, error) {
// PidMountInfo collects the mounts for a specific process ID. If the process
// ID is unknown, it is better to use `GetMounts` which will inspect
// "/proc/self/mountinfo" instead.
func PidMountInfo(pid int) ([]*MountInfo, error) {
func PidMountInfo(pid int) ([]*Info, error) {
f, err := os.Open(fmt.Sprintf("/proc/%d/mountinfo", pid))
if err != nil {
return nil, err

View file

@ -457,8 +457,8 @@ func TestParseFedoraMountinfoFields(t *testing.T) {
if len(infos) != expectedLength {
t.Fatalf("Expected %d entries, got %d", expectedLength, len(infos))
}
mi := MountInfo{
Id: 15,
mi := Info{
ID: 15,
Parent: 35,
Major: 0,
Minor: 3,

View file

@ -7,6 +7,6 @@ import (
"runtime"
)
func parseMountTable() ([]*MountInfo, error) {
func parseMountTable() ([]*Info, error) {
return nil, fmt.Errorf("mount.parseMountTable is not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
}

View file

@ -107,7 +107,7 @@ func TestSubtreePrivate(t *testing.T) {
}
// Testing that when a target is a shared mount,
// then child mounts propogate to the source
// then child mounts propagate to the source
func TestSubtreeShared(t *testing.T) {
tmp := path.Join(os.TempDir(), "mount-tests")
if err := os.MkdirAll(tmp, 0777); err != nil {

View file

@ -5,31 +5,58 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/sockets"
"github.com/docker/docker/pkg/tlsconfig"
)
const (
versionMimetype = "application/vnd.docker.plugins.v1+json"
versionMimetype = "application/vnd.docker.plugins.v1.1+json"
defaultTimeOut = 30
)
func NewClient(addr string) *Client {
type remoteError struct {
method string
err string
}
func (e *remoteError) Error() string {
return fmt.Sprintf("Plugin Error: %s, %s", e.err, e.method)
}
// NewClient creates a new plugin client (http).
func NewClient(addr string, tlsConfig tlsconfig.Options) (*Client, error) {
tr := &http.Transport{}
c, err := tlsconfig.Client(tlsConfig)
if err != nil {
return nil, err
}
tr.TLSClientConfig = c
protoAndAddr := strings.Split(addr, "://")
configureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])
return &Client{&http.Client{Transport: tr}, protoAndAddr[1]}
sockets.ConfigureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])
scheme := protoAndAddr[0]
if scheme != "https" {
scheme = "http"
}
return &Client{&http.Client{Transport: tr}, scheme, protoAndAddr[1]}, nil
}
// Client represents a plugin client.
type Client struct {
http *http.Client
addr string
http *http.Client // http client to use
scheme string // scheme protocol of the plugin
addr string // http address of the plugin
}
// Call calls the specified method with the specified arguments for the plugin.
// It will retry for 30 seconds if a failure occurs when calling.
func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error {
return c.callWithRetry(serviceMethod, args, ret, true)
}
@ -45,7 +72,7 @@ func (c *Client) callWithRetry(serviceMethod string, args interface{}, ret inter
return err
}
req.Header.Add("Accept", versionMimetype)
req.URL.Scheme = "http"
req.URL.Scheme = c.scheme
req.URL.Host = c.addr
var retries int
@ -72,9 +99,9 @@ func (c *Client) callWithRetry(serviceMethod string, args interface{}, ret inter
if resp.StatusCode != http.StatusOK {
remoteErr, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("Plugin Error: %s", err)
return &remoteError{err.Error(), serviceMethod}
}
return fmt.Errorf("Plugin Error: %s", remoteErr)
return &remoteError{string(remoteErr), serviceMethod}
}
return json.NewDecoder(resp.Body).Decode(&ret)
@ -94,20 +121,5 @@ func backoff(retries int) time.Duration {
}
func abort(start time.Time, timeOff time.Duration) bool {
return timeOff+time.Since(start) > time.Duration(defaultTimeOut)*time.Second
}
func configureTCPTransport(tr *http.Transport, proto, addr string) {
// Why 32? See https://github.com/docker/docker/pull/8035.
timeout := 32 * time.Second
if proto == "unix" {
// No need for compression in local communications.
tr.DisableCompression = true
tr.Dial = func(_, _ string) (net.Conn, error) {
return net.DialTimeout(proto, addr, timeout)
}
} else {
tr.Proxy = http.ProxyFromEnvironment
tr.Dial = (&net.Dialer{Timeout: timeout}).Dial
}
return timeOff+time.Since(start) >= time.Duration(defaultTimeOut)*time.Second
}

View file

@ -7,6 +7,8 @@ import (
"reflect"
"testing"
"time"
"github.com/docker/docker/pkg/tlsconfig"
)
var (
@ -27,7 +29,7 @@ func teardownRemotePluginServer() {
}
func TestFailedConnection(t *testing.T) {
c := NewClient("tcp://127.0.0.1:1")
c, _ := NewClient("tcp://127.0.0.1:1", tlsconfig.Options{InsecureSkipVerify: true})
err := c.callWithRetry("Service.Method", nil, nil, false)
if err == nil {
t.Fatal("Unexpected successful connection")
@ -51,7 +53,7 @@ func TestEchoInputOutput(t *testing.T) {
io.Copy(w, r.Body)
})
c := NewClient(addr)
c, _ := NewClient(addr, tlsconfig.Options{InsecureSkipVerify: true})
var output Manifest
err := c.Call("Test.Echo", m, &output)
if err != nil {
@ -103,3 +105,19 @@ func TestAbortRetry(t *testing.T) {
}
}
}
func TestClientScheme(t *testing.T) {
cases := map[string]string{
"tcp://127.0.0.1:8080": "http",
"unix:///usr/local/plugins/foo": "http",
"http://127.0.0.1:8080": "http",
"https://127.0.0.1:8080": "https",
}
for addr, scheme := range cases {
c, _ := NewClient(addr, tlsconfig.Options{InsecureSkipVerify: true})
if c.scheme != scheme {
t.Fatalf("URL scheme mismatch, expected %s, got %s", scheme, c.scheme)
}
}
}

View file

@ -1,6 +1,7 @@
package plugins
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
@ -10,52 +11,56 @@ import (
"strings"
)
const defaultLocalRegistry = "/usr/share/docker/plugins"
var (
// ErrNotFound plugin not found
ErrNotFound = errors.New("Plugin not found")
socketsPath = "/run/docker/plugins"
specsPaths = []string{"/etc/docker/plugins", "/usr/lib/docker/plugins"}
)
// Registry defines behavior of a registry of plugins.
type Registry interface {
// Plugins lists all plugins.
Plugins() ([]*Plugin, error)
// Plugin returns the plugin registered with the given name (or returns an error).
Plugin(name string) (*Plugin, error)
}
type LocalRegistry struct {
path string
}
func newLocalRegistry(path string) *LocalRegistry {
if len(path) == 0 {
path = defaultLocalRegistry
}
return &LocalRegistry{path}
// LocalRegistry defines a registry that is local (using unix socket).
type LocalRegistry struct{}
func newLocalRegistry() LocalRegistry {
return LocalRegistry{}
}
// Plugin returns the plugin registered with the given name (or returns an error).
func (l *LocalRegistry) Plugin(name string) (*Plugin, error) {
filepath := filepath.Join(l.path, name)
specpath := filepath + ".spec"
if fi, err := os.Stat(specpath); err == nil {
return readPluginInfo(specpath, fi)
socketpaths := pluginPaths(socketsPath, name, ".sock")
for _, p := range socketpaths {
if fi, err := os.Stat(p); err == nil && fi.Mode()&os.ModeSocket != 0 {
return newLocalPlugin(name, "unix://"+p), nil
}
}
socketpath := filepath + ".sock"
if fi, err := os.Stat(socketpath); err == nil {
return readPluginInfo(socketpath, fi)
var txtspecpaths []string
for _, p := range specsPaths {
txtspecpaths = append(txtspecpaths, pluginPaths(p, name, ".spec")...)
txtspecpaths = append(txtspecpaths, pluginPaths(p, name, ".json")...)
}
for _, p := range txtspecpaths {
if _, err := os.Stat(p); err == nil {
if strings.HasSuffix(p, ".json") {
return readPluginJSONInfo(name, p)
}
return readPluginInfo(name, p)
}
}
return nil, ErrNotFound
}
func readPluginInfo(path string, fi os.FileInfo) (*Plugin, error) {
name := strings.Split(fi.Name(), ".")[0]
if fi.Mode()&os.ModeSocket != 0 {
return &Plugin{
Name: name,
Addr: "unix://" + path,
}, nil
}
func readPluginInfo(name, path string) (*Plugin, error) {
content, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
@ -71,8 +76,31 @@ func readPluginInfo(path string, fi os.FileInfo) (*Plugin, error) {
return nil, fmt.Errorf("Unknown protocol")
}
return &Plugin{
Name: name,
Addr: addr,
}, nil
return newLocalPlugin(name, addr), nil
}
func readPluginJSONInfo(name, path string) (*Plugin, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
var p Plugin
if err := json.NewDecoder(f).Decode(&p); err != nil {
return nil, err
}
p.Name = name
if len(p.TLSConfig.CAFile) == 0 {
p.TLSConfig.InsecureSkipVerify = true
}
return &p, nil
}
func pluginPaths(base, name, ext string) []string {
return []string{
filepath.Join(base, name+ext),
filepath.Join(base, name, name+ext),
}
}

View file

@ -10,62 +10,72 @@ import (
"testing"
)
func TestUnknownLocalPath(t *testing.T) {
func setup(t *testing.T) (string, func()) {
tmpdir, err := ioutil.TempDir("", "docker-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
backup := socketsPath
socketsPath = tmpdir
specsPaths = []string{tmpdir}
l := newLocalRegistry(filepath.Join(tmpdir, "unknown"))
_, err = l.Plugin("foo")
if err == nil || err != ErrNotFound {
t.Fatalf("Expected error for unknown directory")
return tmpdir, func() {
socketsPath = backup
os.RemoveAll(tmpdir)
}
}
func TestLocalSocket(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "docker-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
l, err := net.Listen("unix", filepath.Join(tmpdir, "echo.sock"))
if err != nil {
t.Fatal(err)
}
defer l.Close()
tmpdir, unregister := setup(t)
defer unregister()
r := newLocalRegistry(tmpdir)
p, err := r.Plugin("echo")
if err != nil {
t.Fatal(err)
cases := []string{
filepath.Join(tmpdir, "echo.sock"),
filepath.Join(tmpdir, "echo", "echo.sock"),
}
pp, err := r.Plugin("echo")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(p, pp) {
t.Fatalf("Expected %v, was %v\n", p, pp)
}
for _, c := range cases {
if err := os.MkdirAll(filepath.Dir(c), 0755); err != nil {
t.Fatal(err)
}
if p.Name != "echo" {
t.Fatalf("Expected plugin `echo`, got %s\n", p.Name)
}
l, err := net.Listen("unix", c)
if err != nil {
t.Fatal(err)
}
addr := fmt.Sprintf("unix://%s/echo.sock", tmpdir)
if p.Addr != addr {
t.Fatalf("Expected plugin addr `%s`, got %s\n", addr, p.Addr)
r := newLocalRegistry()
p, err := r.Plugin("echo")
if err != nil {
t.Fatal(err)
}
pp, err := r.Plugin("echo")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(p, pp) {
t.Fatalf("Expected %v, was %v\n", p, pp)
}
if p.Name != "echo" {
t.Fatalf("Expected plugin `echo`, got %s\n", p.Name)
}
addr := fmt.Sprintf("unix://%s", c)
if p.Addr != addr {
t.Fatalf("Expected plugin addr `%s`, got %s\n", addr, p.Addr)
}
if p.TLSConfig.InsecureSkipVerify != true {
t.Fatalf("Expected TLS verification to be skipped")
}
l.Close()
}
}
func TestFileSpecPlugin(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "docker-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tmpdir, unregister := setup(t)
defer unregister()
cases := []struct {
path string
@ -74,16 +84,21 @@ func TestFileSpecPlugin(t *testing.T) {
fail bool
}{
{filepath.Join(tmpdir, "echo.spec"), "echo", "unix://var/lib/docker/plugins/echo.sock", false},
{filepath.Join(tmpdir, "echo", "echo.spec"), "echo", "unix://var/lib/docker/plugins/echo.sock", false},
{filepath.Join(tmpdir, "foo.spec"), "foo", "tcp://localhost:8080", false},
{filepath.Join(tmpdir, "foo", "foo.spec"), "foo", "tcp://localhost:8080", false},
{filepath.Join(tmpdir, "bar.spec"), "bar", "localhost:8080", true}, // unknown transport
}
for _, c := range cases {
if err = ioutil.WriteFile(c.path, []byte(c.addr), 0644); err != nil {
if err := os.MkdirAll(filepath.Dir(c.path), 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(c.path, []byte(c.addr), 0644); err != nil {
t.Fatal(err)
}
r := newLocalRegistry(tmpdir)
r := newLocalRegistry()
p, err := r.Plugin(c.name)
if c.fail && err == nil {
continue
@ -100,5 +115,55 @@ func TestFileSpecPlugin(t *testing.T) {
if p.Addr != c.addr {
t.Fatalf("Expected plugin addr `%s`, got %s\n", c.addr, p.Addr)
}
if p.TLSConfig.InsecureSkipVerify != true {
t.Fatalf("Expected TLS verification to be skipped")
}
}
}
func TestFileJSONSpecPlugin(t *testing.T) {
tmpdir, unregister := setup(t)
defer unregister()
p := filepath.Join(tmpdir, "example.json")
spec := `{
"Name": "plugin-example",
"Addr": "https://example.com/docker/plugin",
"TLSConfig": {
"CAFile": "/usr/shared/docker/certs/example-ca.pem",
"CertFile": "/usr/shared/docker/certs/example-cert.pem",
"KeyFile": "/usr/shared/docker/certs/example-key.pem"
}
}`
if err := ioutil.WriteFile(p, []byte(spec), 0644); err != nil {
t.Fatal(err)
}
r := newLocalRegistry()
plugin, err := r.Plugin("example")
if err != nil {
t.Fatal(err)
}
if plugin.Name != "example" {
t.Fatalf("Expected plugin `plugin-example`, got %s\n", plugin.Name)
}
if plugin.Addr != "https://example.com/docker/plugin" {
t.Fatalf("Expected plugin addr `https://example.com/docker/plugin`, got %s\n", plugin.Addr)
}
if plugin.TLSConfig.CAFile != "/usr/shared/docker/certs/example-ca.pem" {
t.Fatalf("Expected plugin CA `/usr/shared/docker/certs/example-ca.pem`, got %s\n", plugin.TLSConfig.CAFile)
}
if plugin.TLSConfig.CertFile != "/usr/shared/docker/certs/example-cert.pem" {
t.Fatalf("Expected plugin Certificate `/usr/shared/docker/certs/example-cert.pem`, got %s\n", plugin.TLSConfig.CertFile)
}
if plugin.TLSConfig.KeyFile != "/usr/shared/docker/certs/example-key.pem" {
t.Fatalf("Expected plugin Key `/usr/shared/docker/certs/example-key.pem`, got %s\n", plugin.TLSConfig.KeyFile)
}
}

View file

@ -0,0 +1,68 @@
Plugin RPC Generator
====================
Generates go code from a Go interface definition for proxying between the plugin
API and the subsystem being extended.
## Usage
Given an interface definition:
```go
type volumeDriver interface {
Create(name string, opts opts) (err error)
Remove(name string) (err error)
Path(name string) (mountpoint string, err error)
Mount(name string) (mountpoint string, err error)
Unmount(name string) (err error)
}
```
**Note**: All function options and return values must be named in the definition.
Run the generator:
```bash
$ pluginrpc-gen --type volumeDriver --name VolumeDriver -i volumes/drivers/extpoint.go -o volumes/drivers/proxy.go
```
Where:
- `--type` is the name of the interface to use
- `--name` is the subsystem that the plugin "Implements"
- `-i` is the input file containing the interface definition
- `-o` is the output file where the the generated code should go
**Note**: The generated code will use the same package name as the one defined in the input file
Optionally, you can skip functions on the interface that should not be
implemented in the generated proxy code by passing in the function name to `--skip`.
This flag can be specified multiple times.
You can also add build tags that should be prepended to the generated code by
supplying `--tag`. This flag can be specified multiple times.
## Known issues
The parser can currently only handle types which are not specifically a map or
a slice.
You can, however, create a type that uses a map or a slice internally, for instance:
```go
type opts map[string]string
```
This `opts` type will work, whreas using a `map[string]string` directly will not.
## go-generate
You can also use this with go-generate, which is pretty awesome.
To do so, place the code at the top of the file which contains the interface
definition (i.e., the input file):
```go
//go:generate pluginrpc-gen -i $GOFILE -o proxy.go -type volumeDriver -name VolumeDriver
```
Then cd to the package dir and run `go generate`
**Note**: the `pluginrpc-gen` binary must be within your `$PATH`

View file

@ -0,0 +1,41 @@
package foo
type wobble struct {
Some string
Val string
Inception *wobble
}
// Fooer is an empty interface used for tests.
type Fooer interface{}
// Fooer2 is an interface used for tests.
type Fooer2 interface {
Foo()
}
// Fooer3 is an interface used for tests.
type Fooer3 interface {
Foo()
Bar(a string)
Baz(a string) (err error)
Qux(a, b string) (val string, err error)
Wobble() (w *wobble)
Wiggle() (w wobble)
}
// Fooer4 is an interface used for tests.
type Fooer4 interface {
Foo() error
}
// Bar is an interface used for tests.
type Bar interface {
Boo(a string, b string) (s string, err error)
}
// Fooer5 is an interface used for tests.
type Fooer5 interface {
Foo()
Bar
}

View file

@ -0,0 +1,91 @@
package main
import (
"bytes"
"flag"
"fmt"
"go/format"
"io/ioutil"
"os"
"unicode"
"unicode/utf8"
)
type stringSet struct {
values map[string]struct{}
}
func (s stringSet) String() string {
return ""
}
func (s stringSet) Set(value string) error {
s.values[value] = struct{}{}
return nil
}
func (s stringSet) GetValues() map[string]struct{} {
return s.values
}
var (
typeName = flag.String("type", "", "interface type to generate plugin rpc proxy for")
rpcName = flag.String("name", *typeName, "RPC name, set if different from type")
inputFile = flag.String("i", "", "input file path")
outputFile = flag.String("o", *inputFile+"_proxy.go", "output file path")
skipFuncs map[string]struct{}
flSkipFuncs = stringSet{make(map[string]struct{})}
flBuildTags = stringSet{make(map[string]struct{})}
)
func errorOut(msg string, err error) {
if err == nil {
return
}
fmt.Fprintf(os.Stderr, "%s: %v\n", msg, err)
os.Exit(1)
}
func checkFlags() error {
if *outputFile == "" {
return fmt.Errorf("missing required flag `-o`")
}
if *inputFile == "" {
return fmt.Errorf("missing required flag `-i`")
}
return nil
}
func main() {
flag.Var(flSkipFuncs, "skip", "skip parsing for function")
flag.Var(flBuildTags, "tag", "build tags to add to generated files")
flag.Parse()
skipFuncs = flSkipFuncs.GetValues()
errorOut("error", checkFlags())
pkg, err := Parse(*inputFile, *typeName)
errorOut(fmt.Sprintf("error parsing requested type %s", *typeName), err)
var analysis = struct {
InterfaceType string
RPCName string
BuildTags map[string]struct{}
*ParsedPkg
}{toLower(*typeName), *rpcName, flBuildTags.GetValues(), pkg}
var buf bytes.Buffer
errorOut("parser error", generatedTempl.Execute(&buf, analysis))
src, err := format.Source(buf.Bytes())
errorOut("error formating generated source", err)
errorOut("error writing file", ioutil.WriteFile(*outputFile, src, 0644))
}
func toLower(s string) string {
if s == "" {
return ""
}
r, n := utf8.DecodeRuneInString(s)
return string(unicode.ToLower(r)) + s[n:]
}

View file

@ -0,0 +1,163 @@
package main
import (
"errors"
"fmt"
"go/ast"
"go/parser"
"go/token"
"reflect"
)
var errBadReturn = errors.New("found return arg with no name: all args must be named")
type errUnexpectedType struct {
expected string
actual interface{}
}
func (e errUnexpectedType) Error() string {
return fmt.Sprintf("got wrong type expecting %s, got: %v", e.expected, reflect.TypeOf(e.actual))
}
// ParsedPkg holds information about a package that has been parsed,
// its name and the list of functions.
type ParsedPkg struct {
Name string
Functions []function
}
type function struct {
Name string
Args []arg
Returns []arg
Doc string
}
type arg struct {
Name string
ArgType string
}
func (a *arg) String() string {
return a.Name + " " + a.ArgType
}
// Parse parses the given file for an interface definition with the given name.
func Parse(filePath string, objName string) (*ParsedPkg, error) {
fs := token.NewFileSet()
pkg, err := parser.ParseFile(fs, filePath, nil, parser.AllErrors)
if err != nil {
return nil, err
}
p := &ParsedPkg{}
p.Name = pkg.Name.Name
obj, exists := pkg.Scope.Objects[objName]
if !exists {
return nil, fmt.Errorf("could not find object %s in %s", objName, filePath)
}
if obj.Kind != ast.Typ {
return nil, fmt.Errorf("exected type, got %s", obj.Kind)
}
spec, ok := obj.Decl.(*ast.TypeSpec)
if !ok {
return nil, errUnexpectedType{"*ast.TypeSpec", obj.Decl}
}
iface, ok := spec.Type.(*ast.InterfaceType)
if !ok {
return nil, errUnexpectedType{"*ast.InterfaceType", spec.Type}
}
p.Functions, err = parseInterface(iface)
if err != nil {
return nil, err
}
return p, nil
}
func parseInterface(iface *ast.InterfaceType) ([]function, error) {
var functions []function
for _, field := range iface.Methods.List {
switch f := field.Type.(type) {
case *ast.FuncType:
method, err := parseFunc(field)
if err != nil {
return nil, err
}
if method == nil {
continue
}
functions = append(functions, *method)
case *ast.Ident:
spec, ok := f.Obj.Decl.(*ast.TypeSpec)
if !ok {
return nil, errUnexpectedType{"*ast.TypeSpec", f.Obj.Decl}
}
iface, ok := spec.Type.(*ast.InterfaceType)
if !ok {
return nil, errUnexpectedType{"*ast.TypeSpec", spec.Type}
}
funcs, err := parseInterface(iface)
if err != nil {
fmt.Println(err)
continue
}
functions = append(functions, funcs...)
default:
return nil, errUnexpectedType{"*astFuncType or *ast.Ident", f}
}
}
return functions, nil
}
func parseFunc(field *ast.Field) (*function, error) {
f := field.Type.(*ast.FuncType)
method := &function{Name: field.Names[0].Name}
if _, exists := skipFuncs[method.Name]; exists {
fmt.Println("skipping:", method.Name)
return nil, nil
}
if f.Params != nil {
args, err := parseArgs(f.Params.List)
if err != nil {
return nil, err
}
method.Args = args
}
if f.Results != nil {
returns, err := parseArgs(f.Results.List)
if err != nil {
return nil, fmt.Errorf("error parsing function returns for %q: %v", method.Name, err)
}
method.Returns = returns
}
return method, nil
}
func parseArgs(fields []*ast.Field) ([]arg, error) {
var args []arg
for _, f := range fields {
if len(f.Names) == 0 {
return nil, errBadReturn
}
for _, name := range f.Names {
var typeName string
switch argType := f.Type.(type) {
case *ast.Ident:
typeName = argType.Name
case *ast.StarExpr:
i, ok := argType.X.(*ast.Ident)
if !ok {
return nil, errUnexpectedType{"*ast.Ident", f.Type}
}
typeName = "*" + i.Name
default:
return nil, errUnexpectedType{"*ast.Ident or *ast.StarExpr", f.Type}
}
args = append(args, arg{name.Name, typeName})
}
}
return args, nil
}

View file

@ -0,0 +1,168 @@
package main
import (
"fmt"
"path/filepath"
"runtime"
"strings"
"testing"
)
const testFixture = "fixtures/foo.go"
func TestParseEmptyInterface(t *testing.T) {
pkg, err := Parse(testFixture, "Fooer")
if err != nil {
t.Fatal(err)
}
assertName(t, "foo", pkg.Name)
assertNum(t, 0, len(pkg.Functions))
}
func TestParseNonInterfaceType(t *testing.T) {
_, err := Parse(testFixture, "wobble")
if _, ok := err.(errUnexpectedType); !ok {
t.Fatal("expected type error when parsing non-interface type")
}
}
func TestParseWithOneFunction(t *testing.T) {
pkg, err := Parse(testFixture, "Fooer2")
if err != nil {
t.Fatal(err)
}
assertName(t, "foo", pkg.Name)
assertNum(t, 1, len(pkg.Functions))
assertName(t, "Foo", pkg.Functions[0].Name)
assertNum(t, 0, len(pkg.Functions[0].Args))
assertNum(t, 0, len(pkg.Functions[0].Returns))
}
func TestParseWithMultipleFuncs(t *testing.T) {
pkg, err := Parse(testFixture, "Fooer3")
if err != nil {
t.Fatal(err)
}
assertName(t, "foo", pkg.Name)
assertNum(t, 6, len(pkg.Functions))
f := pkg.Functions[0]
assertName(t, "Foo", f.Name)
assertNum(t, 0, len(f.Args))
assertNum(t, 0, len(f.Returns))
f = pkg.Functions[1]
assertName(t, "Bar", f.Name)
assertNum(t, 1, len(f.Args))
assertNum(t, 0, len(f.Returns))
arg := f.Args[0]
assertName(t, "a", arg.Name)
assertName(t, "string", arg.ArgType)
f = pkg.Functions[2]
assertName(t, "Baz", f.Name)
assertNum(t, 1, len(f.Args))
assertNum(t, 1, len(f.Returns))
arg = f.Args[0]
assertName(t, "a", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Returns[0]
assertName(t, "err", arg.Name)
assertName(t, "error", arg.ArgType)
f = pkg.Functions[3]
assertName(t, "Qux", f.Name)
assertNum(t, 2, len(f.Args))
assertNum(t, 2, len(f.Returns))
arg = f.Args[0]
assertName(t, "a", f.Args[0].Name)
assertName(t, "string", f.Args[0].ArgType)
arg = f.Args[1]
assertName(t, "b", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Returns[0]
assertName(t, "val", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Returns[1]
assertName(t, "err", arg.Name)
assertName(t, "error", arg.ArgType)
f = pkg.Functions[4]
assertName(t, "Wobble", f.Name)
assertNum(t, 0, len(f.Args))
assertNum(t, 1, len(f.Returns))
arg = f.Returns[0]
assertName(t, "w", arg.Name)
assertName(t, "*wobble", arg.ArgType)
f = pkg.Functions[5]
assertName(t, "Wiggle", f.Name)
assertNum(t, 0, len(f.Args))
assertNum(t, 1, len(f.Returns))
arg = f.Returns[0]
assertName(t, "w", arg.Name)
assertName(t, "wobble", arg.ArgType)
}
func TestParseWithUnamedReturn(t *testing.T) {
_, err := Parse(testFixture, "Fooer4")
if !strings.HasSuffix(err.Error(), errBadReturn.Error()) {
t.Fatalf("expected ErrBadReturn, got %v", err)
}
}
func TestEmbeddedInterface(t *testing.T) {
pkg, err := Parse(testFixture, "Fooer5")
if err != nil {
t.Fatal(err)
}
assertName(t, "foo", pkg.Name)
assertNum(t, 2, len(pkg.Functions))
f := pkg.Functions[0]
assertName(t, "Foo", f.Name)
assertNum(t, 0, len(f.Args))
assertNum(t, 0, len(f.Returns))
f = pkg.Functions[1]
assertName(t, "Boo", f.Name)
assertNum(t, 2, len(f.Args))
assertNum(t, 2, len(f.Returns))
arg := f.Args[0]
assertName(t, "a", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Args[1]
assertName(t, "b", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Returns[0]
assertName(t, "s", arg.Name)
assertName(t, "string", arg.ArgType)
arg = f.Returns[1]
assertName(t, "err", arg.Name)
assertName(t, "error", arg.ArgType)
}
func assertName(t *testing.T, expected, actual string) {
if expected != actual {
fatalOut(t, fmt.Sprintf("expected name to be `%s`, got: %s", expected, actual))
}
}
func assertNum(t *testing.T, expected, actual int) {
if expected != actual {
fatalOut(t, fmt.Sprintf("expected number to be %d, got: %d", expected, actual))
}
}
func fatalOut(t *testing.T, msg string) {
_, file, ln, _ := runtime.Caller(2)
t.Fatalf("%s:%d: %s", filepath.Base(file), ln, msg)
}

View file

@ -0,0 +1,97 @@
package main
import (
"strings"
"text/template"
)
func printArgs(args []arg) string {
var argStr []string
for _, arg := range args {
argStr = append(argStr, arg.String())
}
return strings.Join(argStr, ", ")
}
func marshalType(t string) string {
switch t {
case "error":
// convert error types to plain strings to ensure the values are encoded/decoded properly
return "string"
default:
return t
}
}
func isErr(t string) bool {
switch t {
case "error":
return true
default:
return false
}
}
// Need to use this helper due to issues with go-vet
func buildTag(s string) string {
return "+build " + s
}
var templFuncs = template.FuncMap{
"printArgs": printArgs,
"marshalType": marshalType,
"isErr": isErr,
"lower": strings.ToLower,
"title": strings.Title,
"tag": buildTag,
}
var generatedTempl = template.Must(template.New("rpc_cient").Funcs(templFuncs).Parse(`
// generated code - DO NOT EDIT
{{ range $k, $v := .BuildTags }}
// {{ tag $k }} {{ end }}
package {{ .Name }}
import "errors"
type client interface{
Call(string, interface{}, interface{}) error
}
type {{ .InterfaceType }}Proxy struct {
client
}
{{ range .Functions }}
type {{ $.InterfaceType }}Proxy{{ .Name }}Request struct{
{{ range .Args }}
{{ title .Name }} {{ .ArgType }} {{ end }}
}
type {{ $.InterfaceType }}Proxy{{ .Name }}Response struct{
{{ range .Returns }}
{{ title .Name }} {{ marshalType .ArgType }} {{ end }}
}
func (pp *{{ $.InterfaceType }}Proxy) {{ .Name }}({{ printArgs .Args }}) ({{ printArgs .Returns }}) {
var(
req {{ $.InterfaceType }}Proxy{{ .Name }}Request
ret {{ $.InterfaceType }}Proxy{{ .Name }}Response
)
{{ range .Args }}
req.{{ title .Name }} = {{ lower .Name }} {{ end }}
if err = pp.Call("{{ $.RPCName }}.{{ .Name }}", req, &ret); err != nil {
return
}
{{ range $r := .Returns }}
{{ if isErr .ArgType }}
if ret.{{ title .Name }} != "" {
{{ lower .Name }} = errors.New(ret.{{ title .Name }})
} {{ end }}
{{ if isErr .ArgType | not }} {{ lower .Name }} = ret.{{ title .Name }} {{ end }} {{ end }}
return
}
{{ end }}
`))

View file

@ -1,13 +1,38 @@
// Package plugins provides structures and helper functions to manage Docker
// plugins.
//
// Docker discovers plugins by looking for them in the plugin directory whenever
// a user or container tries to use one by name. UNIX domain socket files must
// be located under /run/docker/plugins, whereas spec files can be located
// either under /etc/docker/plugins or /usr/lib/docker/plugins. This is handled
// by the Registry interface, which lets you list all plugins or get a plugin by
// its name if it exists.
//
// The plugins need to implement an HTTP server and bind this to the UNIX socket
// or the address specified in the spec files.
// A handshake is send at /Plugin.Activate, and plugins are expected to return
// a Manifest with a list of of Docker subsystems which this plugin implements.
//
// In order to use a plugins, you can use the ``Get`` with the name of the
// plugin and the subsystem it implements.
//
// plugin, err := plugins.Get("example", "VolumeDriver")
// if err != nil {
// return fmt.Errorf("Error looking up volume plugin example: %v", err)
// }
package plugins
import (
"errors"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/tlsconfig"
)
var (
// ErrNotImplements is returned if the plugin does not implement the requested driver.
ErrNotImplements = errors.New("Plugin does not implement the requested driver")
)
@ -21,27 +46,59 @@ var (
extpointHandlers = make(map[string]func(string, *Client))
)
// Manifest lists what a plugin implements.
type Manifest struct {
// List of subsystem the plugin implements.
Implements []string
}
// Plugin is the definition of a docker plugin.
type Plugin struct {
Name string
Addr string
Client *Client
Manifest *Manifest
// Name of the plugin
Name string `json:"-"`
// Address of the plugin
Addr string
// TLS configuration of the plugin
TLSConfig tlsconfig.Options
// Client attached to the plugin
Client *Client `json:"-"`
// Manifest of the plugin (see above)
Manifest *Manifest `json:"-"`
activatErr error
activateOnce sync.Once
}
func newLocalPlugin(name, addr string) *Plugin {
return &Plugin{
Name: name,
Addr: addr,
TLSConfig: tlsconfig.Options{InsecureSkipVerify: true},
}
}
func (p *Plugin) activate() error {
m := new(Manifest)
p.Client = NewClient(p.Addr)
err := p.Client.Call("Plugin.Activate", nil, m)
p.activateOnce.Do(func() {
p.activatErr = p.activateWithLock()
})
return p.activatErr
}
func (p *Plugin) activateWithLock() error {
c, err := NewClient(p.Addr, p.TLSConfig)
if err != nil {
return err
}
p.Client = c
m := new(Manifest)
if err = p.Client.Call("Plugin.Activate", nil, m); err != nil {
return err
}
logrus.Debugf("%s's manifest: %v", p.Name, m)
p.Manifest = m
for _, iface := range m.Implements {
handler, handled := extpointHandlers[iface]
if !handled {
@ -53,34 +110,58 @@ func (p *Plugin) activate() error {
}
func load(name string) (*Plugin, error) {
registry := newLocalRegistry("")
pl, err := registry.Plugin(name)
if err != nil {
return nil, err
return loadWithRetry(name, true)
}
func loadWithRetry(name string, retry bool) (*Plugin, error) {
registry := newLocalRegistry()
start := time.Now()
var retries int
for {
pl, err := registry.Plugin(name)
if err != nil {
if !retry {
return nil, err
}
timeOff := backoff(retries)
if abort(start, timeOff) {
return nil, err
}
retries++
logrus.Warnf("Unable to locate plugin: %s, retrying in %v", name, timeOff)
time.Sleep(timeOff)
continue
}
storage.Lock()
storage.plugins[name] = pl
storage.Unlock()
err = pl.activate()
if err != nil {
storage.Lock()
delete(storage.plugins, name)
storage.Unlock()
}
return pl, err
}
if err := pl.activate(); err != nil {
return nil, err
}
return pl, nil
}
func get(name string) (*Plugin, error) {
storage.Lock()
defer storage.Unlock()
pl, ok := storage.plugins[name]
storage.Unlock()
if ok {
return pl, nil
return pl, pl.activate()
}
pl, err := load(name)
if err != nil {
return nil, err
}
logrus.Debugf("Plugin: %v", pl)
storage.plugins[name] = pl
return pl, nil
return load(name)
}
// Get returns the plugin given the specified name and requested implementation.
func Get(name, imp string) (*Plugin, error) {
pl, err := get(name)
if err != nil {
@ -95,6 +176,7 @@ func Get(name, imp string) (*Plugin, error) {
return nil, ErrNotImplements
}
// Handle adds the specified function to the extpointHandlers.
func Handle(iface string, fn func(string, *Client)) {
extpointHandlers[iface] = fn
}

View file

@ -1,3 +1,5 @@
// Package proxy provides a network Proxy interface and implementations for TCP
// and UDP.
package proxy
import (
@ -5,18 +7,24 @@ import (
"net"
)
// Proxy defines the behavior of a proxy. It forwards traffic back and forth
// between two endpoints : the frontend and the backend.
// It can be used to do software port-mapping between two addresses.
// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000
// to the backend (container) at 172.17.42.108:4000.
type Proxy interface {
// Start forwarding traffic back and forth the front and back-end
// addresses.
// Run starts forwarding traffic back and forth between the front
// and back-end addresses.
Run()
// Stop forwarding traffic and close both ends of the Proxy.
// Close stops forwarding traffic and close both ends of the Proxy.
Close()
// Return the address on which the proxy is listening.
// FrontendAddr returns the address on which the proxy is listening.
FrontendAddr() net.Addr
// Return the proxied address.
// BackendAddr returns the proxied address.
BackendAddr() net.Addr
}
// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr.
func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
switch frontendAddr.(type) {
case *net.UDPAddr:

View file

@ -4,16 +4,25 @@ import (
"net"
)
// StubProxy is a proxy that is a stub (does nothing).
type StubProxy struct {
frontendAddr net.Addr
backendAddr net.Addr
}
func (p *StubProxy) Run() {}
func (p *StubProxy) Close() {}
func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr }
func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr }
// Run does nothing.
func (p *StubProxy) Run() {}
// Close does nothing.
func (p *StubProxy) Close() {}
// FrontendAddr returns the frontend address.
func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr }
// BackendAddr returns the backend address.
func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr }
// NewStubProxy creates a new StubProxy
func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
return &StubProxy{
frontendAddr: frontendAddr,

View file

@ -8,12 +8,15 @@ import (
"github.com/Sirupsen/logrus"
)
// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to
// handle TCP traffic forwarding between the frontend and backend addresses.
type TCPProxy struct {
listener *net.TCPListener
frontendAddr *net.TCPAddr
backendAddr *net.TCPAddr
}
// NewTCPProxy creates a new TCPProxy.
func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
listener, err := net.ListenTCP("tcp", frontendAddr)
if err != nil {
@ -53,7 +56,7 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
go broker(client, backend)
go broker(backend, client)
var transferred int64 = 0
var transferred int64
for i := 0; i < 2; i++ {
select {
case written := <-event:
@ -72,6 +75,7 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
backend.Close()
}
// Run starts forwarding the traffic using TCP.
func (proxy *TCPProxy) Run() {
quit := make(chan bool)
defer close(quit)
@ -85,6 +89,11 @@ func (proxy *TCPProxy) Run() {
}
}
func (proxy *TCPProxy) Close() { proxy.listener.Close() }
// Close stops forwarding the traffic.
func (proxy *TCPProxy) Close() { proxy.listener.Close() }
// FrontendAddr returns the TCP address on which the proxy is listening.
func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
// BackendAddr returns the TCP proxied address.
func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr }

View file

@ -12,8 +12,10 @@ import (
)
const (
// UDPConnTrackTimeout is the timeout used for UDP connection tracking
UDPConnTrackTimeout = 90 * time.Second
UDPBufSize = 65507
// UDPBufSize is the buffer size for the UDP proxy
UDPBufSize = 65507
)
// A net.Addr where the IP is split into two fields so you can use it as a key
@ -41,6 +43,9 @@ func newConnTrackKey(addr *net.UDPAddr) *connTrackKey {
type connTrackMap map[connTrackKey]*net.UDPConn
// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy
// interface to handle UDP traffic forwarding between the frontend and backend
// addresses.
type UDPProxy struct {
listener *net.UDPConn
frontendAddr *net.UDPAddr
@ -49,6 +54,7 @@ type UDPProxy struct {
connTrackLock sync.Mutex
}
// NewUDPProxy creates a new UDPProxy.
func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) {
listener, err := net.ListenUDP("udp", frontendAddr)
if err != nil {
@ -96,6 +102,7 @@ func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr
}
}
// Run starts forwarding the traffic using UDP.
func (proxy *UDPProxy) Run() {
readBuf := make([]byte, UDPBufSize)
for {
@ -135,6 +142,7 @@ func (proxy *UDPProxy) Run() {
}
}
// Close stops forwarding the traffic.
func (proxy *UDPProxy) Close() {
proxy.listener.Close()
proxy.connTrackLock.Lock()
@ -144,8 +152,11 @@ func (proxy *UDPProxy) Close() {
}
}
// FrontendAddr returns the UDP address on which the proxy is listening.
func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
// BackendAddr returns the proxied UDP address.
func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
func isClosedError(err error) bool {
/* This comparison is ugly, but unfortunately, net.go doesn't export errClosing.

View file

@ -0,0 +1,71 @@
package random
import (
cryptorand "crypto/rand"
"io"
"math"
"math/big"
"math/rand"
"sync"
"time"
)
// Rand is a global *rand.Rand instance, which initilized with NewSource() source.
var Rand = rand.New(NewSource())
// Reader is a global, shared instance of a pseudorandom bytes generator.
// It doesn't consume entropy.
var Reader io.Reader = &reader{rnd: Rand}
// copypaste from standard math/rand
type lockedSource struct {
lk sync.Mutex
src rand.Source
}
func (r *lockedSource) Int63() (n int64) {
r.lk.Lock()
n = r.src.Int63()
r.lk.Unlock()
return
}
func (r *lockedSource) Seed(seed int64) {
r.lk.Lock()
r.src.Seed(seed)
r.lk.Unlock()
}
// NewSource returns math/rand.Source safe for concurrent use and initialized
// with current unix-nano timestamp
func NewSource() rand.Source {
var seed int64
if cryptoseed, err := cryptorand.Int(cryptorand.Reader, big.NewInt(math.MaxInt64)); err != nil {
// This should not happen, but worst-case fallback to time-based seed.
seed = time.Now().UnixNano()
} else {
seed = cryptoseed.Int64()
}
return &lockedSource{
src: rand.NewSource(seed),
}
}
type reader struct {
rnd *rand.Rand
}
func (r *reader) Read(b []byte) (int, error) {
i := 0
for {
val := r.rnd.Int63()
for val > 0 {
b[i] = byte(val)
i++
if i == len(b) {
return i, nil
}
val >>= 8
}
}
}

View file

@ -0,0 +1,22 @@
package random
import (
"math/rand"
"sync"
"testing"
)
// for go test -v -race
func TestConcurrency(t *testing.T) {
rnd := rand.New(NewSource())
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
rnd.Int63()
wg.Done()
}()
}
wg.Wait()
}

View file

@ -0,0 +1,23 @@
// +build freebsd
package reexec
import (
"os/exec"
)
// Self returns the path to the current process's binary.
// Uses os.Args[0].
func Self() string {
return naiveSelf()
}
// Command returns *exec.Cmd which have Path as current binary.
// For example if current binary is "docker" at "/usr/bin/", then cmd.Path will
// be set to "/usr/bin/docker".
func Command(args ...string) *exec.Cmd {
return &exec.Cmd{
Path: Self(),
Args: args,
}
}

View file

@ -7,6 +7,16 @@ import (
"syscall"
)
// Self returns the path to the current process's binary.
// Returns "/proc/self/exe".
func Self() string {
return "/proc/self/exe"
}
// Command returns *exec.Cmd which have Path as current binary. Also it setting
// SysProcAttr.Pdeathsig to SIGTERM.
// This will use the in-memory version (/proc/self/exe) of the current binary,
// it is thus safe to delete or replace the on-disk binary (os.Args[0]).
func Command(args ...string) *exec.Cmd {
return &exec.Cmd{
Path: Self(),

View file

@ -1,4 +1,4 @@
// +build !linux,!windows
// +build !linux,!windows,!freebsd
package reexec
@ -6,6 +6,7 @@ import (
"os/exec"
)
// Command is unsupported on operating systems apart from Linux and Windows.
func Command(args ...string) *exec.Cmd {
return nil
}

View file

@ -6,6 +6,15 @@ import (
"os/exec"
)
// Self returns the path to the current process's binary.
// Uses os.Args[0].
func Self() string {
return naiveSelf()
}
// Command returns *exec.Cmd which have Path as current binary.
// For example if current binary is "docker.exe" at "C:\", then cmd.Path will
// be set to "C:\docker.exe".
func Command(args ...string) *exec.Cmd {
return &exec.Cmd{
Path: Self(),

View file

@ -30,8 +30,7 @@ func Init() bool {
return false
}
// Self returns the path to the current processes binary
func Self() string {
func naiveSelf() string {
name := os.Args[0]
if filepath.Base(name) == name {
if lp, err := exec.LookPath(name); err == nil {

View file

@ -0,0 +1 @@
This package provides helper functions for dealing with signals across various operating systems

View file

@ -0,0 +1,44 @@
// Package signal provides helper functions for dealing with signals across
// various operating systems.
package signal
import (
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
)
// CatchAll catches all signals and relays them to the specified channel.
func CatchAll(sigc chan os.Signal) {
handledSigs := []os.Signal{}
for _, s := range SignalMap {
handledSigs = append(handledSigs, s)
}
signal.Notify(sigc, handledSigs...)
}
// StopCatch stops catching the signals and closes the specified channel.
func StopCatch(sigc chan os.Signal) {
signal.Stop(sigc)
close(sigc)
}
// ParseSignal translates a string to a valid syscall signal.
// It returns an error if the signal map doesn't include the given signal.
func ParseSignal(rawSignal string) (syscall.Signal, error) {
s, err := strconv.Atoi(rawSignal)
if err == nil {
if s == 0 {
return -1, fmt.Errorf("Invalid signal: %s", rawSignal)
}
return syscall.Signal(s), nil
}
signal, ok := SignalMap[strings.TrimPrefix(strings.ToUpper(rawSignal), "SIG")]
if !ok {
return -1, fmt.Errorf("Invalid signal: %s", rawSignal)
}
return signal, nil
}

View file

@ -0,0 +1,41 @@
package signal
import (
"syscall"
)
// SignalMap is a map of Darwin signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUG": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CONT": syscall.SIGCONT,
"EMT": syscall.SIGEMT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INFO": syscall.SIGINFO,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"PIPE": syscall.SIGPIPE,
"PROF": syscall.SIGPROF,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

View file

@ -0,0 +1,43 @@
package signal
import (
"syscall"
)
// SignalMap is a map of FreeBSD signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUF": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CONT": syscall.SIGCONT,
"EMT": syscall.SIGEMT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INFO": syscall.SIGINFO,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"LWP": syscall.SIGLWP,
"PIPE": syscall.SIGPIPE,
"PROF": syscall.SIGPROF,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"THR": syscall.SIGTHR,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

View file

@ -0,0 +1,44 @@
package signal
import (
"syscall"
)
// SignalMap is a map of Linux signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUS": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CLD": syscall.SIGCLD,
"CONT": syscall.SIGCONT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"PIPE": syscall.SIGPIPE,
"POLL": syscall.SIGPOLL,
"PROF": syscall.SIGPROF,
"PWR": syscall.SIGPWR,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STKFLT": syscall.SIGSTKFLT,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"UNUSED": syscall.SIGUNUSED,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

View file

@ -0,0 +1,19 @@
// +build !windows
package signal
import (
"syscall"
)
// Signals used in api/client (no windows equivalent, use
// invalid signals so they don't get handled)
const (
// SIGCHLD is a signal sent to a process when a child process terminates, is interrupted, or resumes after being interrupted.
SIGCHLD = syscall.SIGCHLD
// SIGWINCH is a signal sent to a process when its controlling terminal changes its size
SIGWINCH = syscall.SIGWINCH
// DefaultStopSignal is the syscall signal used to stop a container in unix systems.
DefaultStopSignal = "SIGTERM"
)

View file

@ -0,0 +1,10 @@
// +build !linux,!darwin,!freebsd
package signal
import (
"syscall"
)
// SignalMap is an empty map of signals for unsupported platform.
var SignalMap = map[string]syscall.Signal{}

View file

@ -0,0 +1,16 @@
// +build windows
package signal
import (
"syscall"
)
// Signals used in api/client (no windows equivalent, use
// invalid signals so they don't get handled)
const (
SIGCHLD = syscall.Signal(0xff)
SIGWINCH = syscall.Signal(0xff)
// DefaultStopSignal is the syscall signal used to stop a container in windows systems.
DefaultStopSignal = "15"
)

View file

@ -0,0 +1,74 @@
package signal
import (
"os"
gosignal "os/signal"
"runtime"
"sync/atomic"
"syscall"
"github.com/Sirupsen/logrus"
)
// Trap sets up a simplified signal "trap", appropriate for common
// behavior expected from a vanilla unix command-line tool in general
// (and the Docker engine in particular).
//
// * If SIGINT or SIGTERM are received, `cleanup` is called, then the process is terminated.
// * If SIGINT or SIGTERM are received 3 times before cleanup is complete, then cleanup is
// skipped and the process is terminated immediately (allows force quit of stuck daemon)
// * A SIGQUIT always causes an exit without cleanup, with a goroutine dump preceding exit.
//
func Trap(cleanup func()) {
c := make(chan os.Signal, 1)
// we will handle INT, TERM, QUIT here
signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT}
gosignal.Notify(c, signals...)
go func() {
interruptCount := uint32(0)
for sig := range c {
go func(sig os.Signal) {
logrus.Infof("Processing signal '%v'", sig)
switch sig {
case os.Interrupt, syscall.SIGTERM:
if atomic.LoadUint32(&interruptCount) < 3 {
// Initiate the cleanup only once
if atomic.AddUint32(&interruptCount, 1) == 1 {
// Call the provided cleanup handler
cleanup()
os.Exit(0)
} else {
return
}
} else {
// 3 SIGTERM/INT signals received; force exit without cleanup
logrus.Infof("Forcing docker daemon shutdown without cleanup; 3 interrupts received")
}
case syscall.SIGQUIT:
DumpStacks()
logrus.Infof("Forcing docker daemon shutdown without cleanup on SIGQUIT")
}
//for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal #
os.Exit(128 + int(sig.(syscall.Signal)))
}(sig)
}
}()
}
// DumpStacks dumps the runtime stack.
func DumpStacks() {
var (
buf []byte
stackSize int
)
bufferLen := 16384
for stackSize == len(buf) {
buf = make([]byte, bufferLen)
stackSize = runtime.Stack(buf, true)
bufferLen *= 2
}
buf = buf[:stackSize]
// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
// traces won't show up in the log.
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
}

View file

@ -0,0 +1,48 @@
// Package sockets provides helper functions to create and configure Unix or TCP
// sockets.
package sockets
import (
"crypto/tls"
"net"
"net/http"
"time"
"github.com/docker/docker/pkg/listenbuffer"
)
// NewTCPSocket creates a TCP socket listener with the specified address and
// and the specified tls configuration. If TLSConfig is set, will encapsulate the
// TCP listener inside a TLS one.
// The channel passed is used to activate the listenbuffer when the caller is ready
// to accept connections.
func NewTCPSocket(addr string, tlsConfig *tls.Config, activate <-chan struct{}) (net.Listener, error) {
l, err := listenbuffer.NewListenBuffer("tcp", addr, activate)
if err != nil {
return nil, err
}
if tlsConfig != nil {
tlsConfig.NextProtos = []string{"http/1.1"}
l = tls.NewListener(l, tlsConfig)
}
return l, nil
}
// ConfigureTCPTransport configures the specified Transport according to the
// specified proto and addr.
// If the proto is unix (using a unix socket to communicate) the compression
// is disabled.
func ConfigureTCPTransport(tr *http.Transport, proto, addr string) {
// Why 32? See https://github.com/docker/docker/pull/8035.
timeout := 32 * time.Second
if proto == "unix" {
// No need for compression in local communications.
tr.DisableCompression = true
tr.Dial = func(_, _ string) (net.Conn, error) {
return net.DialTimeout(proto, addr, timeout)
}
} else {
tr.Proxy = http.ProxyFromEnvironment
tr.Dial = (&net.Dialer{Timeout: timeout}).Dial
}
}

View file

@ -0,0 +1,83 @@
// +build linux freebsd
package sockets
import (
"fmt"
"net"
"os"
"strconv"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/listenbuffer"
"github.com/opencontainers/runc/libcontainer/user"
)
// NewUnixSocket creates a unix socket with the specified path and group.
// The channel passed is used to activate the listenbuffer when the caller is ready
// to accept connections.
func NewUnixSocket(path, group string, activate <-chan struct{}) (net.Listener, error) {
if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
return nil, err
}
mask := syscall.Umask(0777)
defer syscall.Umask(mask)
l, err := listenbuffer.NewListenBuffer("unix", path, activate)
if err != nil {
return nil, err
}
if err := setSocketGroup(path, group); err != nil {
l.Close()
return nil, err
}
if err := os.Chmod(path, 0660); err != nil {
l.Close()
return nil, err
}
return l, nil
}
func setSocketGroup(path, group string) error {
if group == "" {
return nil
}
if err := changeGroup(path, group); err != nil {
if group != "docker" {
return err
}
logrus.Debugf("Warning: could not change group %s to docker: %v", path, err)
}
return nil
}
func changeGroup(path string, nameOrGid string) error {
gid, err := lookupGidByName(nameOrGid)
if err != nil {
return err
}
logrus.Debugf("%s group found. gid: %d", nameOrGid, gid)
return os.Chown(path, 0, gid)
}
func lookupGidByName(nameOrGid string) (int, error) {
groupFile, err := user.GetGroupPath()
if err != nil {
return -1, err
}
groups, err := user.ParseGroupFileFilter(groupFile, func(g user.Group) bool {
return g.Name == nameOrGid || strconv.Itoa(g.Gid) == nameOrGid
})
if err != nil {
return -1, err
}
if groups != nil && len(groups) > 0 {
return groups[0].Gid, nil
}
gid, err := strconv.Atoi(nameOrGid)
if err == nil {
logrus.Warnf("Could not find GID %d", gid)
return gid, nil
}
return -1, fmt.Errorf("Group %s not found", nameOrGid)
}

View file

@ -1,3 +1,4 @@
// Package stringid provides helper functions for dealing with string identifiers
package stringid
import (
@ -6,13 +7,15 @@ import (
"io"
"regexp"
"strconv"
"github.com/docker/docker/pkg/random"
)
const shortLen = 12
var validShortID = regexp.MustCompile("^[a-z0-9]{12}$")
// Determine if an arbitrary string *looks like* a short ID.
// IsShortID determines if an arbitrary string *looks like* a short ID.
func IsShortID(id string) bool {
return validShortID.MatchString(id)
}
@ -29,20 +32,36 @@ func TruncateID(id string) string {
return id[:trimTo]
}
// GenerateRandomID returns an unique id
func GenerateRandomID() string {
func generateID(crypto bool) string {
b := make([]byte, 32)
var r io.Reader = random.Reader
if crypto {
r = rand.Reader
}
for {
id := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, id); err != nil {
if _, err := io.ReadFull(r, b); err != nil {
panic(err) // This shouldn't happen
}
value := hex.EncodeToString(id)
id := hex.EncodeToString(b)
// if we try to parse the truncated for as an int and we don't have
// an error then the value is all numberic and causes issues when
// used as a hostname. ref #3869
if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil {
if _, err := strconv.ParseInt(TruncateID(id), 10, 64); err == nil {
continue
}
return value
return id
}
}
// GenerateRandomID returns an unique id.
func GenerateRandomID() string {
return generateID(true)
}
// GenerateNonCryptoID generates unique id without using cryptographically
// secure sources of random.
// It helps you to save entropy.
func GenerateNonCryptoID() string {
return generateID(false)
}

View file

@ -1,4 +1,5 @@
Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks
Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks,
as well as a Windows long-path aware version of filepath.EvalSymlinks
from the [Go standard library](https://golang.org/pkg/path/filepath).
The code from filepath.EvalSymlinks has been adapted in fs.go.

View file

@ -12,15 +12,18 @@ import (
"os"
"path/filepath"
"strings"
"github.com/docker/docker/pkg/system"
)
// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an absolute path
// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
// absolute path. This function handles paths in a platform-agnostic manner.
func FollowSymlinkInScope(path, root string) (string, error) {
path, err := filepath.Abs(path)
path, err := filepath.Abs(filepath.FromSlash(path))
if err != nil {
return "", err
}
root, err = filepath.Abs(root)
root, err = filepath.Abs(filepath.FromSlash(root))
if err != nil {
return "", err
}
@ -119,7 +122,7 @@ func evalSymlinksInScope(path, root string) (string, error) {
if err != nil {
return "", err
}
if filepath.IsAbs(dest) {
if system.IsAbs(dest) {
b.Reset()
}
path = dest + string(filepath.Separator) + path
@ -129,3 +132,12 @@ func evalSymlinksInScope(path, root string) (string, error) {
// what's happening here
return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil
}
// EvalSymlinks returns the path name after the evaluation of any symbolic
// links.
// If path is relative the result will be relative to the current directory,
// unless one of the components is an absolute symbolic link.
// This version has been updated to support long paths prepended with `\\?\`.
func EvalSymlinks(path string) (string, error) {
return evalSymlinks(path)
}

View file

@ -0,0 +1,11 @@
// +build !windows
package symlink
import (
"path/filepath"
)
func evalSymlinks(path string) (string, error) {
return filepath.EvalSymlinks(path)
}

View file

@ -0,0 +1,156 @@
package symlink
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/docker/docker/pkg/longpath"
)
func toShort(path string) (string, error) {
p, err := syscall.UTF16FromString(path)
if err != nil {
return "", err
}
b := p // GetShortPathName says we can reuse buffer
n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
if n > uint32(len(b)) {
b = make([]uint16, n)
n, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
}
return syscall.UTF16ToString(b), nil
}
func toLong(path string) (string, error) {
p, err := syscall.UTF16FromString(path)
if err != nil {
return "", err
}
b := p // GetLongPathName says we can reuse buffer
n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
if n > uint32(len(b)) {
b = make([]uint16, n)
n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
}
b = b[:n]
return syscall.UTF16ToString(b), nil
}
func evalSymlinks(path string) (string, error) {
path, err := walkSymlinks(path)
if err != nil {
return "", err
}
p, err := toShort(path)
if err != nil {
return "", err
}
p, err = toLong(p)
if err != nil {
return "", err
}
// syscall.GetLongPathName does not change the case of the drive letter,
// but the result of EvalSymlinks must be unique, so we have
// EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`).
// Make drive letter upper case.
if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' {
p = string(p[0]+'A'-'a') + p[1:]
} else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' {
p = p[:3] + string(p[4]+'A'-'a') + p[5:]
}
return filepath.Clean(p), nil
}
const utf8RuneSelf = 0x80
func walkSymlinks(path string) (string, error) {
const maxIter = 255
originalPath := path
// consume path by taking each frontmost path element,
// expanding it if it's a symlink, and appending it to b
var b bytes.Buffer
for n := 0; path != ""; n++ {
if n > maxIter {
return "", errors.New("EvalSymlinks: too many links in " + originalPath)
}
// A path beginnging with `\\?\` represents the root, so automatically
// skip that part and begin processing the next segment.
if strings.HasPrefix(path, longpath.Prefix) {
b.WriteString(longpath.Prefix)
path = path[4:]
continue
}
// find next path component, p
var i = -1
for j, c := range path {
if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) {
i = j
break
}
}
var p string
if i == -1 {
p, path = path, ""
} else {
p, path = path[:i], path[i+1:]
}
if p == "" {
if b.Len() == 0 {
// must be absolute path
b.WriteRune(filepath.Separator)
}
continue
}
// If this is the first segment after the long path prefix, accept the
// current segment as a volume root or UNC share and move on to the next.
if b.String() == longpath.Prefix {
b.WriteString(p)
b.WriteRune(filepath.Separator)
continue
}
fi, err := os.Lstat(b.String() + p)
if err != nil {
return "", err
}
if fi.Mode()&os.ModeSymlink == 0 {
b.WriteString(p)
if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') {
b.WriteRune(filepath.Separator)
}
continue
}
// it's a symlink, put it at the front of path
dest, err := os.Readlink(b.String() + p)
if err != nil {
return "", err
}
if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) {
b.Reset()
}
path = dest + string(filepath.Separator) + path
}
return filepath.Clean(b.String()), nil
}

View file

@ -0,0 +1,10 @@
package system
import (
"errors"
)
var (
// ErrNotSupportedPlatform means the platform is not supported.
ErrNotSupportedPlatform = errors.New("platform and architecture is not supported")
)

View file

@ -0,0 +1,83 @@
package system
// This file implements syscalls for Win32 events which are not implemented
// in golang.
import (
"syscall"
"unsafe"
)
var (
procCreateEvent = modkernel32.NewProc("CreateEventW")
procOpenEvent = modkernel32.NewProc("OpenEventW")
procSetEvent = modkernel32.NewProc("SetEvent")
procResetEvent = modkernel32.NewProc("ResetEvent")
procPulseEvent = modkernel32.NewProc("PulseEvent")
)
// CreateEvent implements win32 CreateEventW func in golang. It will create an event object.
func CreateEvent(eventAttributes *syscall.SecurityAttributes, manualReset bool, initialState bool, name string) (handle syscall.Handle, err error) {
namep, _ := syscall.UTF16PtrFromString(name)
var _p1 uint32
if manualReset {
_p1 = 1
}
var _p2 uint32
if initialState {
_p2 = 1
}
r0, _, e1 := procCreateEvent.Call(uintptr(unsafe.Pointer(eventAttributes)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(namep)))
use(unsafe.Pointer(namep))
handle = syscall.Handle(r0)
if handle == syscall.InvalidHandle {
err = e1
}
return
}
// OpenEvent implements win32 OpenEventW func in golang. It opens an event object.
func OpenEvent(desiredAccess uint32, inheritHandle bool, name string) (handle syscall.Handle, err error) {
namep, _ := syscall.UTF16PtrFromString(name)
var _p1 uint32
if inheritHandle {
_p1 = 1
}
r0, _, e1 := procOpenEvent.Call(uintptr(desiredAccess), uintptr(_p1), uintptr(unsafe.Pointer(namep)))
use(unsafe.Pointer(namep))
handle = syscall.Handle(r0)
if handle == syscall.InvalidHandle {
err = e1
}
return
}
// SetEvent implements win32 SetEvent func in golang.
func SetEvent(handle syscall.Handle) (err error) {
return setResetPulse(handle, procSetEvent)
}
// ResetEvent implements win32 ResetEvent func in golang.
func ResetEvent(handle syscall.Handle) (err error) {
return setResetPulse(handle, procResetEvent)
}
// PulseEvent implements win32 PulseEvent func in golang.
func PulseEvent(handle syscall.Handle) (err error) {
return setResetPulse(handle, procPulseEvent)
}
func setResetPulse(handle syscall.Handle, proc *syscall.LazyProc) (err error) {
r0, _, _ := proc.Call(uintptr(handle))
if r0 != 0 {
err = syscall.Errno(r0)
}
return
}
var temp unsafe.Pointer
// use ensures a variable is kept alive without the GC freeing while still needed
func use(p unsafe.Pointer) {
temp = p
}

View file

@ -0,0 +1,19 @@
// +build !windows
package system
import (
"os"
"path/filepath"
)
// MkdirAll creates a directory named path along with any necessary parents,
// with permission specified by attribute perm for all dir created.
func MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
// IsAbs is a platform-specific wrapper for filepath.IsAbs.
func IsAbs(path string) bool {
return filepath.IsAbs(path)
}

View file

@ -0,0 +1,82 @@
// +build windows
package system
import (
"os"
"path/filepath"
"regexp"
"strings"
"syscall"
)
// MkdirAll implementation that is volume path aware for Windows.
func MkdirAll(path string, perm os.FileMode) error {
if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) {
return nil
}
// The rest of this method is copied from os.MkdirAll and should be kept
// as-is to ensure compatibility.
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return &os.PathError{
Op: "mkdir",
Path: path,
Err: syscall.ENOTDIR,
}
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent
err = MkdirAll(path[0:j-1], perm)
if err != nil {
return err
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, perm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
return nil
}
// IsAbs is a platform-specific wrapper for filepath.IsAbs. On Windows,
// golang filepath.IsAbs does not consider a path \windows\system32 as absolute
// as it doesn't start with a drive-letter/colon combination. However, in
// docker we need to verify things such as WORKDIR /windows/system32 in
// a Dockerfile (which gets translated to \windows\system32 when being processed
// by the daemon. This SHOULD be treated as absolute from a docker processing
// perspective.
func IsAbs(path string) bool {
if !filepath.IsAbs(path) {
if !strings.HasPrefix(path, string(os.PathSeparator)) {
return false
}
}
return true
}

View file

@ -0,0 +1,19 @@
// +build !windows
package system
import (
"syscall"
)
// Lstat takes a path to a file and returns
// a system.StatT type pertaining to that file.
//
// Throws an error if the file does not exist
func Lstat(path string) (*StatT, error) {
s := &syscall.Stat_t{}
if err := syscall.Lstat(path, s); err != nil {
return nil, err
}
return fromStatT(s)
}

View file

@ -0,0 +1,28 @@
package system
import (
"os"
"testing"
)
// TestLstat tests Lstat for existing and non existing files
func TestLstat(t *testing.T) {
file, invalid, _, dir := prepareFiles(t)
defer os.RemoveAll(dir)
statFile, err := Lstat(file)
if err != nil {
t.Fatal(err)
}
if statFile == nil {
t.Fatal("returned empty stat for existing file")
}
statInvalid, err := Lstat(invalid)
if err == nil {
t.Fatal("did not return error for non-existing file")
}
if statInvalid != nil {
t.Fatal("returned non-nil stat for non-existing file")
}
}

View file

@ -0,0 +1,25 @@
// +build windows
package system
import (
"os"
)
// Lstat calls os.Lstat to get a fileinfo interface back.
// This is then copied into our own locally defined structure.
// Note the Linux version uses fromStatT to do the copy back,
// but that not strictly necessary when already in an OS specific module.
func Lstat(path string) (*StatT, error) {
fi, err := os.Lstat(path)
if err != nil {
return nil, err
}
return &StatT{
name: fi.Name(),
size: fi.Size(),
mode: fi.Mode(),
modTime: fi.ModTime(),
isDir: fi.IsDir()}, nil
}

View file

@ -0,0 +1,17 @@
package system
// MemInfo contains memory statistics of the host system.
type MemInfo struct {
// Total usable RAM (i.e. physical RAM minus a few reserved bits and the
// kernel binary code).
MemTotal int64
// Amount of free memory.
MemFree int64
// Total amount of swap space available.
SwapTotal int64
// Amount of swap space that is currently unused.
SwapFree int64
}

View file

@ -0,0 +1,66 @@
package system
import (
"bufio"
"io"
"os"
"strconv"
"strings"
"github.com/docker/docker/pkg/units"
)
// ReadMemInfo retrieves memory statistics of the host system and returns a
// MemInfo type.
func ReadMemInfo() (*MemInfo, error) {
file, err := os.Open("/proc/meminfo")
if err != nil {
return nil, err
}
defer file.Close()
return parseMemInfo(file)
}
// parseMemInfo parses the /proc/meminfo file into
// a MemInfo object given a io.Reader to the file.
//
// Throws error if there are problems reading from the file
func parseMemInfo(reader io.Reader) (*MemInfo, error) {
meminfo := &MemInfo{}
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
// Expected format: ["MemTotal:", "1234", "kB"]
parts := strings.Fields(scanner.Text())
// Sanity checks: Skip malformed entries.
if len(parts) < 3 || parts[2] != "kB" {
continue
}
// Convert to bytes.
size, err := strconv.Atoi(parts[1])
if err != nil {
continue
}
bytes := int64(size) * units.KiB
switch parts[0] {
case "MemTotal:":
meminfo.MemTotal = bytes
case "MemFree:":
meminfo.MemFree = bytes
case "SwapTotal:":
meminfo.SwapTotal = bytes
case "SwapFree:":
meminfo.SwapFree = bytes
}
}
// Handle errors that may have occurred during the reading of the file.
if err := scanner.Err(); err != nil {
return nil, err
}
return meminfo, nil
}

View file

@ -0,0 +1,38 @@
package system
import (
"strings"
"testing"
"github.com/docker/docker/pkg/units"
)
// TestMemInfo tests parseMemInfo with a static meminfo string
func TestMemInfo(t *testing.T) {
const input = `
MemTotal: 1 kB
MemFree: 2 kB
SwapTotal: 3 kB
SwapFree: 4 kB
Malformed1:
Malformed2: 1
Malformed3: 2 MB
Malformed4: X kB
`
meminfo, err := parseMemInfo(strings.NewReader(input))
if err != nil {
t.Fatal(err)
}
if meminfo.MemTotal != 1*units.KiB {
t.Fatalf("Unexpected MemTotal: %d", meminfo.MemTotal)
}
if meminfo.MemFree != 2*units.KiB {
t.Fatalf("Unexpected MemFree: %d", meminfo.MemFree)
}
if meminfo.SwapTotal != 3*units.KiB {
t.Fatalf("Unexpected SwapTotal: %d", meminfo.SwapTotal)
}
if meminfo.SwapFree != 4*units.KiB {
t.Fatalf("Unexpected SwapFree: %d", meminfo.SwapFree)
}
}

View file

@ -0,0 +1,8 @@
// +build !linux,!windows
package system
// ReadMemInfo is not supported on platforms other than linux and windows.
func ReadMemInfo() (*MemInfo, error) {
return nil, ErrNotSupportedPlatform
}

View file

@ -0,0 +1,44 @@
package system
import (
"syscall"
"unsafe"
)
var (
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx")
)
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366770(v=vs.85).aspx
type memorystatusex struct {
dwLength uint32
dwMemoryLoad uint32
ullTotalPhys uint64
ullAvailPhys uint64
ullTotalPageFile uint64
ullAvailPageFile uint64
ullTotalVirtual uint64
ullAvailVirtual uint64
ullAvailExtendedVirtual uint64
}
// ReadMemInfo retrieves memory statistics of the host system and returns a
// MemInfo type.
func ReadMemInfo() (*MemInfo, error) {
msi := &memorystatusex{
dwLength: 64,
}
r1, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msi)))
if r1 == 0 {
return &MemInfo{}, nil
}
return &MemInfo{
MemTotal: int64(msi.ullTotalPhys),
MemFree: int64(msi.ullAvailPhys),
SwapTotal: int64(msi.ullTotalPageFile),
SwapFree: int64(msi.ullAvailPageFile),
}, nil
}

View file

@ -0,0 +1,22 @@
// +build !windows
package system
import (
"syscall"
)
// Mknod creates a filesystem node (file, device special file or named pipe) named path
// with attributes specified by mode and dev.
func Mknod(path string, mode uint32, dev int) error {
return syscall.Mknod(path, mode, dev)
}
// Mkdev is used to build the value of linux devices (in /dev/) which specifies major
// and minor number of the newly created device special file.
// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes.
// They are, from low to high: the lower 8 bits of the minor, then 12 bits of the major,
// then the top 12 bits of the minor.
func Mkdev(major int64, minor int64) uint32 {
return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff))
}

View file

@ -0,0 +1,13 @@
// +build windows
package system
// Mknod is not implemented on Windows.
func Mknod(path string, mode uint32, dev int) error {
return ErrNotSupportedPlatform
}
// Mkdev is not implemented on Windows.
func Mkdev(major int64, minor int64) uint32 {
panic("Mkdev not implemented on Windows.")
}

View file

@ -0,0 +1,53 @@
// +build !windows
package system
import (
"syscall"
)
// StatT type contains status of a file. It contains metadata
// like permission, owner, group, size, etc about a file.
type StatT struct {
mode uint32
uid uint32
gid uint32
rdev uint64
size int64
mtim syscall.Timespec
}
// Mode returns file's permission mode.
func (s StatT) Mode() uint32 {
return s.mode
}
// UID returns file's user id of owner.
func (s StatT) UID() uint32 {
return s.uid
}
// Gid returns file's group id of owner.
func (s StatT) Gid() uint32 {
return s.gid
}
// Rdev returns file's device ID (if it's special file).
func (s StatT) Rdev() uint64 {
return s.rdev
}
// Size returns file's size.
func (s StatT) Size() int64 {
return s.size
}
// Mtim returns file's last modification time.
func (s StatT) Mtim() syscall.Timespec {
return s.mtim
}
// GetLastModification returns file's last modification time.
func (s StatT) GetLastModification() syscall.Timespec {
return s.Mtim()
}

View file

@ -0,0 +1,27 @@
package system
import (
"syscall"
)
// fromStatT converts a syscall.Stat_t type to a system.Stat_t type
func fromStatT(s *syscall.Stat_t) (*StatT, error) {
return &StatT{size: s.Size,
mode: uint32(s.Mode),
uid: s.Uid,
gid: s.Gid,
rdev: uint64(s.Rdev),
mtim: s.Mtimespec}, nil
}
// Stat takes a path to a file and returns
// a system.Stat_t type pertaining to that file.
//
// Throws an error if the file does not exist
func Stat(path string) (*StatT, error) {
s := &syscall.Stat_t{}
if err := syscall.Stat(path, s); err != nil {
return nil, err
}
return fromStatT(s)
}

View file

@ -0,0 +1,33 @@
package system
import (
"syscall"
)
// fromStatT converts a syscall.Stat_t type to a system.Stat_t type
func fromStatT(s *syscall.Stat_t) (*StatT, error) {
return &StatT{size: s.Size,
mode: s.Mode,
uid: s.Uid,
gid: s.Gid,
rdev: s.Rdev,
mtim: s.Mtim}, nil
}
// FromStatT exists only on linux, and loads a system.StatT from a
// syscal.Stat_t.
func FromStatT(s *syscall.Stat_t) (*StatT, error) {
return fromStatT(s)
}
// Stat takes a path to a file and returns
// a system.StatT type pertaining to that file.
//
// Throws an error if the file does not exist
func Stat(path string) (*StatT, error) {
s := &syscall.Stat_t{}
if err := syscall.Stat(path, s); err != nil {
return nil, err
}
return fromStatT(s)
}

View file

@ -0,0 +1,37 @@
package system
import (
"os"
"syscall"
"testing"
)
// TestFromStatT tests fromStatT for a tempfile
func TestFromStatT(t *testing.T) {
file, _, _, dir := prepareFiles(t)
defer os.RemoveAll(dir)
stat := &syscall.Stat_t{}
err := syscall.Lstat(file, stat)
s, err := fromStatT(stat)
if err != nil {
t.Fatal(err)
}
if stat.Mode != s.Mode() {
t.Fatal("got invalid mode")
}
if stat.Uid != s.UID() {
t.Fatal("got invalid uid")
}
if stat.Gid != s.Gid() {
t.Fatal("got invalid gid")
}
if stat.Rdev != s.Rdev() {
t.Fatal("got invalid rdev")
}
if stat.Mtim != s.Mtim() {
t.Fatal("got invalid mtim")
}
}

View file

@ -0,0 +1,17 @@
// +build !linux,!windows,!freebsd
package system
import (
"syscall"
)
// fromStatT creates a system.StatT type from a syscall.Stat_t type
func fromStatT(s *syscall.Stat_t) (*StatT, error) {
return &StatT{size: s.Size,
mode: uint32(s.Mode),
uid: s.Uid,
gid: s.Gid,
rdev: uint64(s.Rdev),
mtim: s.Mtimespec}, nil
}

View file

@ -0,0 +1,43 @@
// +build windows
package system
import (
"os"
"time"
)
// StatT type contains status of a file. It contains metadata
// like name, permission, size, etc about a file.
type StatT struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
// Name returns file's name.
func (s StatT) Name() string {
return s.name
}
// Size returns file's size.
func (s StatT) Size() int64 {
return s.size
}
// Mode returns file's permission mode.
func (s StatT) Mode() os.FileMode {
return s.mode
}
// ModTime returns file's last modification time.
func (s StatT) ModTime() time.Time {
return s.modTime
}
// IsDir returns whether file is actually a directory.
func (s StatT) IsDir() bool {
return s.isDir
}

View file

@ -0,0 +1,13 @@
// +build !windows
package system
import (
"syscall"
)
// Umask sets current process's file mode creation mask to newmask
// and return oldmask.
func Umask(newmask int) (oldmask int, err error) {
return syscall.Umask(newmask), nil
}

View file

@ -0,0 +1,9 @@
// +build windows
package system
// Umask is not supported on the windows platform.
func Umask(newmask int) (oldmask int, err error) {
// should not be called on cli code path
return 0, ErrNotSupportedPlatform
}

View file

@ -0,0 +1,14 @@
package system
import "syscall"
// LUtimesNano is not supported by darwin platform.
func LUtimesNano(path string, ts []syscall.Timespec) error {
return ErrNotSupportedPlatform
}
// UtimesNano is used to change access and modification time of path.
// it can't be used for symbol link file.
func UtimesNano(path string, ts []syscall.Timespec) error {
return syscall.UtimesNano(path, ts)
}

View file

@ -0,0 +1,28 @@
package system
import (
"syscall"
"unsafe"
)
// LUtimesNano is used to change access and modification time of the specified path.
// It's used for symbol link file because syscall.UtimesNano doesn't support a NOFOLLOW flag atm.
func LUtimesNano(path string, ts []syscall.Timespec) error {
var _path *byte
_path, err := syscall.BytePtrFromString(path)
if err != nil {
return err
}
if _, _, err := syscall.Syscall(syscall.SYS_LUTIMES, uintptr(unsafe.Pointer(_path)), uintptr(unsafe.Pointer(&ts[0])), 0); err != 0 && err != syscall.ENOSYS {
return err
}
return nil
}
// UtimesNano is used to change access and modification time of the specified path.
// It can't be used for symbol link file.
func UtimesNano(path string, ts []syscall.Timespec) error {
return syscall.UtimesNano(path, ts)
}

Some files were not shown because too many files have changed in this diff Show more