From 1666707cf895e9de63aedaf744ae4923c41f73d2 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 5 Jan 2016 18:34:22 -0500 Subject: [PATCH 1/3] Vendor merge structure tool. Signed-off-by: David Calavera --- hack/vendor.sh | 1 + .../src/github.com/imdario/mergo/.travis.yml | 2 + vendor/src/github.com/imdario/mergo/LICENSE | 28 ++++ vendor/src/github.com/imdario/mergo/README.md | 122 ++++++++++++++ vendor/src/github.com/imdario/mergo/doc.go | 44 +++++ vendor/src/github.com/imdario/mergo/map.go | 154 ++++++++++++++++++ vendor/src/github.com/imdario/mergo/merge.go | 120 ++++++++++++++ vendor/src/github.com/imdario/mergo/mergo.go | 90 ++++++++++ 8 files changed, 561 insertions(+) create mode 100644 vendor/src/github.com/imdario/mergo/.travis.yml create mode 100644 vendor/src/github.com/imdario/mergo/LICENSE create mode 100644 vendor/src/github.com/imdario/mergo/README.md create mode 100644 vendor/src/github.com/imdario/mergo/doc.go create mode 100644 vendor/src/github.com/imdario/mergo/map.go create mode 100644 vendor/src/github.com/imdario/mergo/merge.go create mode 100644 vendor/src/github.com/imdario/mergo/mergo.go diff --git a/hack/vendor.sh b/hack/vendor.sh index 9e94cfc596..f5cb30b9b2 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -24,6 +24,7 @@ clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 clone git github.com/docker/go-connections v0.1.2 clone git github.com/docker/engine-api v0.2.1 clone git github.com/RackSec/srslog 6eb773f331e46fbba8eecb8e794e635e75fc04de +clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages clone git github.com/docker/libnetwork v0.5.4 diff --git a/vendor/src/github.com/imdario/mergo/.travis.yml b/vendor/src/github.com/imdario/mergo/.travis.yml new file mode 100644 index 0000000000..9d91c6339f --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/.travis.yml @@ -0,0 +1,2 @@ +language: go +install: go get -t diff --git a/vendor/src/github.com/imdario/mergo/LICENSE b/vendor/src/github.com/imdario/mergo/LICENSE new file mode 100644 index 0000000000..686680298d --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/src/github.com/imdario/mergo/README.md b/vendor/src/github.com/imdario/mergo/README.md new file mode 100644 index 0000000000..4f0f990fc1 --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/README.md @@ -0,0 +1,122 @@ +# Mergo + +A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. + +Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region Marche. + +![Mergo dall'alto](http://www.comune.mergo.an.it/Siti/Mergo/Immagini/Foto/mergo_dall_alto.jpg) + +## Status + +It is ready for production use. It works fine after extensive use in the wild. + +[![Build Status][1]][2] +[![GoDoc](https://godoc.org/github.com/imdario/mergo?status.svg)](https://godoc.org/github.com/imdario/mergo) + +[1]: https://travis-ci.org/imdario/mergo.png +[2]: https://travis-ci.org/imdario/mergo + +### Important note + +Mergo is intended to assign **only** zero value fields on destination with source value. Since April 6th it works like this. Before it didn't work properly, causing some random overwrites. After some issues and PRs I found it didn't merge as I designed it. Thanks to [imdario/mergo#8](https://github.com/imdario/mergo/pull/8) overwriting functions were added and the wrong behavior was clearly detected. + +If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0). + +### Mergo in the wild + +- [imdario/zas](https://github.com/imdario/zas) +- [GoogleCloudPlatform/kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) +- [soniah/dnsmadeeasy](https://github.com/soniah/dnsmadeeasy) +- [EagerIO/Stout](https://github.com/EagerIO/Stout) +- [lynndylanhurley/defsynth-api](https://github.com/lynndylanhurley/defsynth-api) +- [russross/canvasassignments](https://github.com/russross/canvasassignments) +- [rdegges/cryptly-api](https://github.com/rdegges/cryptly-api) +- [casualjim/exeggutor](https://github.com/casualjim/exeggutor) +- [divshot/gitling](https://github.com/divshot/gitling) +- [RWJMurphy/gorl](https://github.com/RWJMurphy/gorl) +- [andrerocker/deploy42](https://github.com/andrerocker/deploy42) +- [elwinar/rambler](https://github.com/elwinar/rambler) +- [tmaiaroto/gopartman](https://github.com/tmaiaroto/gopartman) +- [jfbus/impressionist](https://github.com/jfbus/impressionist) +- [Jmeyering/zealot](https://github.com/Jmeyering/zealot) +- [godep-migrator/rigger-host](https://github.com/godep-migrator/rigger-host) +- [Dronevery/MultiwaySwitch-Go](https://github.com/Dronevery/MultiwaySwitch-Go) +- [thoas/picfit](https://github.com/thoas/picfit) +- [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server) +- [jnuthong/item_search](https://github.com/jnuthong/item_search) + +## Installation + + go get github.com/imdario/mergo + + // use in your .go code + import ( + "github.com/imdario/mergo" + ) + +## Usage + +You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection). + + if err := mergo.Merge(&dst, src); err != nil { + // ... + } + +Additionally, you can map a map[string]interface{} to a struct (and otherwise, from struct to map), following the same restrictions as in Merge(). Keys are capitalized to find each corresponding exported field. + + if err := mergo.Map(&dst, srcMap); err != nil { + // ... + } + +Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as map[string]interface{}. They will be just assigned as values. + +More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo). + +### Nice example + +```go +package main + +import ( + "fmt" + "github.com/imdario/mergo" +) + +type Foo struct { + A string + B int64 +} + +func main() { + src := Foo{ + A: "one", + } + + dest := Foo{ + A: "two", + B: 2, + } + + mergo.Merge(&dest, src) + + fmt.Println(dest) + // Will print + // {two 2} +} +``` + +Note: if test are failing due missing package, please execute: + + go get gopkg.in/yaml.v1 + +## Contact me + +If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) + +## About + +Written by [Dario Castañé](http://dario.im). + +## License + +[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). diff --git a/vendor/src/github.com/imdario/mergo/doc.go b/vendor/src/github.com/imdario/mergo/doc.go new file mode 100644 index 0000000000..6e9aa7baf3 --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/doc.go @@ -0,0 +1,44 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mergo merges same-type structs and maps by setting default values in zero-value fields. + +Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection). + +Usage + +From my own work-in-progress project: + + type networkConfig struct { + Protocol string + Address string + ServerType string `json: "server_type"` + Port uint16 + } + + type FssnConfig struct { + Network networkConfig + } + + var fssnDefault = FssnConfig { + networkConfig { + "tcp", + "127.0.0.1", + "http", + 31560, + }, + } + + // Inside a function [...] + + if err := mergo.Merge(&config, fssnDefault); err != nil { + log.Fatal(err) + } + + // More code [...] + +*/ +package mergo diff --git a/vendor/src/github.com/imdario/mergo/map.go b/vendor/src/github.com/imdario/mergo/map.go new file mode 100644 index 0000000000..1ed3d716aa --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/map.go @@ -0,0 +1,154 @@ +// Copyright 2014 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "fmt" + "reflect" + "unicode" + "unicode/utf8" +) + +func changeInitialCase(s string, mapper func(rune) rune) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(mapper(r)) + s[n:] +} + +func isExported(field reflect.StructField) bool { + r, _ := utf8.DecodeRuneInString(field.Name) + return r >= 'A' && r <= 'Z' +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, overwrite bool) (err error) { + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + zeroValue := reflect.Value{} + switch dst.Kind() { + case reflect.Map: + dstMap := dst.Interface().(map[string]interface{}) + for i, n := 0, src.NumField(); i < n; i++ { + srcType := src.Type() + field := srcType.Field(i) + if !isExported(field) { + continue + } + fieldName := field.Name + fieldName = changeInitialCase(fieldName, unicode.ToLower) + if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) { + dstMap[fieldName] = src.Field(i).Interface() + } + } + case reflect.Struct: + srcMap := src.Interface().(map[string]interface{}) + for key := range srcMap { + srcValue := srcMap[key] + fieldName := changeInitialCase(key, unicode.ToUpper) + dstElement := dst.FieldByName(fieldName) + if dstElement == zeroValue { + // We discard it because the field doesn't exist. + continue + } + srcElement := reflect.ValueOf(srcValue) + dstKind := dstElement.Kind() + srcKind := srcElement.Kind() + if srcKind == reflect.Ptr && dstKind != reflect.Ptr { + srcElement = srcElement.Elem() + srcKind = reflect.TypeOf(srcElement.Interface()).Kind() + } else if dstKind == reflect.Ptr { + // Can this work? I guess it can't. + if srcKind != reflect.Ptr && srcElement.CanAddr() { + srcPtr := srcElement.Addr() + srcElement = reflect.ValueOf(srcPtr) + srcKind = reflect.Ptr + } + } + if !srcElement.IsValid() { + continue + } + if srcKind == dstKind { + if err = deepMerge(dstElement, srcElement, visited, depth+1, overwrite); err != nil { + return + } + } else { + if srcKind == reflect.Map { + if err = deepMap(dstElement, srcElement, visited, depth+1, overwrite); err != nil { + return + } + } else { + return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind) + } + } + } + } + return +} + +// Map sets fields' values in dst from src. +// src can be a map with string keys or a struct. dst must be the opposite: +// if src is a map, dst must be a valid pointer to struct. If src is a struct, +// dst must be map[string]interface{}. +// It won't merge unexported (private) fields and will do recursively +// any exported field. +// If dst is a map, keys will be src fields' names in lower camel case. +// Missing key in src that doesn't match a field in dst will be skipped. This +// doesn't apply if dst is a map. +// This is separated method from Merge because it is cleaner and it keeps sane +// semantics: merging equal types, mapping different (restricted) types. +func Map(dst, src interface{}) error { + return _map(dst, src, false) +} + +func MapWithOverwrite(dst, src interface{}) error { + return _map(dst, src, true) +} + +func _map(dst, src interface{}, overwrite bool) error { + var ( + vDst, vSrc reflect.Value + err error + ) + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + // To be friction-less, we redirect equal-type arguments + // to deepMerge. Only because arguments can be anything. + if vSrc.Kind() == vDst.Kind() { + return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, overwrite) + } + switch vSrc.Kind() { + case reflect.Struct: + if vDst.Kind() != reflect.Map { + return ErrExpectedMapAsDestination + } + case reflect.Map: + if vDst.Kind() != reflect.Struct { + return ErrExpectedStructAsDestination + } + default: + return ErrNotSupported + } + return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, overwrite) +} diff --git a/vendor/src/github.com/imdario/mergo/merge.go b/vendor/src/github.com/imdario/mergo/merge.go new file mode 100644 index 0000000000..a7dd9d82d1 --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/merge.go @@ -0,0 +1,120 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "reflect" +) + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, overwrite bool) (err error) { + if !src.IsValid() { + return + } + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + switch dst.Kind() { + case reflect.Struct: + for i, n := 0, dst.NumField(); i < n; i++ { + if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, overwrite); err != nil { + return + } + } + case reflect.Map: + for _, key := range src.MapKeys() { + srcElement := src.MapIndex(key) + if !srcElement.IsValid() { + continue + } + dstElement := dst.MapIndex(key) + switch srcElement.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Interface, reflect.Slice: + if srcElement.IsNil() { + continue + } + fallthrough + default: + switch reflect.TypeOf(srcElement.Interface()).Kind() { + case reflect.Struct: + fallthrough + case reflect.Ptr: + fallthrough + case reflect.Map: + if err = deepMerge(dstElement, srcElement, visited, depth+1, overwrite); err != nil { + return + } + } + } + if !isEmptyValue(srcElement) && (overwrite || (!dstElement.IsValid() || isEmptyValue(dst))) { + if dst.IsNil() { + dst.Set(reflect.MakeMap(dst.Type())) + } + dst.SetMapIndex(key, srcElement) + } + } + case reflect.Ptr: + fallthrough + case reflect.Interface: + if src.IsNil() { + break + } else if dst.IsNil() { + if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, overwrite); err != nil { + return + } + default: + if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } + return +} + +// Merge sets fields' values in dst from src if they have a zero +// value of their type. +// dst and src must be valid same-type structs and dst must be +// a pointer to struct. +// It won't merge unexported (private) fields and will do recursively +// any exported field. +func Merge(dst, src interface{}) error { + return merge(dst, src, false) +} + +func MergeWithOverwrite(dst, src interface{}) error { + return merge(dst, src, true) +} + +func merge(dst, src interface{}, overwrite bool) error { + var ( + vDst, vSrc reflect.Value + err error + ) + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + if vDst.Type() != vSrc.Type() { + return ErrDifferentArgumentsTypes + } + return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, overwrite) +} diff --git a/vendor/src/github.com/imdario/mergo/mergo.go b/vendor/src/github.com/imdario/mergo/mergo.go new file mode 100644 index 0000000000..f8a0991ec6 --- /dev/null +++ b/vendor/src/github.com/imdario/mergo/mergo.go @@ -0,0 +1,90 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "errors" + "reflect" +) + +// Errors reported by Mergo when it finds invalid arguments. +var ( + ErrNilArguments = errors.New("src and dst must not be nil") + ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") + ErrNotSupported = errors.New("only structs and maps are supported") + ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") + ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") +) + +// During deepMerge, must keep track of checks that are +// in progress. The comparison algorithm assumes that all +// checks in progress are true when it reencounters them. +// Visited are stored in a map indexed by 17 * a1 + a2; +type visit struct { + ptr uintptr + typ reflect.Type + next *visit +} + +// From src/pkg/encoding/json. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} + +func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { + if dst == nil || src == nil { + err = ErrNilArguments + return + } + vDst = reflect.ValueOf(dst).Elem() + if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { + err = ErrNotSupported + return + } + vSrc = reflect.ValueOf(src) + // We check if vSrc is a pointer to dereference it. + if vSrc.Kind() == reflect.Ptr { + vSrc = vSrc.Elem() + } + return +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) { + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + return // TODO refactor +} From 22a81a2c588a7505fd5f900e2093c44cd51ae142 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Wed, 6 Jan 2016 17:57:02 -0500 Subject: [PATCH 2/3] Add an unsafe memory discovery store for testing. Signed-off-by: David Calavera --- pkg/discovery/memory/memory.go | 82 +++++++++++++++++++++++++++++ pkg/discovery/memory/memory_test.go | 48 +++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 pkg/discovery/memory/memory.go create mode 100644 pkg/discovery/memory/memory_test.go diff --git a/pkg/discovery/memory/memory.go b/pkg/discovery/memory/memory.go new file mode 100644 index 0000000000..f389825e6f --- /dev/null +++ b/pkg/discovery/memory/memory.go @@ -0,0 +1,82 @@ +package memory + +import ( + "time" + + "github.com/docker/docker/pkg/discovery" +) + +// Discovery implements a descovery backend that keeps +// data in memory. +type Discovery struct { + heartbeat time.Duration + values []string +} + +func init() { + Init() +} + +// Init registers the memory backend on demand. +func Init() { + discovery.Register("memory", &Discovery{}) +} + +// Initialize sets the heartbeat for the memory backend. +func (s *Discovery) Initialize(_ string, heartbeat time.Duration, _ time.Duration, _ map[string]string) error { + s.heartbeat = heartbeat + return nil +} + +// Watch sends periodic discovery updates to a channel. +func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { + ch := make(chan discovery.Entries) + errCh := make(chan error) + ticker := time.NewTicker(s.heartbeat) + + go func() { + defer close(errCh) + defer close(ch) + + // Send the initial entries if available. + var currentEntries discovery.Entries + if len(s.values) > 0 { + var err error + currentEntries, err = discovery.CreateEntries(s.values) + if err != nil { + errCh <- err + } else { + ch <- currentEntries + } + } + + // Periodically send updates. + for { + select { + case <-ticker.C: + newEntries, err := discovery.CreateEntries(s.values) + if err != nil { + errCh <- err + continue + } + + // Check if the file has really changed. + if !newEntries.Equals(currentEntries) { + ch <- newEntries + } + currentEntries = newEntries + case <-stopCh: + ticker.Stop() + return + } + } + }() + + return ch, errCh +} + +// Register adds a new address to the discovery. +func (s *Discovery) Register(addr string) error { + s.values = append(s.values, addr) + return nil +} diff --git a/pkg/discovery/memory/memory_test.go b/pkg/discovery/memory/memory_test.go new file mode 100644 index 0000000000..c2da0a068e --- /dev/null +++ b/pkg/discovery/memory/memory_test.go @@ -0,0 +1,48 @@ +package memory + +import ( + "testing" + + "github.com/docker/docker/pkg/discovery" + "github.com/go-check/check" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { check.TestingT(t) } + +type discoverySuite struct{} + +var _ = check.Suite(&discoverySuite{}) + +func (s *discoverySuite) TestWatch(c *check.C) { + d := &Discovery{} + d.Initialize("foo", 1000, 0, nil) + stopCh := make(chan struct{}) + ch, errCh := d.Watch(stopCh) + + // We have to drain the error channel otherwise Watch will get stuck. + go func() { + for range errCh { + } + }() + + expected := discovery.Entries{ + &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, + } + + c.Assert(d.Register("1.1.1.1:1111"), check.IsNil) + c.Assert(<-ch, check.DeepEquals, expected) + + expected = discovery.Entries{ + &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, + &discovery.Entry{Host: "2.2.2.2", Port: "2222"}, + } + + c.Assert(d.Register("2.2.2.2:2222"), check.IsNil) + c.Assert(<-ch, check.DeepEquals, expected) + + // Stop and make sure it closes all channels. + close(stopCh) + c.Assert(<-ch, check.IsNil) + c.Assert(<-errCh, check.IsNil) +} From 677a6b3506107468ed8c00331991afd9176fa0b9 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Thu, 10 Dec 2015 18:35:10 -0500 Subject: [PATCH 3/3] Allow to set daemon and server configurations in a file. Read configuration after flags making this the priority: 1- Apply configuration from file. 2- Apply configuration from flags. Reload configuration when a signal is received, USR2 in Linux: - Reload router if the debug configuration changes. - Reload daemon labels. - Reload cluster discovery. Signed-off-by: David Calavera --- api/server/router_swapper.go | 30 ++++ api/server/server.go | 59 +++++-- daemon/config.go | 216 ++++++++++++++++++++---- daemon/config_test.go | 177 +++++++++++++++++++ daemon/config_unix.go | 16 +- daemon/daemon.go | 95 +++++++++-- daemon/daemon_test.go | 119 +++++++++++++ daemon/discovery.go | 110 ++++++++++-- daemon/discovery_test.go | 61 +++++++ daemon/info.go | 2 +- docker/common.go | 8 +- docker/daemon.go | 119 +++++++++---- docker/daemon_test.go | 91 ++++++++++ docker/daemon_unix.go | 15 ++ docker/daemon_windows.go | 24 +++ docs/reference/commandline/daemon.md | 79 ++++++++- integration-cli/docker_cli_help_test.go | 2 +- man/docker-daemon.8.md | 4 + opts/opts.go | 52 ++++++ opts/opts_test.go | 32 ++++ pkg/discovery/backends.go | 8 +- pkg/discovery/memory/memory.go | 1 + utils/debug.go | 26 +++ 23 files changed, 1218 insertions(+), 128 deletions(-) create mode 100644 api/server/router_swapper.go create mode 100644 daemon/config_test.go create mode 100644 docker/daemon_test.go create mode 100644 utils/debug.go diff --git a/api/server/router_swapper.go b/api/server/router_swapper.go new file mode 100644 index 0000000000..b5f1d06d8d --- /dev/null +++ b/api/server/router_swapper.go @@ -0,0 +1,30 @@ +package server + +import ( + "net/http" + "sync" + + "github.com/gorilla/mux" +) + +// routerSwapper is an http.Handler that allow you to swap +// mux routers. +type routerSwapper struct { + mu sync.Mutex + router *mux.Router +} + +// Swap changes the old router with the new one. +func (rs *routerSwapper) Swap(newRouter *mux.Router) { + rs.mu.Lock() + rs.router = newRouter + rs.mu.Unlock() +} + +// ServeHTTP makes the routerSwapper to implement the http.Handler interface. +func (rs *routerSwapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rs.mu.Lock() + router := rs.router + rs.mu.Unlock() + router.ServeHTTP(w, r) +} diff --git a/api/server/server.go b/api/server/server.go index 03200c414f..f312f23f60 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "net" "net/http" - "os" "strings" "github.com/Sirupsen/logrus" @@ -42,10 +41,11 @@ type Config struct { // Server contains instance details for the server type Server struct { - cfg *Config - servers []*HTTPServer - routers []router.Router - authZPlugins []authorization.Plugin + cfg *Config + servers []*HTTPServer + routers []router.Router + authZPlugins []authorization.Plugin + routerSwapper *routerSwapper } // Addr contains string representation of address and its protocol (tcp, unix...). @@ -80,12 +80,14 @@ func (s *Server) Close() { } } -// ServeAPI loops through all initialized servers and spawns goroutine -// with Server method for each. It sets CreateMux() as Handler also. -func (s *Server) ServeAPI() error { +// serveAPI loops through all initialized servers and spawns goroutine +// with Server method for each. It sets createMux() as Handler also. +func (s *Server) serveAPI() error { + s.initRouterSwapper() + var chErrors = make(chan error, len(s.servers)) for _, srv := range s.servers { - srv.srv.Handler = s.CreateMux() + srv.srv.Handler = s.routerSwapper go func(srv *HTTPServer) { var err error logrus.Infof("API listen on %s", srv.l.Addr()) @@ -186,11 +188,11 @@ func (s *Server) addRouter(r router.Router) { s.routers = append(s.routers, r) } -// CreateMux initializes the main router the server uses. +// createMux initializes the main router the server uses. // we keep enableCors just for legacy usage, need to be removed in the future -func (s *Server) CreateMux() *mux.Router { +func (s *Server) createMux() *mux.Router { m := mux.NewRouter() - if os.Getenv("DEBUG") != "" { + if utils.IsDebugEnabled() { profilerSetup(m, "/debug/") } @@ -207,3 +209,36 @@ func (s *Server) CreateMux() *mux.Router { return m } + +// Wait blocks the server goroutine until it exits. +// It sends an error message if there is any error during +// the API execution. +func (s *Server) Wait(waitChan chan error) { + if err := s.serveAPI(); err != nil { + logrus.Errorf("ServeAPI error: %v", err) + waitChan <- err + return + } + waitChan <- nil +} + +func (s *Server) initRouterSwapper() { + s.routerSwapper = &routerSwapper{ + router: s.createMux(), + } +} + +// Reload reads configuration changes and modifies the +// server according to those changes. +// Currently, only the --debug configuration is taken into account. +func (s *Server) Reload(config *daemon.Config) { + debugEnabled := utils.IsDebugEnabled() + switch { + case debugEnabled && !config.Debug: // disable debug + utils.DisableDebug() + s.routerSwapper.Swap(s.createMux()) + case config.Debug && !debugEnabled: // enable debug + utils.EnableDebug() + s.routerSwapper.Swap(s.createMux()) + } +} diff --git a/daemon/config.go b/daemon/config.go index 8356df846f..a75178faef 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -1,9 +1,19 @@ package daemon import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "strings" + "sync" + + "github.com/Sirupsen/logrus" "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/discovery" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/engine-api/types/container" + "github.com/imdario/mergo" ) const ( @@ -11,42 +21,69 @@ const ( disableNetworkBridge = "none" ) +// LogConfig represents the default log configuration. +// It includes json tags to deserialize configuration from a file +// using the same names that the flags in the command line uses. +type LogConfig struct { + Type string `json:"log-driver,omitempty"` + Config map[string]string `json:"log-opts,omitempty"` +} + +// CommonTLSOptions defines TLS configuration for the daemon server. +// It includes json tags to deserialize configuration from a file +// using the same names that the flags in the command line uses. +type CommonTLSOptions struct { + CAFile string `json:"tlscacert,omitempty"` + CertFile string `json:"tlscert,omitempty"` + KeyFile string `json:"tlskey,omitempty"` +} + // CommonConfig defines the configuration of a docker daemon which are // common across platforms. +// It includes json tags to deserialize configuration from a file +// using the same names that the flags in the command line uses. type CommonConfig struct { - AuthorizationPlugins []string // AuthorizationPlugins holds list of authorization plugins - AutoRestart bool - Bridge bridgeConfig // Bridge holds bridge network specific configuration. - Context map[string][]string - DisableBridge bool - DNS []string - DNSOptions []string - DNSSearch []string - ExecOptions []string - ExecRoot string - GraphDriver string - GraphOptions []string - Labels []string - LogConfig container.LogConfig - Mtu int - Pidfile string - RemappedRoot string - Root string - TrustKeyPath string + AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins + AutoRestart bool `json:"-"` + Bridge bridgeConfig `json:"-"` // Bridge holds bridge network specific configuration. + Context map[string][]string `json:"-"` + DisableBridge bool `json:"-"` + DNS []string `json:"dns,omitempty"` + DNSOptions []string `json:"dns-opts,omitempty"` + DNSSearch []string `json:"dns-search,omitempty"` + ExecOptions []string `json:"exec-opts,omitempty"` + ExecRoot string `json:"exec-root,omitempty"` + GraphDriver string `json:"storage-driver,omitempty"` + GraphOptions []string `json:"storage-opts,omitempty"` + Labels []string `json:"labels,omitempty"` + LogConfig LogConfig `json:"log-config,omitempty"` + Mtu int `json:"mtu,omitempty"` + Pidfile string `json:"pidfile,omitempty"` + Root string `json:"graph,omitempty"` + TrustKeyPath string `json:"-"` // ClusterStore is the storage backend used for the cluster information. It is used by both // multihost networking (to store networks and endpoints information) and by the node discovery // mechanism. - ClusterStore string + ClusterStore string `json:"cluster-store,omitempty"` // ClusterOpts is used to pass options to the discovery package for tuning libkv settings, such // as TLS configuration settings. - ClusterOpts map[string]string + ClusterOpts map[string]string `json:"cluster-store-opts,omitempty"` // ClusterAdvertise is the network endpoint that the Engine advertises for the purpose of node // discovery. This should be a 'host:port' combination on which that daemon instance is // reachable by other hosts. - ClusterAdvertise string + ClusterAdvertise string `json:"cluster-advertise,omitempty"` + + Debug bool `json:"debug,omitempty"` + Hosts []string `json:"hosts,omitempty"` + LogLevel string `json:"log-level,omitempty"` + TLS bool `json:"tls,omitempty"` + TLSVerify bool `json:"tls-verify,omitempty"` + TLSOptions CommonTLSOptions `json:"tls-opts,omitempty"` + + reloadLock sync.Mutex } // InstallCommonFlags adds command-line options to the top-level flag parser for @@ -54,9 +91,9 @@ type CommonConfig struct { // Subsequent calls to `flag.Parse` will populate config with values parsed // from the command-line. func (config *Config) InstallCommonFlags(cmd *flag.FlagSet, usageFn func(string) string) { - cmd.Var(opts.NewListOptsRef(&config.GraphOptions, nil), []string{"-storage-opt"}, usageFn("Set storage driver options")) - cmd.Var(opts.NewListOptsRef(&config.AuthorizationPlugins, nil), []string{"-authorization-plugin"}, usageFn("List authorization plugins in order from first evaluator to last")) - cmd.Var(opts.NewListOptsRef(&config.ExecOptions, nil), []string{"-exec-opt"}, usageFn("Set exec driver options")) + cmd.Var(opts.NewNamedListOptsRef("storage-opts", &config.GraphOptions, nil), []string{"-storage-opt"}, usageFn("Set storage driver options")) + cmd.Var(opts.NewNamedListOptsRef("authorization-plugins", &config.AuthorizationPlugins, nil), []string{"-authorization-plugin"}, usageFn("List authorization plugins in order from first evaluator to last")) + cmd.Var(opts.NewNamedListOptsRef("exec-opts", &config.ExecOptions, nil), []string{"-exec-opt"}, usageFn("Set exec driver options")) cmd.StringVar(&config.Pidfile, []string{"p", "-pidfile"}, defaultPidFile, usageFn("Path to use for daemon PID file")) cmd.StringVar(&config.Root, []string{"g", "-graph"}, defaultGraph, usageFn("Root of the Docker runtime")) cmd.StringVar(&config.ExecRoot, []string{"-exec-root"}, "/var/run/docker", usageFn("Root of the Docker execdriver")) @@ -65,12 +102,131 @@ func (config *Config) InstallCommonFlags(cmd *flag.FlagSet, usageFn func(string) cmd.IntVar(&config.Mtu, []string{"#mtu", "-mtu"}, 0, usageFn("Set the containers network MTU")) // FIXME: why the inconsistency between "hosts" and "sockets"? cmd.Var(opts.NewListOptsRef(&config.DNS, opts.ValidateIPAddress), []string{"#dns", "-dns"}, usageFn("DNS server to use")) - cmd.Var(opts.NewListOptsRef(&config.DNSOptions, nil), []string{"-dns-opt"}, usageFn("DNS options to use")) + cmd.Var(opts.NewNamedListOptsRef("dns-opts", &config.DNSOptions, nil), []string{"-dns-opt"}, usageFn("DNS options to use")) cmd.Var(opts.NewListOptsRef(&config.DNSSearch, opts.ValidateDNSSearch), []string{"-dns-search"}, usageFn("DNS search domains to use")) - cmd.Var(opts.NewListOptsRef(&config.Labels, opts.ValidateLabel), []string{"-label"}, usageFn("Set key=value labels to the daemon")) + cmd.Var(opts.NewNamedListOptsRef("labels", &config.Labels, opts.ValidateLabel), []string{"-label"}, usageFn("Set key=value labels to the daemon")) cmd.StringVar(&config.LogConfig.Type, []string{"-log-driver"}, "json-file", usageFn("Default driver for container logs")) - cmd.Var(opts.NewMapOpts(config.LogConfig.Config, nil), []string{"-log-opt"}, usageFn("Set log driver options")) + cmd.Var(opts.NewNamedMapOpts("log-opts", config.LogConfig.Config, nil), []string{"-log-opt"}, usageFn("Set log driver options")) cmd.StringVar(&config.ClusterAdvertise, []string{"-cluster-advertise"}, "", usageFn("Address or interface name to advertise")) cmd.StringVar(&config.ClusterStore, []string{"-cluster-store"}, "", usageFn("Set the cluster store")) - cmd.Var(opts.NewMapOpts(config.ClusterOpts, nil), []string{"-cluster-store-opt"}, usageFn("Set cluster store options")) + cmd.Var(opts.NewNamedMapOpts("cluster-store-opts", config.ClusterOpts, nil), []string{"-cluster-store-opt"}, usageFn("Set cluster store options")) +} + +func parseClusterAdvertiseSettings(clusterStore, clusterAdvertise string) (string, error) { + if clusterAdvertise == "" { + return "", errDiscoveryDisabled + } + if clusterStore == "" { + return "", fmt.Errorf("invalid cluster configuration. --cluster-advertise must be accompanied by --cluster-store configuration") + } + + advertise, err := discovery.ParseAdvertise(clusterAdvertise) + if err != nil { + return "", fmt.Errorf("discovery advertise parsing failed (%v)", err) + } + return advertise, nil +} + +// ReloadConfiguration reads the configuration in the host and reloads the daemon and server. +func ReloadConfiguration(configFile string, flags *flag.FlagSet, reload func(*Config)) { + logrus.Infof("Got signal to reload configuration, reloading from: %s", configFile) + newConfig, err := getConflictFreeConfiguration(configFile, flags) + if err != nil { + logrus.Error(err) + } else { + reload(newConfig) + } +} + +// MergeDaemonConfigurations reads a configuration file, +// loads the file configuration in an isolated structure, +// and merges the configuration provided from flags on top +// if there are no conflicts. +func MergeDaemonConfigurations(flagsConfig *Config, flags *flag.FlagSet, configFile string) (*Config, error) { + fileConfig, err := getConflictFreeConfiguration(configFile, flags) + if err != nil { + return nil, err + } + + // merge flags configuration on top of the file configuration + if err := mergo.Merge(fileConfig, flagsConfig); err != nil { + return nil, err + } + + return fileConfig, nil +} + +// getConflictFreeConfiguration loads the configuration from a JSON file. +// It compares that configuration with the one provided by the flags, +// and returns an error if there are conflicts. +func getConflictFreeConfiguration(configFile string, flags *flag.FlagSet) (*Config, error) { + b, err := ioutil.ReadFile(configFile) + if err != nil { + return nil, err + } + + var reader io.Reader + if flags != nil { + var jsonConfig map[string]interface{} + reader = bytes.NewReader(b) + if err := json.NewDecoder(reader).Decode(&jsonConfig); err != nil { + return nil, err + } + + if err := findConfigurationConflicts(jsonConfig, flags); err != nil { + return nil, err + } + } + + var config Config + reader = bytes.NewReader(b) + err = json.NewDecoder(reader).Decode(&config) + return &config, err +} + +// findConfigurationConflicts iterates over the provided flags searching for +// duplicated configurations. It returns an error with all the conflicts if +// it finds any. +func findConfigurationConflicts(config map[string]interface{}, flags *flag.FlagSet) error { + var conflicts []string + flatten := make(map[string]interface{}) + for k, v := range config { + if m, ok := v.(map[string]interface{}); ok { + for km, vm := range m { + flatten[km] = vm + } + } else { + flatten[k] = v + } + } + + printConflict := func(name string, flagValue, fileValue interface{}) string { + return fmt.Sprintf("%s: (from flag: %v, from file: %v)", name, flagValue, fileValue) + } + + collectConflicts := func(f *flag.Flag) { + // search option name in the json configuration payload if the value is a named option + if namedOption, ok := f.Value.(opts.NamedOption); ok { + if optsValue, ok := flatten[namedOption.Name()]; ok { + conflicts = append(conflicts, printConflict(namedOption.Name(), f.Value.String(), optsValue)) + } + } else { + // search flag name in the json configuration payload without trailing dashes + for _, name := range f.Names { + name = strings.TrimLeft(name, "-") + + if value, ok := flatten[name]; ok { + conflicts = append(conflicts, printConflict(name, f.Value.String(), value)) + break + } + } + } + } + + flags.Visit(collectConflicts) + + if len(conflicts) > 0 { + return fmt.Errorf("the following directives are specified both as a flag and in the configuration file: %s", strings.Join(conflicts, ", ")) + } + return nil } diff --git a/daemon/config_test.go b/daemon/config_test.go new file mode 100644 index 0000000000..69a199e162 --- /dev/null +++ b/daemon/config_test.go @@ -0,0 +1,177 @@ +package daemon + +import ( + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/mflag" +) + +func TestDaemonConfigurationMerge(t *testing.T) { + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{"debug": true}`)) + f.Close() + + c := &Config{ + CommonConfig: CommonConfig{ + AutoRestart: true, + LogConfig: LogConfig{ + Type: "syslog", + Config: map[string]string{"tag": "test"}, + }, + }, + } + + cc, err := MergeDaemonConfigurations(c, nil, configFile) + if err != nil { + t.Fatal(err) + } + if !cc.Debug { + t.Fatalf("expected %v, got %v\n", true, cc.Debug) + } + if !cc.AutoRestart { + t.Fatalf("expected %v, got %v\n", true, cc.AutoRestart) + } + if cc.LogConfig.Type != "syslog" { + t.Fatalf("expected syslog config, got %q\n", cc.LogConfig) + } +} + +func TestDaemonConfigurationNotFound(t *testing.T) { + _, err := MergeDaemonConfigurations(&Config{}, nil, "/tmp/foo-bar-baz-docker") + if err == nil || !os.IsNotExist(err) { + t.Fatalf("expected does not exist error, got %v", err) + } +} + +func TestDaemonBrokenConfiguration(t *testing.T) { + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{"Debug": tru`)) + f.Close() + + _, err = MergeDaemonConfigurations(&Config{}, nil, configFile) + if err == nil { + t.Fatalf("expected error, got %v", err) + } +} + +func TestParseClusterAdvertiseSettings(t *testing.T) { + _, err := parseClusterAdvertiseSettings("something", "") + if err != errDiscoveryDisabled { + t.Fatalf("expected discovery disabled error, got %v\n", err) + } + + _, err = parseClusterAdvertiseSettings("", "something") + if err == nil { + t.Fatalf("expected discovery store error, got %v\n", err) + } + + _, err = parseClusterAdvertiseSettings("etcd", "127.0.0.1:8080") + if err != nil { + t.Fatal(err) + } +} + +func TestFindConfigurationConflicts(t *testing.T) { + config := map[string]interface{}{"authorization-plugins": "foobar"} + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + + err := findConfigurationConflicts(config, flags) + if err != nil { + t.Fatal(err) + } + + flags.String([]string{"authorization-plugins"}, "", "") + if err := flags.Set("authorization-plugins", "asdf"); err != nil { + t.Fatal(err) + } + + err = findConfigurationConflicts(config, flags) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "authorization-plugins") { + t.Fatalf("expected authorization-plugins conflict, got %v", err) + } +} + +func TestFindConfigurationConflictsWithNamedOptions(t *testing.T) { + config := map[string]interface{}{"hosts": []string{"qwer"}} + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + + var hosts []string + flags.Var(opts.NewNamedListOptsRef("hosts", &hosts, opts.ValidateHost), []string{"H", "-host"}, "Daemon socket(s) to connect to") + if err := flags.Set("-host", "tcp://127.0.0.1:4444"); err != nil { + t.Fatal(err) + } + if err := flags.Set("H", "unix:///var/run/docker.sock"); err != nil { + t.Fatal(err) + } + + err := findConfigurationConflicts(config, flags) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "hosts") { + t.Fatalf("expected hosts conflict, got %v", err) + } +} + +func TestDaemonConfigurationMergeConflicts(t *testing.T) { + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{"debug": true}`)) + f.Close() + + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + flags.Bool([]string{"debug"}, false, "") + flags.Set("debug", "false") + + _, err = MergeDaemonConfigurations(&Config{}, flags, configFile) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "debug") { + t.Fatalf("expected debug conflict, got %v", err) + } +} + +func TestDaemonConfigurationMergeConflictsWithInnerStructs(t *testing.T) { + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{"tlscacert": "/etc/certificates/ca.pem"}`)) + f.Close() + + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + flags.String([]string{"tlscacert"}, "", "") + flags.Set("tlscacert", "~/.docker/ca.pem") + + _, err = MergeDaemonConfigurations(&Config{}, flags, configFile) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "tlscacert") { + t.Fatalf("expected tlscacert conflict, got %v", err) + } +} diff --git a/daemon/config_unix.go b/daemon/config_unix.go index a25df90704..60fb3a9b54 100644 --- a/daemon/config_unix.go +++ b/daemon/config_unix.go @@ -18,18 +18,20 @@ var ( ) // Config defines the configuration of a docker daemon. +// It includes json tags to deserialize configuration from a file +// using the same names that the flags in the command line uses. type Config struct { CommonConfig // Fields below here are platform specific. - CorsHeaders string - EnableCors bool - EnableSelinuxSupport bool - RemappedRoot string - SocketGroup string - CgroupParent string - Ulimits map[string]*units.Ulimit + CorsHeaders string `json:"api-cors-headers,omitempty"` + EnableCors bool `json:"api-enable-cors,omitempty"` + EnableSelinuxSupport bool `json:"selinux-enabled,omitempty"` + RemappedRoot string `json:"userns-remap,omitempty"` + SocketGroup string `json:"group,omitempty"` + CgroupParent string `json:"cgroup-parent,omitempty"` + Ulimits map[string]*units.Ulimit `json:"default-ulimits,omitempty"` } // bridgeConfig stores all the bridge driver specific diff --git a/daemon/daemon.go b/daemon/daemon.go index bbecc677f4..9e0e77e350 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -46,7 +46,6 @@ import ( "github.com/docker/docker/layer" "github.com/docker/docker/migrate/v1" "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/discovery" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/graphdb" "github.com/docker/docker/pkg/idtools" @@ -155,7 +154,7 @@ type Daemon struct { EventsService *events.Events netController libnetwork.NetworkController volumes *store.VolumeStore - discoveryWatcher discovery.Watcher + discoveryWatcher discoveryReloader root string seccompEnabled bool shutdown bool @@ -292,7 +291,7 @@ func (daemon *Daemon) Register(container *container.Container) error { func (daemon *Daemon) restore() error { var ( - debug = os.Getenv("DEBUG") != "" + debug = utils.IsDebugEnabled() currentDriver = daemon.GraphDriverName() containers = make(map[string]*container.Container) ) @@ -772,19 +771,8 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo // Discovery is only enabled when the daemon is launched with an address to advertise. When // initialized, the daemon is registered and we can store the discovery backend as its read-only - // DiscoveryWatcher version. - if config.ClusterStore != "" && config.ClusterAdvertise != "" { - advertise, err := discovery.ParseAdvertise(config.ClusterStore, config.ClusterAdvertise) - if err != nil { - return nil, fmt.Errorf("discovery advertise parsing failed (%v)", err) - } - config.ClusterAdvertise = advertise - d.discoveryWatcher, err = initDiscovery(config.ClusterStore, config.ClusterAdvertise, config.ClusterOpts) - if err != nil { - return nil, fmt.Errorf("discovery initialization failed (%v)", err) - } - } else if config.ClusterAdvertise != "" { - return nil, fmt.Errorf("invalid cluster configuration. --cluster-advertise must be accompanied by --cluster-store configuration") + if err := d.initDiscovery(config); err != nil { + return nil, err } d.netController, err = d.initNetworkController(config) @@ -815,7 +803,10 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo d.configStore = config d.execDriver = ed d.statsCollector = d.newStatsCollector(1 * time.Second) - d.defaultLogConfig = config.LogConfig + d.defaultLogConfig = containertypes.LogConfig{ + Type: config.LogConfig.Type, + Config: config.LogConfig.Config, + } d.RegistryService = registryService d.EventsService = eventsService d.volumes = volStore @@ -1521,6 +1512,76 @@ func (daemon *Daemon) newBaseContainer(id string) *container.Container { return container.NewBaseContainer(id, daemon.containerRoot(id)) } +// initDiscovery initializes the discovery watcher for this daemon. +func (daemon *Daemon) initDiscovery(config *Config) error { + advertise, err := parseClusterAdvertiseSettings(config.ClusterStore, config.ClusterAdvertise) + if err != nil { + if err == errDiscoveryDisabled { + return nil + } + return err + } + + config.ClusterAdvertise = advertise + discoveryWatcher, err := initDiscovery(config.ClusterStore, config.ClusterAdvertise, config.ClusterOpts) + if err != nil { + return fmt.Errorf("discovery initialization failed (%v)", err) + } + + daemon.discoveryWatcher = discoveryWatcher + return nil +} + +// Reload reads configuration changes and modifies the +// daemon according to those changes. +// This are the settings that Reload changes: +// - Daemon labels. +// - Cluster discovery (reconfigure and restart). +func (daemon *Daemon) Reload(config *Config) error { + daemon.configStore.reloadLock.Lock() + defer daemon.configStore.reloadLock.Unlock() + + daemon.configStore.Labels = config.Labels + return daemon.reloadClusterDiscovery(config) +} + +func (daemon *Daemon) reloadClusterDiscovery(config *Config) error { + newAdvertise, err := parseClusterAdvertiseSettings(config.ClusterStore, config.ClusterAdvertise) + if err != nil && err != errDiscoveryDisabled { + return err + } + + // check discovery modifications + if !modifiedDiscoverySettings(daemon.configStore, newAdvertise, config.ClusterStore, config.ClusterOpts) { + return nil + } + + // enable discovery for the first time if it was not previously enabled + if daemon.discoveryWatcher == nil { + discoveryWatcher, err := initDiscovery(config.ClusterStore, newAdvertise, config.ClusterOpts) + if err != nil { + return fmt.Errorf("discovery initialization failed (%v)", err) + } + daemon.discoveryWatcher = discoveryWatcher + } else { + if err == errDiscoveryDisabled { + // disable discovery if it was previously enabled and it's disabled now + daemon.discoveryWatcher.Stop() + } else { + // reload discovery + if err = daemon.discoveryWatcher.Reload(config.ClusterStore, newAdvertise, config.ClusterOpts); err != nil { + return err + } + } + } + + daemon.configStore.ClusterStore = config.ClusterStore + daemon.configStore.ClusterOpts = config.ClusterOpts + daemon.configStore.ClusterAdvertise = newAdvertise + + return nil +} + func convertLnNetworkStats(name string, stats *lntypes.InterfaceStatistics) *libcontainer.NetworkInterface { n := &libcontainer.NetworkInterface{Name: name} n.RxBytes = stats.RxBytes diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index e6550a44da..26e9c2f743 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -4,9 +4,13 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "testing" + "time" "github.com/docker/docker/container" + "github.com/docker/docker/pkg/discovery" + _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/pkg/registrar" "github.com/docker/docker/pkg/truncindex" "github.com/docker/docker/volume" @@ -371,3 +375,118 @@ func TestMerge(t *testing.T) { } } } + +func TestDaemonReloadLabels(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &Config{ + CommonConfig: CommonConfig{ + Labels: []string{"foo:bar"}, + }, + } + + newConfig := &Config{ + CommonConfig: CommonConfig{ + Labels: []string{"foo:baz"}, + }, + } + + daemon.Reload(newConfig) + label := daemon.configStore.Labels[0] + if label != "foo:baz" { + t.Fatalf("Expected daemon label `foo:baz`, got %s", label) + } +} + +func TestDaemonDiscoveryReload(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &Config{ + CommonConfig: CommonConfig{ + ClusterStore: "memory://127.0.0.1", + ClusterAdvertise: "127.0.0.1:3333", + }, + } + + if err := daemon.initDiscovery(daemon.configStore); err != nil { + t.Fatal(err) + } + + expected := discovery.Entries{ + &discovery.Entry{Host: "127.0.0.1", Port: "3333"}, + } + + stopCh := make(chan struct{}) + defer close(stopCh) + ch, errCh := daemon.discoveryWatcher.Watch(stopCh) + + select { + case <-time.After(1 * time.Second): + t.Fatal("failed to get discovery advertisements in time") + case e := <-ch: + if !reflect.DeepEqual(e, expected) { + t.Fatalf("expected %v, got %v\n", expected, e) + } + case e := <-errCh: + t.Fatal(e) + } + + newConfig := &Config{ + CommonConfig: CommonConfig{ + ClusterStore: "memory://127.0.0.1:2222", + ClusterAdvertise: "127.0.0.1:5555", + }, + } + + expected = discovery.Entries{ + &discovery.Entry{Host: "127.0.0.1", Port: "5555"}, + } + + if err := daemon.Reload(newConfig); err != nil { + t.Fatal(err) + } + ch, errCh = daemon.discoveryWatcher.Watch(stopCh) + + select { + case <-time.After(1 * time.Second): + t.Fatal("failed to get discovery advertisements in time") + case e := <-ch: + if !reflect.DeepEqual(e, expected) { + t.Fatalf("expected %v, got %v\n", expected, e) + } + case e := <-errCh: + t.Fatal(e) + } +} + +func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &Config{} + + newConfig := &Config{ + CommonConfig: CommonConfig{ + ClusterStore: "memory://127.0.0.1:2222", + ClusterAdvertise: "127.0.0.1:5555", + }, + } + + expected := discovery.Entries{ + &discovery.Entry{Host: "127.0.0.1", Port: "5555"}, + } + + if err := daemon.Reload(newConfig); err != nil { + t.Fatal(err) + } + stopCh := make(chan struct{}) + defer close(stopCh) + ch, errCh := daemon.discoveryWatcher.Watch(stopCh) + + select { + case <-time.After(1 * time.Second): + t.Fatal("failed to get discovery advertisements in time") + case e := <-ch: + if !reflect.DeepEqual(e, expected) { + t.Fatalf("expected %v, got %v\n", expected, e) + } + case e := <-errCh: + t.Fatal(e) + } +} diff --git a/daemon/discovery.go b/daemon/discovery.go index ef9307de0a..6c4bcc43e9 100644 --- a/daemon/discovery.go +++ b/daemon/discovery.go @@ -1,7 +1,9 @@ package daemon import ( + "errors" "fmt" + "reflect" "strconv" "time" @@ -19,6 +21,24 @@ const ( defaultDiscoveryTTLFactor = 3 ) +var errDiscoveryDisabled = errors.New("discovery is disabled") + +type discoveryReloader interface { + discovery.Watcher + Stop() + Reload(backend, address string, clusterOpts map[string]string) error +} + +type daemonDiscoveryReloader struct { + backend discovery.Backend + ticker *time.Ticker + term chan bool +} + +func (d *daemonDiscoveryReloader) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { + return d.backend.Watch(stopCh) +} + func discoveryOpts(clusterOpts map[string]string) (time.Duration, time.Duration, error) { var ( heartbeat = defaultDiscoveryHeartbeat @@ -57,36 +77,94 @@ func discoveryOpts(clusterOpts map[string]string) (time.Duration, time.Duration, // initDiscovery initialized the nodes discovery subsystem by connecting to the specified backend // and start a registration loop to advertise the current node under the specified address. -func initDiscovery(backend, address string, clusterOpts map[string]string) (discovery.Backend, error) { - - heartbeat, ttl, err := discoveryOpts(clusterOpts) +func initDiscovery(backendAddress, advertiseAddress string, clusterOpts map[string]string) (discoveryReloader, error) { + heartbeat, backend, err := parseDiscoveryOptions(backendAddress, clusterOpts) if err != nil { return nil, err } - discoveryBackend, err := discovery.New(backend, heartbeat, ttl, clusterOpts) - if err != nil { - return nil, err + reloader := &daemonDiscoveryReloader{ + backend: backend, + ticker: time.NewTicker(heartbeat), + term: make(chan bool), } - // We call Register() on the discovery backend in a loop for the whole lifetime of the daemon, // but we never actually Watch() for nodes appearing and disappearing for the moment. - go registrationLoop(discoveryBackend, address, heartbeat) - return discoveryBackend, nil + reloader.advertise(advertiseAddress) + return reloader, nil } -func registerAddr(backend discovery.Backend, addr string) { - if err := backend.Register(addr); err != nil { +func (d *daemonDiscoveryReloader) advertise(address string) { + d.registerAddr(address) + go d.advertiseHeartbeat(address) +} + +func (d *daemonDiscoveryReloader) registerAddr(addr string) { + if err := d.backend.Register(addr); err != nil { log.Warnf("Registering as %q in discovery failed: %v", addr, err) } } -// registrationLoop registers the current node against the discovery backend using the specified +// advertiseHeartbeat registers the current node against the discovery backend using the specified // address. The function never returns, as registration against the backend comes with a TTL and // requires regular heartbeats. -func registrationLoop(discoveryBackend discovery.Backend, address string, heartbeat time.Duration) { - registerAddr(discoveryBackend, address) - for range time.Tick(heartbeat) { - registerAddr(discoveryBackend, address) +func (d *daemonDiscoveryReloader) advertiseHeartbeat(address string) { + for { + select { + case <-d.ticker.C: + d.registerAddr(address) + case <-d.term: + return + } } } + +// Reload makes the watcher to stop advertising and reconfigures it to advertise in a new address. +func (d *daemonDiscoveryReloader) Reload(backendAddress, advertiseAddress string, clusterOpts map[string]string) error { + d.Stop() + + heartbeat, backend, err := parseDiscoveryOptions(backendAddress, clusterOpts) + if err != nil { + return err + } + + d.backend = backend + d.ticker = time.NewTicker(heartbeat) + + d.advertise(advertiseAddress) + return nil +} + +// Stop terminates the discovery advertising. +func (d *daemonDiscoveryReloader) Stop() { + d.ticker.Stop() + d.term <- true +} + +func parseDiscoveryOptions(backendAddress string, clusterOpts map[string]string) (time.Duration, discovery.Backend, error) { + heartbeat, ttl, err := discoveryOpts(clusterOpts) + if err != nil { + return 0, nil, err + } + + backend, err := discovery.New(backendAddress, heartbeat, ttl, clusterOpts) + if err != nil { + return 0, nil, err + } + return heartbeat, backend, nil +} + +// modifiedDiscoverySettings returns whether the discovery configuration has been modified or not. +func modifiedDiscoverySettings(config *Config, backendType, advertise string, clusterOpts map[string]string) bool { + if config.ClusterStore != backendType || config.ClusterAdvertise != advertise { + return true + } + + if (config.ClusterOpts == nil && clusterOpts == nil) || + (config.ClusterOpts == nil && len(clusterOpts) == 0) || + (len(config.ClusterOpts) == 0 && clusterOpts == nil) { + return false + } + + return !reflect.DeepEqual(config.ClusterOpts, clusterOpts) +} diff --git a/daemon/discovery_test.go b/daemon/discovery_test.go index e65aecb8c6..c761a697ef 100644 --- a/daemon/discovery_test.go +++ b/daemon/discovery_test.go @@ -89,3 +89,64 @@ func TestDiscoveryOpts(t *testing.T) { t.Fatalf("TTL - Expected : %v, Actual : %v", expected, ttl) } } + +func TestModifiedDiscoverySettings(t *testing.T) { + cases := []struct { + current *Config + modified *Config + expected bool + }{ + { + current: discoveryConfig("foo", "bar", map[string]string{}), + modified: discoveryConfig("foo", "bar", map[string]string{}), + expected: false, + }, + { + current: discoveryConfig("foo", "bar", map[string]string{"foo": "bar"}), + modified: discoveryConfig("foo", "bar", map[string]string{"foo": "bar"}), + expected: false, + }, + { + current: discoveryConfig("foo", "bar", map[string]string{}), + modified: discoveryConfig("foo", "bar", nil), + expected: false, + }, + { + current: discoveryConfig("foo", "bar", nil), + modified: discoveryConfig("foo", "bar", map[string]string{}), + expected: false, + }, + { + current: discoveryConfig("foo", "bar", nil), + modified: discoveryConfig("baz", "bar", nil), + expected: true, + }, + { + current: discoveryConfig("foo", "bar", nil), + modified: discoveryConfig("foo", "baz", nil), + expected: true, + }, + { + current: discoveryConfig("foo", "bar", nil), + modified: discoveryConfig("foo", "bar", map[string]string{"foo": "bar"}), + expected: true, + }, + } + + for _, c := range cases { + got := modifiedDiscoverySettings(c.current, c.modified.ClusterStore, c.modified.ClusterAdvertise, c.modified.ClusterOpts) + if c.expected != got { + t.Fatalf("expected %v, got %v: current config %q, new config %q", c.expected, got, c.current, c.modified) + } + } +} + +func discoveryConfig(backendAddr, advertiseAddr string, opts map[string]string) *Config { + return &Config{ + CommonConfig: CommonConfig{ + ClusterStore: backendAddr, + ClusterAdvertise: advertiseAddr, + ClusterOpts: opts, + }, + } +} diff --git a/daemon/info.go b/daemon/info.go index f5f6f96c89..804d6e4709 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -79,7 +79,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { IPv4Forwarding: !sysInfo.IPv4ForwardingDisabled, BridgeNfIptables: !sysInfo.BridgeNfCallIptablesDisabled, BridgeNfIP6tables: !sysInfo.BridgeNfCallIP6tablesDisabled, - Debug: os.Getenv("DEBUG") != "", + Debug: utils.IsDebugEnabled(), NFd: fileutils.GetTotalUsedFds(), NGoroutines: runtime.NumGoroutine(), SystemTime: time.Now().Format(time.RFC3339Nano), diff --git a/docker/common.go b/docker/common.go index 250924694f..893de7109e 100644 --- a/docker/common.go +++ b/docker/common.go @@ -21,7 +21,6 @@ const ( ) var ( - daemonFlags *flag.FlagSet commonFlags = &cli.CommonFlags{FlagSet: new(flag.FlagSet)} dockerCertPath = os.Getenv("DOCKER_CERT_PATH") @@ -50,7 +49,7 @@ func init() { cmd.StringVar(&tlsOptions.CertFile, []string{"-tlscert"}, filepath.Join(dockerCertPath, defaultCertFile), "Path to TLS certificate file") cmd.StringVar(&tlsOptions.KeyFile, []string{"-tlskey"}, filepath.Join(dockerCertPath, defaultKeyFile), "Path to TLS key file") - cmd.Var(opts.NewListOptsRef(&commonFlags.Hosts, opts.ValidateHost), []string{"H", "-host"}, "Daemon socket(s) to connect to") + cmd.Var(opts.NewNamedListOptsRef("hosts", &commonFlags.Hosts, opts.ValidateHost), []string{"H", "-host"}, "Daemon socket(s) to connect to") } func postParseCommon() { @@ -67,11 +66,6 @@ func postParseCommon() { logrus.SetLevel(logrus.InfoLevel) } - if commonFlags.Debug { - os.Setenv("DEBUG", "1") - logrus.SetLevel(logrus.DebugLevel) - } - // Regardless of whether the user sets it to true or false, if they // specify --tlsverify at all then we need to turn on tls // TLSVerify can be true even if not set due to DOCKER_TLS_VERIFY env var, so we need to check that here as well diff --git a/docker/daemon.go b/docker/daemon.go index e65cb77713..a8422122f7 100644 --- a/docker/daemon.go +++ b/docker/daemon.go @@ -30,23 +30,34 @@ import ( "github.com/docker/go-connections/tlsconfig" ) -const daemonUsage = " docker daemon [ --help | ... ]\n" +const ( + daemonUsage = " docker daemon [ --help | ... ]\n" + daemonConfigFileFlag = "-config-file" +) var ( daemonCli cli.Handler = NewDaemonCli() ) +// DaemonCli represents the daemon CLI. +type DaemonCli struct { + *daemon.Config + registryOptions *registry.Options + flags *flag.FlagSet +} + func presentInHelp(usage string) string { return usage } func absentFromHelp(string) string { return "" } // NewDaemonCli returns a pre-configured daemon CLI func NewDaemonCli() *DaemonCli { - daemonFlags = cli.Subcmd("daemon", nil, "Enable daemon mode", true) + daemonFlags := cli.Subcmd("daemon", nil, "Enable daemon mode", true) // TODO(tiborvass): remove InstallFlags? daemonConfig := new(daemon.Config) daemonConfig.LogConfig.Config = make(map[string]string) daemonConfig.ClusterOpts = make(map[string]string) + daemonConfig.InstallFlags(daemonFlags, presentInHelp) daemonConfig.InstallFlags(flag.CommandLine, absentFromHelp) registryOptions := new(registry.Options) @@ -57,6 +68,7 @@ func NewDaemonCli() *DaemonCli { return &DaemonCli{ Config: daemonConfig, registryOptions: registryOptions, + flags: daemonFlags, } } @@ -101,12 +113,6 @@ func migrateKey() (err error) { return nil } -// DaemonCli represents the daemon CLI. -type DaemonCli struct { - *daemon.Config - registryOptions *registry.Options -} - func getGlobalFlag() (globalFlag *flag.Flag) { defer func() { if x := recover(); x != nil { @@ -136,15 +142,27 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { os.Exit(1) } else { // allow new form `docker daemon -D` - flag.Merge(daemonFlags, commonFlags.FlagSet) + flag.Merge(cli.flags, commonFlags.FlagSet) } - daemonFlags.ParseFlags(args, true) + configFile := cli.flags.String([]string{daemonConfigFileFlag}, defaultDaemonConfigFile, "Daemon configuration file") + + cli.flags.ParseFlags(args, true) commonFlags.PostParse() if commonFlags.TrustKey == "" { commonFlags.TrustKey = filepath.Join(getDaemonConfDir(), defaultTrustKeyFile) } + cliConfig, err := loadDaemonCliConfig(cli.Config, cli.flags, commonFlags, *configFile) + if err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + cli.Config = cliConfig + + if cli.Config.Debug { + utils.EnableDebug() + } if utils.ExperimentalBuild() { logrus.Warn("Running experimental build") @@ -184,12 +202,18 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { serverConfig = setPlatformServerConfig(serverConfig, cli.Config) defaultHost := opts.DefaultHost - if commonFlags.TLSOptions != nil { - if !commonFlags.TLSOptions.InsecureSkipVerify { - // server requires and verifies client's certificate - commonFlags.TLSOptions.ClientAuth = tls.RequireAndVerifyClientCert + if cli.Config.TLS { + tlsOptions := tlsconfig.Options{ + CAFile: cli.Config.TLSOptions.CAFile, + CertFile: cli.Config.TLSOptions.CertFile, + KeyFile: cli.Config.TLSOptions.KeyFile, } - tlsConfig, err := tlsconfig.Server(*commonFlags.TLSOptions) + + if cli.Config.TLSVerify { + // server requires and verifies client's certificate + tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert + } + tlsConfig, err := tlsconfig.Server(tlsOptions) if err != nil { logrus.Fatal(err) } @@ -197,22 +221,23 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { defaultHost = opts.DefaultTLSHost } - if len(commonFlags.Hosts) == 0 { - commonFlags.Hosts = make([]string, 1) + if len(cli.Config.Hosts) == 0 { + cli.Config.Hosts = make([]string, 1) } - for i := 0; i < len(commonFlags.Hosts); i++ { + for i := 0; i < len(cli.Config.Hosts); i++ { var err error - if commonFlags.Hosts[i], err = opts.ParseHost(defaultHost, commonFlags.Hosts[i]); err != nil { - logrus.Fatalf("error parsing -H %s : %v", commonFlags.Hosts[i], err) + if cli.Config.Hosts[i], err = opts.ParseHost(defaultHost, cli.Config.Hosts[i]); err != nil { + logrus.Fatalf("error parsing -H %s : %v", cli.Config.Hosts[i], err) } - } - for _, protoAddr := range commonFlags.Hosts { + + protoAddr := cli.Config.Hosts[i] protoAddrParts := strings.SplitN(protoAddr, "://", 2) if len(protoAddrParts) != 2 { logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr) } serverConfig.Addrs = append(serverConfig.Addrs, apiserver.Addr{Proto: protoAddrParts[0], Addr: protoAddrParts[1]}) } + api, err := apiserver.New(serverConfig) if err != nil { logrus.Fatal(err) @@ -245,18 +270,21 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { api.InitRouters(d) + reload := func(config *daemon.Config) { + if err := d.Reload(config); err != nil { + logrus.Errorf("Error reconfiguring the daemon: %v", err) + return + } + api.Reload(config) + } + + setupConfigReloadTrap(*configFile, cli.flags, reload) + // The serve API routine never exits unless an error occurs // We need to start it as a goroutine and wait on it so // daemon doesn't exit serveAPIWait := make(chan error) - go func() { - if err := api.ServeAPI(); err != nil { - logrus.Errorf("ServeAPI error: %v", err) - serveAPIWait <- err - return - } - serveAPIWait <- nil - }() + go api.Wait(serveAPIWait) signal.Trap(func() { api.Close() @@ -303,3 +331,34 @@ func shutdownDaemon(d *daemon.Daemon, timeout time.Duration) { logrus.Error("Force shutdown daemon") } } + +func loadDaemonCliConfig(config *daemon.Config, daemonFlags *flag.FlagSet, commonConfig *cli.CommonFlags, configFile string) (*daemon.Config, error) { + config.Debug = commonConfig.Debug + config.Hosts = commonConfig.Hosts + config.LogLevel = commonConfig.LogLevel + config.TLS = commonConfig.TLS + config.TLSVerify = commonConfig.TLSVerify + config.TLSOptions = daemon.CommonTLSOptions{} + + if commonConfig.TLSOptions != nil { + config.TLSOptions.CAFile = commonConfig.TLSOptions.CAFile + config.TLSOptions.CertFile = commonConfig.TLSOptions.CertFile + config.TLSOptions.KeyFile = commonConfig.TLSOptions.KeyFile + } + + if configFile != "" { + c, err := daemon.MergeDaemonConfigurations(config, daemonFlags, configFile) + if err != nil { + if daemonFlags.IsSet(daemonConfigFileFlag) || !os.IsNotExist(err) { + return nil, fmt.Errorf("unable to configure the Docker daemon with file %s: %v\n", configFile, err) + } + } + // the merged configuration can be nil if the config file didn't exist. + // leave the current configuration as it is if when that happens. + if c != nil { + config = c + } + } + + return config, nil +} diff --git a/docker/daemon_test.go b/docker/daemon_test.go new file mode 100644 index 0000000000..bc519e7467 --- /dev/null +++ b/docker/daemon_test.go @@ -0,0 +1,91 @@ +// +build daemon + +package main + +import ( + "io/ioutil" + "strings" + "testing" + + "github.com/docker/docker/cli" + "github.com/docker/docker/daemon" + "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/mflag" + "github.com/docker/go-connections/tlsconfig" +) + +func TestLoadDaemonCliConfigWithoutOverriding(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{ + Debug: true, + } + + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + loadedConfig, err := loadDaemonCliConfig(c, flags, common, "/tmp/fooobarbaz") + if err != nil { + t.Fatal(err) + } + if loadedConfig == nil { + t.Fatalf("expected configuration %v, got nil", c) + } + if !loadedConfig.Debug { + t.Fatalf("expected debug to be copied from the common flags, got false") + } +} + +func TestLoadDaemonCliConfigWithTLS(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{ + TLS: true, + TLSOptions: &tlsconfig.Options{ + CAFile: "/tmp/ca.pem", + }, + } + + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + loadedConfig, err := loadDaemonCliConfig(c, flags, common, "/tmp/fooobarbaz") + if err != nil { + t.Fatal(err) + } + if loadedConfig == nil { + t.Fatalf("expected configuration %v, got nil", c) + } + if loadedConfig.TLSOptions.CAFile != "/tmp/ca.pem" { + t.Fatalf("expected /tmp/ca.pem, got %s: %q", loadedConfig.TLSOptions.CAFile, loadedConfig) + } +} + +func TestLoadDaemonCliConfigWithConflicts(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{} + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{"labels": ["l3=foo"]}`)) + f.Close() + + var labels []string + + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + flags.String([]string{daemonConfigFileFlag}, "", "") + flags.Var(opts.NewNamedListOptsRef("labels", &labels, opts.ValidateLabel), []string{"-label"}, "") + + flags.Set(daemonConfigFileFlag, configFile) + if err := flags.Set("-label", "l1=bar"); err != nil { + t.Fatal(err) + } + if err := flags.Set("-label", "l2=baz"); err != nil { + t.Fatal(err) + } + + _, err = loadDaemonCliConfig(c, flags, common, configFile) + if err == nil { + t.Fatalf("expected configuration error, got nil") + } + if !strings.Contains(err.Error(), "labels") { + t.Fatalf("expected labels conflict, got %v", err) + } +} diff --git a/docker/daemon_unix.go b/docker/daemon_unix.go index 7754130d34..eba0beef6d 100644 --- a/docker/daemon_unix.go +++ b/docker/daemon_unix.go @@ -5,15 +5,19 @@ package main import ( "fmt" "os" + "os/signal" "syscall" apiserver "github.com/docker/docker/api/server" "github.com/docker/docker/daemon" + "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/system" _ "github.com/docker/docker/daemon/execdriver/native" ) +const defaultDaemonConfigFile = "/etc/docker/daemon.json" + func setPlatformServerConfig(serverConfig *apiserver.Config, daemonCfg *daemon.Config) *apiserver.Config { serverConfig.SocketGroup = daemonCfg.SocketGroup serverConfig.EnableCors = daemonCfg.EnableCors @@ -48,3 +52,14 @@ func setDefaultUmask() error { func getDaemonConfDir() string { return "/etc/docker" } + +// setupConfigReloadTrap configures the USR2 signal to reload the configuration. +func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(*daemon.Config)) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + go func() { + for range c { + daemon.ReloadConfiguration(configFile, flags, reload) + } + }() +} diff --git a/docker/daemon_windows.go b/docker/daemon_windows.go index a9301529c3..307bbcc39b 100644 --- a/docker/daemon_windows.go +++ b/docker/daemon_windows.go @@ -3,12 +3,19 @@ package main import ( + "fmt" "os" + "syscall" + "github.com/Sirupsen/logrus" apiserver "github.com/docker/docker/api/server" "github.com/docker/docker/daemon" + "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/pkg/system" ) +var defaultDaemonConfigFile = os.Getenv("programdata") + string(os.PathSeparator) + "docker" + string(os.PathSeparator) + "config" + string(os.PathSeparator) + "daemon.json" + func setPlatformServerConfig(serverConfig *apiserver.Config, daemonCfg *daemon.Config) *apiserver.Config { return serverConfig } @@ -31,3 +38,20 @@ func getDaemonConfDir() string { // notifySystem sends a message to the host when the server is ready to be used func notifySystem() { } + +// setupConfigReloadTrap configures a Win32 event to reload the configuration. +func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(*daemon.Config)) { + go func() { + sa := syscall.SecurityAttributes{ + Length: 0, + } + ev := "Global\\docker-daemon-config-" + fmt.Sprint(os.Getpid()) + if h, _ := system.CreateEvent(&sa, false, false, ev); h != 0 { + logrus.Debugf("Config reload - waiting signal at %s", ev) + for { + syscall.WaitForSingleObject(h, syscall.INFINITE) + daemon.ReloadConfiguration(configFile, flags, reload) + } + } + }() +} diff --git a/docs/reference/commandline/daemon.md b/docs/reference/commandline/daemon.md index 11bc223dc8..843fe13647 100644 --- a/docs/reference/commandline/daemon.md +++ b/docs/reference/commandline/daemon.md @@ -27,6 +27,7 @@ weight = -1 --cluster-store="" URL of the distributed storage backend --cluster-advertise="" Address of the daemon instance on the cluster --cluster-store-opt=map[] Set cluster options + --config-file=/etc/docker/daemon.json Daemon configuration file --dns=[] DNS server to use --dns-opt=[] DNS options to use --dns-search=[] DNS search domains to use @@ -776,7 +777,7 @@ set like this: /usr/local/bin/docker daemon -D -g /var/lib/docker -H unix:// > /var/lib/docker-machine/docker.log 2>&1 -# Default cgroup parent +## Default cgroup parent The `--cgroup-parent` option allows you to set the default cgroup parent to use for containers. If this option is not set, it defaults to `/docker` for @@ -794,3 +795,79 @@ creates the cgroup in `/sys/fs/cgroup/memory/daemoncgroup/foobar` This setting can also be set per container, using the `--cgroup-parent` option on `docker create` and `docker run`, and takes precedence over the `--cgroup-parent` option on the daemon. + +## Daemon configuration file + +The `--config-file` option allows you to set any configuration option +for the daemon in a JSON format. This file uses the same flag names as keys, +except for flags that allow several entries, where it uses the plural +of the flag name, e.g., `labels` for the `label` flag. By default, +docker tries to load a configuration file from `/etc/docker/daemon.json` +on Linux and `%programdata%\docker\config\daemon.json` on Windows. + +The options set in the configuration file must not conflict with options set +via flags. The docker daemon fails to start if an option is duplicated between +the file and the flags, regardless their value. We do this to avoid +silently ignore changes introduced in configuration reloads. +For example, the daemon fails to start if you set daemon labels +in the configuration file and also set daemon labels via the `--label` flag. + +Options that are not present in the file are ignored when the daemon starts. +This is a full example of the allowed configuration options in the file: + +```json +{ + "authorization-plugins": [], + "dns": [], + "dns-opts": [], + "dns-search": [], + "exec-opts": [], + "exec-root": "", + "storage-driver": "", + "storage-opts": "", + "labels": [], + "log-config": { + "log-driver": "", + "log-opts": [] + }, + "mtu": 0, + "pidfile": "", + "graph": "", + "cluster-store": "", + "cluster-store-opts": [], + "cluster-advertise": "", + "debug": true, + "hosts": [], + "log-level": "", + "tls": true, + "tls-verify": true, + "tls-opts": { + "tlscacert": "", + "tlscert": "", + "tlskey": "" + }, + "api-cors-headers": "", + "selinux-enabled": false, + "userns-remap": "", + "group": "", + "cgroup-parent": "", + "default-ulimits": {} +} +``` + +### Configuration reloading + +Some options can be reconfigured when the daemon is running without requiring +to restart the process. We use the `SIGHUP` signal in Linux to reload, and a global event +in Windows with the key `Global\docker-daemon-config-$PID`. The options can +be modified in the configuration file but still will check for conflicts with +the provided flags. The daemon fails to reconfigure itself +if there are conflicts, but it won't stop execution. + +The list of currently supported options that can be reconfigured is this: + +- `debug`: it changes the daemon to debug mode when set to true. +- `label`: it replaces the daemon labels with a new set of labels. +- `cluster-store`: it reloads the discovery store with the new address. +- `cluster-store-opts`: it uses the new options to reload the discovery store. +- `cluster-advertise`: it modifies the address advertised after reloading. diff --git a/integration-cli/docker_cli_help_test.go b/integration-cli/docker_cli_help_test.go index 7d9a902cfb..c8ebfd3d18 100644 --- a/integration-cli/docker_cli_help_test.go +++ b/integration-cli/docker_cli_help_test.go @@ -133,7 +133,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { // Check each line for lots of stuff lines := strings.Split(out, "\n") for _, line := range lines { - c.Assert(len(line), checker.LessOrEqualThan, 103, check.Commentf("Help for %q is too long:\n%s", cmd, line)) + c.Assert(len(line), checker.LessOrEqualThan, 107, check.Commentf("Help for %q is too long:\n%s", cmd, line)) if scanForHome && strings.Contains(line, `"`+home) { c.Fatalf("Help for %q should use ~ instead of %q on:\n%s", diff --git a/man/docker-daemon.8.md b/man/docker-daemon.8.md index 233a6c8433..fc5a7fd9a7 100644 --- a/man/docker-daemon.8.md +++ b/man/docker-daemon.8.md @@ -14,6 +14,7 @@ docker-daemon - Enable daemon mode [**--cluster-store**[=*[]*]] [**--cluster-advertise**[=*[]*]] [**--cluster-store-opt**[=*map[]*]] +[**--config-file**[=*/etc/docker/daemon.json*]] [**-D**|**--debug**] [**--default-gateway**[=*DEFAULT-GATEWAY*]] [**--default-gateway-v6**[=*DEFAULT-GATEWAY-V6*]] @@ -96,6 +97,9 @@ format. **--cluster-store-opt**="" Specifies options for the Key/Value store. +**--config-file**="/etc/docker/daemon.json" + Specifies the JSON file path to load the configuration from. + **-D**, **--debug**=*true*|*false* Enable debug mode. Default is false. diff --git a/opts/opts.go b/opts/opts.go index abc9ab8a18..05aadbe74b 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -100,6 +100,35 @@ func (opts *ListOpts) Len() int { return len((*opts.values)) } +// NamedOption is an interface that list and map options +// with names implement. +type NamedOption interface { + Name() string +} + +// NamedListOpts is a ListOpts with a configuration name. +// This struct is useful to keep reference to the assigned +// field name in the internal configuration struct. +type NamedListOpts struct { + name string + ListOpts +} + +var _ NamedOption = &NamedListOpts{} + +// NewNamedListOptsRef creates a reference to a new NamedListOpts struct. +func NewNamedListOptsRef(name string, values *[]string, validator ValidatorFctType) *NamedListOpts { + return &NamedListOpts{ + name: name, + ListOpts: *NewListOptsRef(values, validator), + } +} + +// Name returns the name of the NamedListOpts in the configuration. +func (o *NamedListOpts) Name() string { + return o.name +} + //MapOpts holds a map of values and a validation function. type MapOpts struct { values map[string]string @@ -145,6 +174,29 @@ func NewMapOpts(values map[string]string, validator ValidatorFctType) *MapOpts { } } +// NamedMapOpts is a MapOpts struct with a configuration name. +// This struct is useful to keep reference to the assigned +// field name in the internal configuration struct. +type NamedMapOpts struct { + name string + MapOpts +} + +var _ NamedOption = &NamedMapOpts{} + +// NewNamedMapOpts creates a reference to a new NamedMapOpts struct. +func NewNamedMapOpts(name string, values map[string]string, validator ValidatorFctType) *NamedMapOpts { + return &NamedMapOpts{ + name: name, + MapOpts: *NewMapOpts(values, validator), + } +} + +// Name returns the name of the NamedMapOpts in the configuration. +func (o *NamedMapOpts) Name() string { + return o.name +} + // ValidatorFctType defines a validator function that returns a validated string and/or an error. type ValidatorFctType func(val string) (string, error) diff --git a/opts/opts_test.go b/opts/opts_test.go index da86b21fa3..9f41e47864 100644 --- a/opts/opts_test.go +++ b/opts/opts_test.go @@ -198,3 +198,35 @@ func logOptsValidator(val string) (string, error) { } return "", fmt.Errorf("invalid key %s", vals[0]) } + +func TestNamedListOpts(t *testing.T) { + var v []string + o := NewNamedListOptsRef("foo-name", &v, nil) + + o.Set("foo") + if o.String() != "[foo]" { + t.Errorf("%s != [foo]", o.String()) + } + if o.Name() != "foo-name" { + t.Errorf("%s != foo-name", o.Name()) + } + if len(v) != 1 { + t.Errorf("expected foo to be in the values, got %v", v) + } +} + +func TestNamedMapOpts(t *testing.T) { + tmpMap := make(map[string]string) + o := NewNamedMapOpts("max-name", tmpMap, nil) + + o.Set("max-size=1") + if o.String() != "map[max-size:1]" { + t.Errorf("%s != [map[max-size:1]", o.String()) + } + if o.Name() != "max-name" { + t.Errorf("%s != max-name", o.Name()) + } + if _, exist := tmpMap["max-size"]; !exist { + t.Errorf("expected map-size to be in the values, got %v", tmpMap) + } +} diff --git a/pkg/discovery/backends.go b/pkg/discovery/backends.go index 875a26c442..f150115a1d 100644 --- a/pkg/discovery/backends.go +++ b/pkg/discovery/backends.go @@ -12,12 +12,8 @@ import ( var ( // Backends is a global map of discovery backends indexed by their // associated scheme. - backends map[string]Backend -) - -func init() { backends = make(map[string]Backend) -} +) // Register makes a discovery backend available by the provided scheme. // If Register is called twice with the same scheme an error is returned. @@ -42,7 +38,7 @@ func parse(rawurl string) (string, string) { // ParseAdvertise parses the --cluster-advertise daemon config which accepts // : or : -func ParseAdvertise(store, advertise string) (string, error) { +func ParseAdvertise(advertise string) (string, error) { var ( iface *net.Interface addrs []net.Addr diff --git a/pkg/discovery/memory/memory.go b/pkg/discovery/memory/memory.go index f389825e6f..777a9a16a4 100644 --- a/pkg/discovery/memory/memory.go +++ b/pkg/discovery/memory/memory.go @@ -25,6 +25,7 @@ func Init() { // Initialize sets the heartbeat for the memory backend. func (s *Discovery) Initialize(_ string, heartbeat time.Duration, _ time.Duration, _ map[string]string) error { s.heartbeat = heartbeat + s.values = make([]string, 0) return nil } diff --git a/utils/debug.go b/utils/debug.go new file mode 100644 index 0000000000..d203891129 --- /dev/null +++ b/utils/debug.go @@ -0,0 +1,26 @@ +package utils + +import ( + "os" + + "github.com/Sirupsen/logrus" +) + +// EnableDebug sets the DEBUG env var to true +// and makes the logger to log at debug level. +func EnableDebug() { + os.Setenv("DEBUG", "1") + logrus.SetLevel(logrus.DebugLevel) +} + +// DisableDebug sets the DEBUG env var to false +// and makes the logger to log at info level. +func DisableDebug() { + os.Setenv("DEBUG", "") + logrus.SetLevel(logrus.InfoLevel) +} + +// IsDebugEnabled checks whether the debug flag is set or not. +func IsDebugEnabled() bool { + return os.Getenv("DEBUG") != "" +}