Vendoring libnetwork 452dff1

Signed-off-by: Alessandro Boch <aboch@docker.com>
This commit is contained in:
Alessandro Boch 2016-06-14 13:45:24 -07:00
parent 6381ed14d1
commit 5b79122146
39 changed files with 1289 additions and 118 deletions

View File

@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
clone git github.com/imdario/mergo 0.2.1
#get libnetwork packages
clone git github.com/docker/libnetwork e8da32ce5693f0ed6823d59c8415baf76c0809ea
clone git github.com/docker/libnetwork 452dff166e0abd9455b07c835613197f078a34de
clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d
clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

View File

@ -15,9 +15,10 @@ import (
// Config encapsulates configurations of various Libnetwork components
type Config struct {
Daemon DaemonCfg
Cluster ClusterCfg
Scopes map[string]*datastore.ScopeCfg
Daemon DaemonCfg
Cluster ClusterCfg
Scopes map[string]*datastore.ScopeCfg
ActiveSandboxes map[string]interface{}
}
// DaemonCfg represents libnetwork core configuration
@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
c.Scopes[datastore.LocalScope].Client.Config = config
}
}
// OptionActiveSandboxes function returns an option setter for passing the sandboxes
// which were active during previous daemon life
func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
return func(c *Config) {
c.ActiveSandboxes = sandboxes
}
}

View File

@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
}
}
// Reserve pools first before doing cleanup. This is because
// if the pools are not populated properly, the cleanups of
// endpoint/network and sandbox below will not be able to
// release ip subnets and addresses properly into the pool
// because the pools won't exist.
// Reserve pools first before doing cleanup. Otherwise the
// cleanups of endpoint/network and sandbox below will
// generate many unnecessary warnings
c.reservePools()
// Cleanup resources
c.sandboxCleanup()
c.sandboxCleanup(c.cfg.ActiveSandboxes)
c.cleanupLocalEndpoints()
c.networkCleanup()
@ -671,9 +669,27 @@ func (c *controller) reservePools() {
c.Gateway = n.ipamV6Info[i].Gateway.IP.String()
}
}
// Reserve pools
if err := n.ipamAllocate(); err != nil {
log.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)
}
// Reserve existing endpoints' addresses
ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
if err != nil {
log.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())
continue
}
epl, err := n.getEndpointsFromStore()
if err != nil {
log.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())
continue
}
for _, ep := range epl {
if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {
log.Warnf("Failed to reserve current adress for endpoint %q (%s) on network %q (%s)",
ep.Name(), ep.ID(), n.Name(), n.ID())
}
}
}
}
@ -832,7 +848,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
if sb.config.useDefaultSandBox {
c.sboxOnce.Do(func() {
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
})
if err != nil {
@ -844,7 +860,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
}
if sb.osSbox == nil && !sb.config.useExternalKey {
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
}
}

View File

@ -86,25 +86,52 @@ out:
return kmap, nil
}
func (c *cache) add(kvObject KVObject) error {
func (c *cache) add(kvObject KVObject, atomic bool) error {
kmap, err := c.kmap(kvObject)
if err != nil {
return err
}
c.Lock()
// If atomic is true, cache needs to maintain its own index
// for atomicity and the add needs to be atomic.
if atomic {
if prev, ok := kmap[Key(kvObject.Key()...)]; ok {
if prev.Index() != kvObject.Index() {
c.Unlock()
return ErrKeyModified
}
}
// Increment index
index := kvObject.Index()
index++
kvObject.SetIndex(index)
}
kmap[Key(kvObject.Key()...)] = kvObject
c.Unlock()
return nil
}
func (c *cache) del(kvObject KVObject) error {
func (c *cache) del(kvObject KVObject, atomic bool) error {
kmap, err := c.kmap(kvObject)
if err != nil {
return err
}
c.Lock()
// If atomic is true, cache needs to maintain its own index
// for atomicity and del needs to be atomic.
if atomic {
if prev, ok := kmap[Key(kvObject.Key()...)]; ok {
if prev.Index() != kvObject.Index() {
c.Unlock()
return ErrKeyModified
}
}
}
delete(kmap, Key(kvObject.Key()...))
c.Unlock()
return nil

View File

@ -410,7 +410,9 @@ func (ds *datastore) PutObjectAtomic(kvObject KVObject) error {
add_cache:
if ds.cache != nil {
return ds.cache.add(kvObject)
// If persistent store is skipped, sequencing needs to
// happen in cache.
return ds.cache.add(kvObject, kvObject.Skip())
}
return nil
@ -435,7 +437,9 @@ func (ds *datastore) PutObject(kvObject KVObject) error {
add_cache:
if ds.cache != nil {
return ds.cache.add(kvObject)
// If persistent store is skipped, sequencing needs to
// happen in cache.
return ds.cache.add(kvObject, kvObject.Skip())
}
return nil
@ -537,7 +541,9 @@ func (ds *datastore) DeleteObject(kvObject KVObject) error {
// cleaup the cache first
if ds.cache != nil {
ds.cache.del(kvObject)
// If persistent store is skipped, sequencing needs to
// happen in cache.
ds.cache.del(kvObject, kvObject.Skip())
}
if kvObject.Skip() {
@ -572,7 +578,9 @@ func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error {
del_cache:
// cleanup the cache only if AtomicDelete went through successfully
if ds.cache != nil {
return ds.cache.del(kvObject)
// If persistent store is skipped, sequencing needs to
// happen in cache.
return ds.cache.del(kvObject, kvObject.Skip())
}
return nil
@ -585,7 +593,9 @@ func (ds *datastore) DeleteTree(kvObject KVObject) error {
// cleaup the cache first
if ds.cache != nil {
ds.cache.del(kvObject)
// If persistent store is skipped, sequencing needs to
// happen in cache.
ds.cache.del(kvObject, kvObject.Skip())
}
if kvObject.Skip() {

View File

@ -91,6 +91,7 @@ type connectivityConfiguration struct {
type bridgeEndpoint struct {
id string
nid string
srcName string
addr *net.IPNet
addrv6 *net.IPNet
@ -99,6 +100,8 @@ type bridgeEndpoint struct {
containerConfig *containerConfiguration
extConnConfig *connectivityConfiguration
portMapping []types.PortBinding // Operation port bindings
dbIndex uint64
dbExists bool
}
type bridgeNetwork struct {
@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
// Create and add the endpoint
n.Lock()
endpoint := &bridgeEndpoint{id: eid, config: epConfig}
endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}
n.endpoints[eid] = endpoint
n.Unlock()
@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
}
}
if err = d.storeUpdate(endpoint); err != nil {
return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err)
}
return nil
}
@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
d.nlh.LinkDel(link)
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
}
return nil
}
@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
return err
}
if err = d.storeUpdate(endpoint); err != nil {
endpoint.portMapping = nil
return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
}
if !network.config.EnableICC {
return d.link(network, endpoint, true)
}

View File

@ -12,7 +12,13 @@ import (
"github.com/docker/libnetwork/types"
)
const bridgePrefix = "bridge"
const (
// network config prefix was not specific enough.
// To be backward compatible, need custom endpoint
// prefix with different root
bridgePrefix = "bridge"
bridgeEndpointPrefix = "bridge-endpoint"
)
func (d *driver) initStore(option map[string]interface{}) error {
if data, ok := option[netlabel.LocalKVClient]; ok {
@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error {
return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)
}
return d.populateNetworks()
err = d.populateNetworks()
if err != nil {
return err
}
err = d.populateEndpoints()
if err != nil {
return err
}
}
return nil
@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error {
if err = d.createNetwork(ncfg); err != nil {
logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)
}
logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
}
return nil
}
func (d *driver) populateEndpoints() error {
kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})
if err != nil && err != datastore.ErrKeyNotFound {
return fmt.Errorf("failed to get bridge endpoints from store: %v", err)
}
if err == datastore.ErrKeyNotFound {
return nil
}
for _, kvo := range kvol {
ep := kvo.(*bridgeEndpoint)
n, ok := d.networks[ep.nid]
if !ok {
logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])
if err := d.storeDelete(ep); err != nil {
logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])
}
continue
}
n.endpoints[ep.id] = ep
n.restorePortAllocations(ep)
logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
}
return nil
@ -184,7 +228,7 @@ func (ncfg *networkConfiguration) Exists() bool {
}
func (ncfg *networkConfiguration) Skip() bool {
return ncfg.DefaultBridge
return false
}
func (ncfg *networkConfiguration) New() datastore.KVObject {
@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
func (ncfg *networkConfiguration) DataScope() string {
return datastore.LocalScope
}
func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
epMap["id"] = ep.id
epMap["nid"] = ep.nid
epMap["SrcName"] = ep.srcName
epMap["MacAddress"] = ep.macAddress.String()
epMap["Addr"] = ep.addr.String()
if ep.addrv6 != nil {
epMap["Addrv6"] = ep.addrv6.String()
}
epMap["Config"] = ep.config
epMap["ContainerConfig"] = ep.containerConfig
epMap["ExternalConnConfig"] = ep.extConnConfig
epMap["PortMapping"] = ep.portMapping
return json.Marshal(epMap)
}
func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error {
var (
err error
epMap map[string]interface{}
)
if err = json.Unmarshal(b, &epMap); err != nil {
return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err)
}
if v, ok := epMap["MacAddress"]; ok {
if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {
return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addr"]; ok {
if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addrv6"]; ok {
if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
}
}
ep.id = epMap["id"].(string)
ep.nid = epMap["nid"].(string)
ep.srcName = epMap["SrcName"].(string)
d, _ := json.Marshal(epMap["Config"])
if err := json.Unmarshal(d, &ep.config); err != nil {
logrus.Warnf("Failed to decode endpoint config %v", err)
}
d, _ = json.Marshal(epMap["ContainerConfig"])
if err := json.Unmarshal(d, &ep.containerConfig); err != nil {
logrus.Warnf("Failed to decode endpoint container config %v", err)
}
d, _ = json.Marshal(epMap["ExternalConnConfig"])
if err := json.Unmarshal(d, &ep.extConnConfig); err != nil {
logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)
}
d, _ = json.Marshal(epMap["PortMapping"])
if err := json.Unmarshal(d, &ep.portMapping); err != nil {
logrus.Warnf("Failed to decode endpoint port mapping %v", err)
}
return nil
}
func (ep *bridgeEndpoint) Key() []string {
return []string{bridgeEndpointPrefix, ep.id}
}
func (ep *bridgeEndpoint) KeyPrefix() []string {
return []string{bridgeEndpointPrefix}
}
func (ep *bridgeEndpoint) Value() []byte {
b, err := json.Marshal(ep)
if err != nil {
return nil
}
return b
}
func (ep *bridgeEndpoint) SetValue(value []byte) error {
return json.Unmarshal(value, ep)
}
func (ep *bridgeEndpoint) Index() uint64 {
return ep.dbIndex
}
func (ep *bridgeEndpoint) SetIndex(index uint64) {
ep.dbIndex = index
ep.dbExists = true
}
func (ep *bridgeEndpoint) Exists() bool {
return ep.dbExists
}
func (ep *bridgeEndpoint) Skip() bool {
return false
}
func (ep *bridgeEndpoint) New() datastore.KVObject {
return &bridgeEndpoint{}
}
func (ep *bridgeEndpoint) CopyTo(o datastore.KVObject) error {
dstEp := o.(*bridgeEndpoint)
*dstEp = *ep
return nil
}
func (ep *bridgeEndpoint) DataScope() string {
return datastore.LocalScope
}
func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
if ep.extConnConfig == nil ||
ep.extConnConfig.ExposedPorts == nil ||
ep.extConnConfig.PortBindings == nil {
return
}
tmp := ep.extConnConfig.PortBindings
ep.extConnConfig.PortBindings = ep.portMapping
_, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy)
if err != nil {
logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err)
}
ep.extConnConfig.PortBindings = tmp
}

View File

@ -36,11 +36,14 @@ type driver struct {
}
type endpoint struct {
id string
mac net.HardwareAddr
addr *net.IPNet
addrv6 *net.IPNet
srcName string
id string
nid string
mac net.HardwareAddr
addr *net.IPNet
addrv6 *net.IPNet
srcName string
dbIndex uint64
dbExists bool
}
type network struct {

View File

@ -28,9 +28,9 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
}
ep := &endpoint{
id: eid,
nid: nid,
addr: ifInfo.Address(),
addrv6: ifInfo.AddressIPv6(),
mac: ifInfo.MacAddress(),
}
if ep.addr == nil {
return fmt.Errorf("create endpoint was not passed an IP address")
@ -51,6 +51,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
}
}
}
if err := d.storeUpdate(ep); err != nil {
return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
}
n.addEndpoint(ep)
return nil
@ -74,5 +79,9 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
ns.NlHandle().LinkDel(link)
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
}
return nil
}

View File

@ -116,6 +116,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
if err != nil {
return err
}
if err = d.storeUpdate(ep); err != nil {
return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
}
return nil
}

View File

@ -3,6 +3,7 @@ package ipvlan
import (
"encoding/json"
"fmt"
"net"
"github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/datastore"
@ -11,7 +12,11 @@ import (
"github.com/docker/libnetwork/types"
)
const ipvlanPrefix = "ipvlan" // prefix used for persistent driver storage
const (
ipvlanPrefix = "ipvlan"
ipvlanNetworkPrefix = ipvlanPrefix + "/network"
ipvlanEndpointPrefix = ipvlanPrefix + "/endpoint"
)
// networkConfiguration for this driver's network specific configuration
type configuration struct {
@ -58,7 +63,7 @@ func (d *driver) initStore(option map[string]interface{}) error {
// populateNetworks is invoked at driver init to recreate persistently stored networks
func (d *driver) populateNetworks() error {
kvol, err := d.store.List(datastore.Key(ipvlanPrefix), &configuration{})
kvol, err := d.store.List(datastore.Key(ipvlanNetworkPrefix), &configuration{})
if err != nil && err != datastore.ErrKeyNotFound {
return fmt.Errorf("failed to get ipvlan network configurations from store: %v", err)
}
@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
return nil
}
func (d *driver) populateEndpoints() error {
kvol, err := d.store.List(datastore.Key(ipvlanEndpointPrefix), &endpoint{})
if err != nil && err != datastore.ErrKeyNotFound {
return fmt.Errorf("failed to get ipvlan endpoints from store: %v", err)
}
if err == datastore.ErrKeyNotFound {
return nil
}
for _, kvo := range kvol {
ep := kvo.(*endpoint)
n, ok := d.networks[ep.nid]
if !ok {
logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.nid[0:7])
if err := d.storeDelete(ep); err != nil {
logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.nid[0:7])
}
continue
}
n.endpoints[ep.id] = ep
logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
}
return nil
}
// storeUpdate used to update persistent ipvlan network records as they are created
func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
if d.store == nil {
@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
}
func (config *configuration) Key() []string {
return []string{ipvlanPrefix, config.ID}
return []string{ipvlanNetworkPrefix, config.ID}
}
func (config *configuration) KeyPrefix() []string {
return []string{ipvlanPrefix}
return []string{ipvlanNetworkPrefix}
}
func (config *configuration) Value() []byte {
@ -214,3 +247,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
func (config *configuration) DataScope() string {
return datastore.LocalScope
}
func (ep *endpoint) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
epMap["id"] = ep.id
epMap["nid"] = ep.nid
epMap["SrcName"] = ep.srcName
if len(ep.mac) != 0 {
epMap["MacAddress"] = ep.mac.String()
}
if ep.addr != nil {
epMap["Addr"] = ep.addr.String()
}
if ep.addrv6 != nil {
epMap["Addrv6"] = ep.addrv6.String()
}
return json.Marshal(epMap)
}
func (ep *endpoint) UnmarshalJSON(b []byte) error {
var (
err error
epMap map[string]interface{}
)
if err = json.Unmarshal(b, &epMap); err != nil {
return fmt.Errorf("Failed to unmarshal to ipvlan endpoint: %v", err)
}
if v, ok := epMap["MacAddress"]; ok {
if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
return types.InternalErrorf("failed to decode ipvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addr"]; ok {
if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode ipvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addrv6"]; ok {
if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode ipvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
}
}
ep.id = epMap["id"].(string)
ep.nid = epMap["nid"].(string)
ep.srcName = epMap["SrcName"].(string)
return nil
}
func (ep *endpoint) Key() []string {
return []string{ipvlanEndpointPrefix, ep.id}
}
func (ep *endpoint) KeyPrefix() []string {
return []string{ipvlanEndpointPrefix}
}
func (ep *endpoint) Value() []byte {
b, err := json.Marshal(ep)
if err != nil {
return nil
}
return b
}
func (ep *endpoint) SetValue(value []byte) error {
return json.Unmarshal(value, ep)
}
func (ep *endpoint) Index() uint64 {
return ep.dbIndex
}
func (ep *endpoint) SetIndex(index uint64) {
ep.dbIndex = index
ep.dbExists = true
}
func (ep *endpoint) Exists() bool {
return ep.dbExists
}
func (ep *endpoint) Skip() bool {
return false
}
func (ep *endpoint) New() datastore.KVObject {
return &endpoint{}
}
func (ep *endpoint) CopyTo(o datastore.KVObject) error {
dstEp := o.(*endpoint)
*dstEp = *ep
return nil
}
func (ep *endpoint) DataScope() string {
return datastore.LocalScope
}

View File

@ -38,11 +38,14 @@ type driver struct {
}
type endpoint struct {
id string
mac net.HardwareAddr
addr *net.IPNet
addrv6 *net.IPNet
srcName string
id string
nid string
mac net.HardwareAddr
addr *net.IPNet
addrv6 *net.IPNet
srcName string
dbIndex uint64
dbExists bool
}
type network struct {

View File

@ -26,6 +26,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
}
ep := &endpoint{
id: eid,
nid: nid,
addr: ifInfo.Address(),
addrv6: ifInfo.AddressIPv6(),
mac: ifInfo.MacAddress(),
@ -55,6 +56,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
}
}
}
if err := d.storeUpdate(ep); err != nil {
return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
}
n.addEndpoint(ep)
return nil
@ -77,6 +83,8 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
ns.NlHandle().LinkDel(link)
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
}
return nil
}

View File

@ -77,7 +77,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
if err != nil {
return err
}
if err := d.storeUpdate(ep); err != nil {
return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
}
return nil
}

View File

@ -3,6 +3,7 @@ package macvlan
import (
"encoding/json"
"fmt"
"net"
"github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/datastore"
@ -11,7 +12,11 @@ import (
"github.com/docker/libnetwork/types"
)
const macvlanPrefix = "macvlan" // prefix used for persistent driver storage
const (
macvlanPrefix = "macvlan"
macvlanNetworkPrefix = macvlanPrefix + "/network"
macvlanEndpointPrefix = macvlanPrefix + "/endpoint"
)
// networkConfiguration for this driver's network specific configuration
type configuration struct {
@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
return nil
}
func (d *driver) populateEndpoints() error {
kvol, err := d.store.List(datastore.Key(macvlanEndpointPrefix), &endpoint{})
if err != nil && err != datastore.ErrKeyNotFound {
return fmt.Errorf("failed to get macvlan endpoints from store: %v", err)
}
if err == datastore.ErrKeyNotFound {
return nil
}
for _, kvo := range kvol {
ep := kvo.(*endpoint)
n, ok := d.networks[ep.nid]
if !ok {
logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.nid[0:7])
if err := d.storeDelete(ep); err != nil {
logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.nid[0:7])
}
continue
}
n.endpoints[ep.id] = ep
logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
}
return nil
}
// storeUpdate used to update persistent macvlan network records as they are created
func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
if d.store == nil {
@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
}
func (config *configuration) Key() []string {
return []string{macvlanPrefix, config.ID}
return []string{macvlanNetworkPrefix, config.ID}
}
func (config *configuration) KeyPrefix() []string {
return []string{macvlanPrefix}
return []string{macvlanNetworkPrefix}
}
func (config *configuration) Value() []byte {
@ -216,3 +249,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
func (config *configuration) DataScope() string {
return datastore.LocalScope
}
func (ep *endpoint) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
epMap["id"] = ep.id
epMap["nid"] = ep.nid
epMap["SrcName"] = ep.srcName
if len(ep.mac) != 0 {
epMap["MacAddress"] = ep.mac.String()
}
if ep.addr != nil {
epMap["Addr"] = ep.addr.String()
}
if ep.addrv6 != nil {
epMap["Addrv6"] = ep.addrv6.String()
}
return json.Marshal(epMap)
}
func (ep *endpoint) UnmarshalJSON(b []byte) error {
var (
err error
epMap map[string]interface{}
)
if err = json.Unmarshal(b, &epMap); err != nil {
return fmt.Errorf("Failed to unmarshal to macvlan endpoint: %v", err)
}
if v, ok := epMap["MacAddress"]; ok {
if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
return types.InternalErrorf("failed to decode macvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addr"]; ok {
if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode macvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
}
}
if v, ok := epMap["Addrv6"]; ok {
if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode macvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
}
}
ep.id = epMap["id"].(string)
ep.nid = epMap["nid"].(string)
ep.srcName = epMap["SrcName"].(string)
return nil
}
func (ep *endpoint) Key() []string {
return []string{macvlanEndpointPrefix, ep.id}
}
func (ep *endpoint) KeyPrefix() []string {
return []string{macvlanEndpointPrefix}
}
func (ep *endpoint) Value() []byte {
b, err := json.Marshal(ep)
if err != nil {
return nil
}
return b
}
func (ep *endpoint) SetValue(value []byte) error {
return json.Unmarshal(value, ep)
}
func (ep *endpoint) Index() uint64 {
return ep.dbIndex
}
func (ep *endpoint) SetIndex(index uint64) {
ep.dbIndex = index
ep.dbExists = true
}
func (ep *endpoint) Exists() bool {
return ep.dbExists
}
func (ep *endpoint) Skip() bool {
return false
}
func (ep *endpoint) New() datastore.KVObject {
return &endpoint{}
}
func (ep *endpoint) CopyTo(o datastore.KVObject) error {
dstEp := o.(*endpoint)
*dstEp = *ep
return nil
}
func (ep *endpoint) DataScope() string {
return datastore.LocalScope
}

View File

@ -10,6 +10,7 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/iptables"
"github.com/docker/libnetwork/ns"
"github.com/docker/libnetwork/types"
"github.com/vishvananda/netlink"
"strconv"
@ -214,12 +215,12 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
var (
crypt *netlink.XfrmStateAlgo
action = "Removing"
xfrmProgram = netlink.XfrmStateDel
xfrmProgram = ns.NlHandle().XfrmStateDel
)
if add {
action = "Adding"
xfrmProgram = netlink.XfrmStateAdd
xfrmProgram = ns.NlHandle().XfrmStateAdd
crypt = &netlink.XfrmStateAlgo{Name: "cbc(aes)", Key: k.value}
}
@ -278,10 +279,10 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
action := "Removing"
xfrmProgram := netlink.XfrmPolicyDel
xfrmProgram := ns.NlHandle().XfrmPolicyDel
if add {
action = "Adding"
xfrmProgram = netlink.XfrmPolicyAdd
xfrmProgram = ns.NlHandle().XfrmPolicyAdd
}
fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src))
@ -322,7 +323,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
}
func saExists(sa *netlink.XfrmState) (bool, error) {
_, err := netlink.XfrmStateGet(sa)
_, err := ns.NlHandle().XfrmStateGet(sa)
switch err {
case nil:
return true, nil
@ -336,7 +337,7 @@ func saExists(sa *netlink.XfrmState) (bool, error) {
}
func spExists(sp *netlink.XfrmPolicy) (bool, error) {
_, err := netlink.XfrmPolicyGet(sp)
_, err := ns.NlHandle().XfrmPolicyGet(sp)
switch err {
case nil:
return true, nil
@ -482,7 +483,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
Limits: netlink.XfrmStateLimits{TimeSoft: timeout},
}
log.Infof("Updating rSA0{%s}", rSA0)
if err := netlink.XfrmStateUpdate(rSA0); err != nil {
if err := ns.NlHandle().XfrmStateUpdate(rSA0); err != nil {
log.Warnf("Failed to update rSA0{%s}: %v", rSA0, err)
}
}
@ -518,7 +519,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
},
}
log.Infof("Updating fSP{%s}", fSP1)
if err := netlink.XfrmPolicyUpdate(fSP1); err != nil {
if err := ns.NlHandle().XfrmPolicyUpdate(fSP1); err != nil {
log.Warnf("Failed to update fSP{%s}: %v", fSP1, err)
}
@ -533,7 +534,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
Limits: netlink.XfrmStateLimits{TimeHard: timeout},
}
log.Infof("Removing fSA0{%s}", fSA0)
if err := netlink.XfrmStateUpdate(fSA0); err != nil {
if err := ns.NlHandle().XfrmStateUpdate(fSA0); err != nil {
log.Warnf("Failed to remove fSA0{%s}: %v", fSA0, err)
}
}

View File

@ -40,11 +40,11 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
}
if err := n.joinSandbox(); err != nil {
if err := n.joinSandbox(false); err != nil {
return fmt.Errorf("network sandbox join failed: %v", err)
}
if err := n.joinSubnetSandbox(s); err != nil {
if err := n.joinSubnetSandbox(s, false); err != nil {
return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
}
@ -61,6 +61,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
ep.ifName = containerIfName
if err := d.writeEndpointToStore(ep); err != nil {
return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
}
nlh := ns.NlHandle()
// Set the container interface and its peer MTU to 1450 to allow

View File

@ -1,22 +1,30 @@
package overlay
import (
"encoding/json"
"fmt"
"net"
log "github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/datastore"
"github.com/docker/libnetwork/driverapi"
"github.com/docker/libnetwork/netutils"
"github.com/docker/libnetwork/ns"
"github.com/docker/libnetwork/types"
)
type endpointTable map[string]*endpoint
const overlayEndpointPrefix = "overlay/endpoint"
type endpoint struct {
id string
ifName string
mac net.HardwareAddr
addr *net.IPNet
id string
nid string
ifName string
mac net.HardwareAddr
addr *net.IPNet
dbExists bool
dbIndex uint64
}
func (n *network) endpoint(eid string) *endpoint {
@ -60,6 +68,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
ep := &endpoint{
id: eid,
nid: n.id,
addr: ifInfo.Address(),
mac: ifInfo.MacAddress(),
}
@ -80,6 +89,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
n.addEndpoint(ep)
if err := d.writeEndpointToStore(ep); err != nil {
return fmt.Errorf("failed to update overlay endpoint %s to local store: %v", ep.id[0:7], err)
}
return nil
}
@ -102,6 +115,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
n.deleteEndpoint(eid)
if err := d.deleteEndpointFromStore(ep); err != nil {
log.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
}
if ep.ifName == "" {
return nil
}
@ -121,3 +138,122 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
return make(map[string]interface{}, 0), nil
}
func (d *driver) deleteEndpointFromStore(e *endpoint) error {
if d.localStore == nil {
return fmt.Errorf("overlay local store not initialized, ep not deleted")
}
if err := d.localStore.DeleteObjectAtomic(e); err != nil {
return err
}
return nil
}
func (d *driver) writeEndpointToStore(e *endpoint) error {
if d.localStore == nil {
return fmt.Errorf("overlay local store not initialized, ep not added")
}
if err := d.localStore.PutObjectAtomic(e); err != nil {
return err
}
return nil
}
func (ep *endpoint) DataScope() string {
return datastore.LocalScope
}
func (ep *endpoint) New() datastore.KVObject {
return &endpoint{}
}
func (ep *endpoint) CopyTo(o datastore.KVObject) error {
dstep := o.(*endpoint)
*dstep = *ep
return nil
}
func (ep *endpoint) Key() []string {
return []string{overlayEndpointPrefix, ep.id}
}
func (ep *endpoint) KeyPrefix() []string {
return []string{overlayEndpointPrefix}
}
func (ep *endpoint) Index() uint64 {
return ep.dbIndex
}
func (ep *endpoint) SetIndex(index uint64) {
ep.dbIndex = index
ep.dbExists = true
}
func (ep *endpoint) Exists() bool {
return ep.dbExists
}
func (ep *endpoint) Skip() bool {
return false
}
func (ep *endpoint) Value() []byte {
b, err := json.Marshal(ep)
if err != nil {
return nil
}
return b
}
func (ep *endpoint) SetValue(value []byte) error {
return json.Unmarshal(value, ep)
}
func (ep *endpoint) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
epMap["id"] = ep.id
epMap["nid"] = ep.nid
if ep.ifName != "" {
epMap["ifName"] = ep.ifName
}
if ep.addr != nil {
epMap["addr"] = ep.addr.String()
}
if len(ep.mac) != 0 {
epMap["mac"] = ep.mac.String()
}
return json.Marshal(epMap)
}
func (ep *endpoint) UnmarshalJSON(value []byte) error {
var (
err error
epMap map[string]interface{}
)
json.Unmarshal(value, &epMap)
ep.id = epMap["id"].(string)
ep.nid = epMap["nid"].(string)
if v, ok := epMap["mac"]; ok {
if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string))
}
}
if v, ok := epMap["addr"]; ok {
if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err)
}
}
if v, ok := epMap["ifName"]; ok {
ep.ifName = v.(string)
}
return nil
}

View File

@ -195,21 +195,21 @@ func (n *network) incEndpointCount() {
n.joinCnt++
}
func (n *network) joinSandbox() error {
func (n *network) joinSandbox(restore bool) error {
// If there is a race between two go routines here only one will win
// the other will wait.
n.once.Do(func() {
// save the error status of initSandbox in n.initErr so that
// all the racing go routines are able to know the status.
n.initErr = n.initSandbox()
n.initErr = n.initSandbox(restore)
})
return n.initErr
}
func (n *network) joinSubnetSandbox(s *subnet) error {
func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {
s.once.Do(func() {
s.initErr = n.initSubnetSandbox(s)
s.initErr = n.initSubnetSandbox(s, restore)
})
return s.initErr
}
@ -386,9 +386,33 @@ func isOverlap(nw *net.IPNet) bool {
return false
}
func (n *network) initSubnetSandbox(s *subnet) error {
brName := n.generateBridgeName(s)
vxlanName := n.generateVxlanName(s)
func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {
sbox := n.sandbox()
// restore overlay osl sandbox
Ifaces := make(map[string][]osl.IfaceOption)
brIfaceOption := make([]osl.IfaceOption, 2)
brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP))
brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true))
Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption
err := sbox.Restore(Ifaces, nil, nil, nil)
if err != nil {
return err
}
Ifaces = make(map[string][]osl.IfaceOption)
vxlanIfaceOption := make([]osl.IfaceOption, 1)
vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption
err = sbox.Restore(Ifaces, nil, nil, nil)
if err != nil {
return err
}
return nil
}
func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
if hostMode {
// Try to delete stale bridge interface if it exists
@ -451,6 +475,19 @@ func (n *network) initSubnetSandbox(s *subnet) error {
}
}
return nil
}
func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
brName := n.generateBridgeName(s)
vxlanName := n.generateVxlanName(s)
if restore {
n.restoreSubnetSandbox(s, brName, vxlanName)
} else {
n.setupSubnetSandbox(s, brName, vxlanName)
}
n.Lock()
s.vxlanName = vxlanName
s.brName = brName
@ -494,32 +531,45 @@ func (n *network) cleanupStaleSandboxes() {
})
}
func (n *network) initSandbox() error {
func (n *network) initSandbox(restore bool) error {
n.Lock()
n.initEpoch++
n.Unlock()
networkOnce.Do(networkOnceInit)
if hostMode {
if err := addNetworkChain(n.id[:12]); err != nil {
return err
if !restore {
if hostMode {
if err := addNetworkChain(n.id[:12]); err != nil {
return err
}
}
// If there are any stale sandboxes related to this network
// from previous daemon life clean it up here
n.cleanupStaleSandboxes()
}
// If there are any stale sandboxes related to this network
// from previous daemon life clean it up here
n.cleanupStaleSandboxes()
// In the restore case network sandbox already exist; but we don't know
// what epoch number it was created with. It has to be retrieved by
// searching the net namespaces.
key := ""
if restore {
key = osl.GenerateKey("-" + n.id)
} else {
key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id)
}
sbox, err := osl.NewSandbox(
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
sbox, err := osl.NewSandbox(key, !hostMode, restore)
if err != nil {
return fmt.Errorf("could not create network sandbox: %v", err)
return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
}
n.setSandbox(sbox)
n.driver.peerDbUpdateSandbox(n.id)
if !restore {
n.driver.peerDbUpdateSandbox(n.id)
}
var nlSock *nl.NetlinkSocket
sbox.InvokeFunc(func() {

View File

@ -13,6 +13,7 @@ import (
"github.com/docker/libnetwork/driverapi"
"github.com/docker/libnetwork/idm"
"github.com/docker/libnetwork/netlabel"
"github.com/docker/libnetwork/osl"
"github.com/docker/libnetwork/types"
"github.com/hashicorp/serf/serf"
)
@ -41,6 +42,7 @@ type driver struct {
serfInstance *serf.Serf
networks networkTable
store datastore.DataStore
localStore datastore.DataStore
vxlanIdm *idm.Idm
once sync.Once
joinOnce sync.Once
@ -74,9 +76,75 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
}
}
if data, ok := config[netlabel.LocalKVClient]; ok {
var err error
dsc, ok := data.(discoverapi.DatastoreConfigData)
if !ok {
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
}
d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
if err != nil {
return types.InternalErrorf("failed to initialize local data store: %v", err)
}
}
d.restoreEndpoints()
return dc.RegisterDriver(networkType, d, c)
}
// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
func (d *driver) restoreEndpoints() error {
if d.localStore == nil {
logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing")
return nil
}
kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
if err != nil && err != datastore.ErrKeyNotFound {
return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
}
if err == datastore.ErrKeyNotFound {
return nil
}
for _, kvo := range kvol {
ep := kvo.(*endpoint)
n := d.network(ep.nid)
if n == nil {
logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid, ep.id)
continue
}
n.addEndpoint(ep)
s := n.getSubnetforIP(ep.addr)
if s == nil {
return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
}
if err := n.joinSandbox(true); err != nil {
return fmt.Errorf("restore network sandbox failed: %v", err)
}
if err := n.joinSubnetSandbox(s, true); err != nil {
return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err)
}
Ifaces := make(map[string][]osl.IfaceOption)
vethIfaceOption := make([]osl.IfaceOption, 1)
vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption
err := n.sbox.Restore(Ifaces, nil, nil, nil)
if err != nil {
return fmt.Errorf("failed to restore overlay sandbox: %v", err)
}
n.incEndpointCount()
d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true)
}
return nil
}
// Fini cleans up the driver resources
func Fini(drv driverapi.Driver) {
d := drv.(*driver)

View File

@ -271,7 +271,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
}
if err := n.joinSubnetSandbox(s); err != nil {
if err := n.joinSubnetSandbox(s, false); err != nil {
return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
}

View File

@ -2,14 +2,10 @@
package libnetwork
import (
"github.com/docker/libnetwork/drivers/ipvlan"
"github.com/docker/libnetwork/drivers/macvlan"
)
import "github.com/docker/libnetwork/drivers/ipvlan"
func additionalDrivers() []initializer {
return []initializer{
{macvlan.Init, "macvlan"},
{ipvlan.Init, "ipvlan"},
}
}

View File

@ -3,6 +3,7 @@ package libnetwork
import (
"github.com/docker/libnetwork/drivers/bridge"
"github.com/docker/libnetwork/drivers/host"
"github.com/docker/libnetwork/drivers/macvlan"
"github.com/docker/libnetwork/drivers/null"
"github.com/docker/libnetwork/drivers/overlay"
"github.com/docker/libnetwork/drivers/remote"
@ -12,6 +13,7 @@ func getInitializers() []initializer {
in := []initializer{
{bridge.Init, "bridge"},
{host.Init, "host"},
{macvlan.Init, "macvlan"},
{null.Init, "null"},
{remote.Init, "remote"},
{overlay.Init, "overlay"},

View File

@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
epMap["name"] = ep.name
epMap["id"] = ep.id
epMap["ep_iface"] = ep.iface
epMap["joinInfo"] = ep.joinInfo
epMap["exposed_ports"] = ep.exposedPorts
if ep.generic != nil {
epMap["generic"] = ep.generic
@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
ib, _ := json.Marshal(epMap["ep_iface"])
json.Unmarshal(ib, &ep.iface)
jb, _ := json.Marshal(epMap["joinInfo"])
json.Unmarshal(jb, &ep.joinInfo)
tb, _ := json.Marshal(epMap["exposed_ports"])
var tPorts []types.TransportPort
json.Unmarshal(tb, &tPorts)
@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
ep.iface.CopyTo(dstEp.iface)
}
if ep.joinInfo != nil {
dstEp.joinInfo = &endpointJoinInfo{}
ep.joinInfo.CopyTo(dstEp.joinInfo)
}
dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
copy(dstEp.exposedPorts, ep.exposedPorts)
@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
}
func (c *controller) cleanupLocalEndpoints() {
// Get used endpoints
eps := make(map[string]interface{})
for _, sb := range c.sandboxes {
for _, ep := range sb.endpoints {
eps[ep.id] = true
}
}
nl, err := c.getNetworksForScope(datastore.LocalScope)
if err != nil {
log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
}
for _, ep := range epl {
if _, ok := eps[ep.id]; ok {
continue
}
log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
if err := ep.Delete(true); err != nil {
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

View File

@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
ep.joinInfo.disableGatewayService = true
}
func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
if epj.gw != nil {
epMap["gw"] = epj.gw.String()
}
if epj.gw6 != nil {
epMap["gw6"] = epj.gw6.String()
}
epMap["disableGatewayService"] = epj.disableGatewayService
epMap["StaticRoutes"] = epj.StaticRoutes
return json.Marshal(epMap)
}
func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
var (
err error
epMap map[string]interface{}
)
if err = json.Unmarshal(b, &epMap); err != nil {
return err
}
if v, ok := epMap["gw"]; ok {
epj.gw6 = net.ParseIP(v.(string))
}
if v, ok := epMap["gw6"]; ok {
epj.gw6 = net.ParseIP(v.(string))
}
epj.disableGatewayService = epMap["disableGatewayService"].(bool)
var tStaticRoute []types.StaticRoute
if v, ok := epMap["StaticRoutes"]; ok {
tb, _ := json.Marshal(v)
var tStaticRoute []types.StaticRoute
json.Unmarshal(tb, &tStaticRoute)
}
var StaticRoutes []*types.StaticRoute
for _, r := range tStaticRoute {
StaticRoutes = append(StaticRoutes, &r)
}
epj.StaticRoutes = StaticRoutes
return nil
}
func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
dstEpj.disableGatewayService = epj.disableGatewayService
dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
copy(dstEpj.StaticRoutes, epj.StaticRoutes)
dstEpj.gw = types.GetIPCopy(epj.gw)
dstEpj.gw = types.GetIPCopy(epj.gw6)
return nil
}

View File

@ -311,7 +311,16 @@ func (nDB *NetworkDB) bulkSyncTables() {
nid := networks[0]
networks = networks[1:]
completed, err := nDB.bulkSync(nid, false)
nDB.RLock()
nodes := nDB.networkNodes[nid]
nDB.RUnlock()
// No peer nodes on this network. Move on.
if len(nodes) == 0 {
continue
}
completed, err := nDB.bulkSync(nid, nodes, false)
if err != nil {
logrus.Errorf("periodic bulk sync failure for network %s: %v", nid, err)
continue
@ -334,11 +343,7 @@ func (nDB *NetworkDB) bulkSyncTables() {
}
}
func (nDB *NetworkDB) bulkSync(nid string, all bool) ([]string, error) {
nDB.RLock()
nodes := nDB.networkNodes[nid]
nDB.RUnlock()
func (nDB *NetworkDB) bulkSync(nid string, nodes []string, all bool) ([]string, error) {
if !all {
// If not all, then just pick one.
nodes = nDB.mRandomNodes(1, nodes)

View File

@ -2,6 +2,7 @@ package networkdb
import (
"fmt"
"net"
"time"
"github.com/Sirupsen/logrus"
@ -210,8 +211,13 @@ func (nDB *NetworkDB) handleBulkSync(buf []byte) {
return
}
var nodeAddr net.IP
if node, ok := nDB.nodes[bsm.NodeName]; ok {
nodeAddr = node.Addr
}
if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
logrus.Errorf("Error in responding to bulk sync from node %s: %v", nDB.nodes[bsm.NodeName].Addr, err)
logrus.Errorf("Error in responding to bulk sync from node %s: %v", nodeAddr, err)
}
}

View File

@ -14,6 +14,7 @@ func (e *eventDelegate) NotifyJoin(n *memberlist.Node) {
func (e *eventDelegate) NotifyLeave(n *memberlist.Node) {
e.nDB.deleteNodeTableEntries(n.Name)
e.nDB.deleteNetworkNodeEntries(n.Name)
e.nDB.Lock()
delete(e.nDB.nodes, n.Name)
e.nDB.Unlock()

View File

@ -286,6 +286,23 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {
return nil
}
func (nDB *NetworkDB) deleteNetworkNodeEntries(deletedNode string) {
nDB.Lock()
for nid, nodes := range nDB.networkNodes {
updatedNodes := make([]string, 0, len(nodes))
for _, node := range nodes {
if node == deletedNode {
continue
}
updatedNodes = append(updatedNodes, node)
}
nDB.networkNodes[nid] = updatedNodes
}
nDB.Unlock()
}
func (nDB *NetworkDB) deleteNodeTableEntries(node string) {
nDB.Lock()
nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {
@ -359,6 +376,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
RetransmitMult: 4,
}
nDB.networkNodes[nid] = append(nDB.networkNodes[nid], nDB.config.NodeName)
networkNodes := nDB.networkNodes[nid]
nDB.Unlock()
if err := nDB.sendNetworkEvent(nid, NetworkEventTypeJoin, ltime); err != nil {
@ -366,7 +384,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
}
logrus.Debugf("%s: joined network %s", nDB.config.NodeName, nid)
if _, err := nDB.bulkSync(nid, true); err != nil {
if _, err := nDB.bulkSync(nid, networkNodes, true); err != nil {
logrus.Errorf("Error bulk syncing while joining network %s: %v", nid, err)
}

View File

@ -2,10 +2,13 @@ package osl
import (
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
@ -133,6 +136,39 @@ func GC() {
// container id.
func GenerateKey(containerID string) string {
maxLen := 12
// Read sandbox key from host for overlay
if strings.HasPrefix(containerID, "-") {
var (
index int
indexStr string
tmpkey string
)
dir, err := ioutil.ReadDir(prefix)
if err != nil {
return ""
}
for _, v := range dir {
id := v.Name()
if strings.HasSuffix(id, containerID[:maxLen-1]) {
indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
tmpindex, err := strconv.Atoi(indexStr)
if err != nil {
return ""
}
if tmpindex > index {
index = tmpindex
tmpkey = id
}
}
}
containerID = tmpkey
if containerID == "" {
return ""
}
}
if len(containerID) < maxLen {
maxLen = len(containerID)
}
@ -142,10 +178,14 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
err := createNetworkNamespace(key, osCreate)
if err != nil {
return nil, err
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
if !isRestore {
err := createNetworkNamespace(key, osCreate)
if err != nil {
return nil, err
}
} else {
once.Do(createBasePath)
}
n := &networkNamespace{path: key, isDefault: !osCreate}
@ -347,3 +387,105 @@ func (n *networkNamespace) Destroy() error {
addToGarbagePaths(n.path)
return nil
}
// Restore restore the network namespace
func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
// restore interfaces
for name, opts := range ifsopt {
if !strings.Contains(name, "+") {
return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
}
seps := strings.Split(name, "+")
srcName := seps[0]
dstPrefix := seps[1]
i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
i.processInterfaceOptions(opts...)
if i.master != "" {
i.dstMaster = n.findDst(i.master, true)
if i.dstMaster == "" {
return fmt.Errorf("could not find an appropriate master %q for %q",
i.master, i.srcName)
}
}
if n.isDefault {
i.dstName = i.srcName
} else {
links, err := n.nlHandle.LinkList()
if err != nil {
return fmt.Errorf("failed to retrieve list of links in network namespace %q during restore", n.path)
}
// due to the docker network connect/disconnect, so the dstName should
// restore from the namespace
for _, link := range links {
addrs, err := n.nlHandle.AddrList(link, netlink.FAMILY_V4)
if err != nil {
return err
}
ifaceName := link.Attrs().Name
if strings.HasPrefix(ifaceName, "vxlan") {
if i.dstName == "vxlan" {
i.dstName = ifaceName
break
}
}
// find the interface name by ip
if i.address != nil {
for _, addr := range addrs {
if addr.IPNet.String() == i.address.String() {
i.dstName = ifaceName
break
}
continue
}
if i.dstName == ifaceName {
break
}
}
// This is to find the interface name of the pair in overlay sandbox
if strings.HasPrefix(ifaceName, "veth") {
if i.master != "" && i.dstName == "veth" {
i.dstName = ifaceName
}
}
}
var index int
indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
if indexStr != "" {
index, err = strconv.Atoi(indexStr)
if err != nil {
return err
}
}
index++
n.Lock()
if index > n.nextIfIndex {
n.nextIfIndex = index
}
n.iFaces = append(n.iFaces, i)
n.Unlock()
}
}
// restore routes
for _, r := range routes {
n.Lock()
n.staticRoutes = append(n.staticRoutes, r)
n.Unlock()
}
// restore gateway
if len(gw) > 0 {
n.Lock()
n.gw = gw
n.Unlock()
}
if len(gw6) > 0 {
n.Lock()
n.gwv6 = gw6
n.Unlock()
}
return nil
}

View File

@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, nil
}

View File

@ -58,6 +58,9 @@ type Sandbox interface {
// Destroy the sandbox
Destroy() error
// restore sandbox
Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
}
// NeighborOptionSetter interface defines the option setter methods for interface options

View File

@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, nil
}

View File

@ -11,7 +11,7 @@ var (
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, ErrNotImplemented
}

View File

@ -19,6 +19,13 @@ func init() {
reexec.Register("setup-resolver", reexecSetupResolver)
}
const (
// outputChain used for docker embed dns
outputChain = "DOCKER_OUTPUT"
//postroutingchain used for docker embed dns
postroutingchain = "DOCKER_POSTROUTING"
)
func reexecSetupResolver() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
@ -31,10 +38,10 @@ func reexecSetupResolver() {
_, ipPort, _ := net.SplitHostPort(os.Args[2])
_, tcpPort, _ := net.SplitHostPort(os.Args[3])
rules := [][]string{
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
}
f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
@ -50,6 +57,23 @@ func reexecSetupResolver() {
os.Exit(3)
}
// insert outputChain and postroutingchain
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
if err == nil {
iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
} else {
iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
}
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
if err == nil {
iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
} else {
iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
}
for _, rule := range rules {
if iptables.RawCombinedOutputNative(rule...) != nil {
log.Errorf("setting up rule failed, %v", rule)

View File

@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
osSbox.Destroy()
}
func (sb *sandbox) restoreOslSandbox() error {
var routes []*types.StaticRoute
// restore osl sandbox
Ifaces := make(map[string][]osl.IfaceOption)
for _, ep := range sb.endpoints {
var ifaceOptions []osl.IfaceOption
ep.Lock()
joinInfo := ep.joinInfo
i := ep.iface
ep.Unlock()
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
}
if i.mac != nil {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
}
if len(i.llAddrs) != 0 {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
}
Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
if joinInfo != nil {
for _, r := range joinInfo.StaticRoutes {
routes = append(routes, r)
}
}
if ep.needResolver() {
sb.startResolver()
}
}
gwep := sb.getGatewayEndpoint()
if gwep == nil {
return nil
}
// restore osl sandbox
err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
if err != nil {
return err
}
return nil
}
func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
sb.Lock()
if sb.osSbox == nil {

View File

@ -139,6 +139,16 @@ func (sb *sandbox) updateParentHosts() error {
return nil
}
func (sb *sandbox) restorePath() {
if sb.config.resolvConfPath == "" {
sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
}
sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
if sb.config.hostsPath == "" {
sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
}
}
func (sb *sandbox) setupDNS() error {
var newRC *resolvconf.File

View File

@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
return nil
}
func (sb *sandbox) restorePath() {
}
func (sb *sandbox) updateHostsFile(ifaceIP string) error {
return nil
}

View File

@ -20,12 +20,13 @@ type epState struct {
}
type sbState struct {
ID string
Cid string
c *controller
dbIndex uint64
dbExists bool
Eps []epState
ID string
Cid string
c *controller
dbIndex uint64
dbExists bool
Eps []epState
EpPriority map[string]int
}
func (sbs *sbState) Key() []string {
@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
dstSbs.Cid = sbs.Cid
dstSbs.dbIndex = sbs.dbIndex
dstSbs.dbExists = sbs.dbExists
dstSbs.EpPriority = sbs.EpPriority
for _, eps := range sbs.Eps {
dstSbs.Eps = append(dstSbs.Eps, eps)
@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string {
func (sb *sandbox) storeUpdate() error {
sbs := &sbState{
c: sb.controller,
ID: sb.id,
Cid: sb.containerID,
c: sb.controller,
ID: sb.id,
Cid: sb.containerID,
EpPriority: sb.epPriority,
}
retry:
@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
return sb.controller.deleteFromStore(sbs)
}
func (c *controller) sandboxCleanup() {
func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
store := c.getStore(datastore.LocalScope)
if store == nil {
logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
controller: sbs.c,
containerID: sbs.Cid,
endpoints: epHeap{},
epPriority: map[string]int{},
dbIndex: sbs.dbIndex,
isStub: true,
dbExists: true,
}
sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
msg := " for cleanup"
create := true
isRestore := false
if val, ok := activeSandboxes[sb.ID()]; ok {
msg = ""
sb.isStub = false
isRestore = true
opts := val.([]SandboxOption)
sb.processOptions(opts...)
sb.restorePath()
create = !sb.config.useDefaultSandBox
heap.Init(&sb.endpoints)
}
sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
if err != nil {
logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
continue
}
@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
}
}
heap.Push(&sb.endpoints, ep)
}
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
if err := sb.delete(true); err != nil {
logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
if _, ok := activeSandboxes[sb.ID()]; !ok {
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
if err := sb.delete(true); err != nil {
logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
}
continue
}
// reconstruct osl sandbox field
if !sb.config.useDefaultSandBox {
if err := sb.restoreOslSandbox(); err != nil {
logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
continue
}
} else {
c.sboxOnce.Do(func() {
c.defOsSbox = sb.osSbox
})
}
for _, ep := range sb.endpoints {
// Watch for service records
if !c.isAgent() {
c.watchSvcRecord(ep)
}
}
}
}