Add network restore to support docker live restore container
Signed-off-by: Lei Jitang <leijitang@huawei.com>
This commit is contained in:
parent
bbba96fcc5
commit
055c5dd496
|
@ -18,6 +18,7 @@ type Config struct {
|
||||||
Daemon DaemonCfg
|
Daemon DaemonCfg
|
||||||
Cluster ClusterCfg
|
Cluster ClusterCfg
|
||||||
Scopes map[string]*datastore.ScopeCfg
|
Scopes map[string]*datastore.ScopeCfg
|
||||||
|
ActiveSandboxes map[string]interface{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DaemonCfg represents libnetwork core configuration
|
// DaemonCfg represents libnetwork core configuration
|
||||||
|
@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
|
||||||
c.Scopes[datastore.LocalScope].Client.Config = config
|
c.Scopes[datastore.LocalScope].Client.Config = config
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OptionActiveSandboxes function returns an option setter for passing the sandboxes
|
||||||
|
// which were active during previous daemon life
|
||||||
|
func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
|
||||||
|
return func(c *Config) {
|
||||||
|
c.ActiveSandboxes = sandboxes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reserve pools first before doing cleanup. This is because
|
// Reserve pools first before doing cleanup. Otherwise the
|
||||||
// if the pools are not populated properly, the cleanups of
|
// cleanups of endpoint/network and sandbox below will
|
||||||
// endpoint/network and sandbox below will not be able to
|
// generate many unnecessary warnings
|
||||||
// release ip subnets and addresses properly into the pool
|
|
||||||
// because the pools won't exist.
|
|
||||||
c.reservePools()
|
c.reservePools()
|
||||||
|
|
||||||
// Cleanup resources
|
// Cleanup resources
|
||||||
c.sandboxCleanup()
|
c.sandboxCleanup(c.cfg.ActiveSandboxes)
|
||||||
c.cleanupLocalEndpoints()
|
c.cleanupLocalEndpoints()
|
||||||
c.networkCleanup()
|
c.networkCleanup()
|
||||||
|
|
||||||
|
@ -832,7 +830,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
|
||||||
|
|
||||||
if sb.config.useDefaultSandBox {
|
if sb.config.useDefaultSandBox {
|
||||||
c.sboxOnce.Do(func() {
|
c.sboxOnce.Do(func() {
|
||||||
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
|
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -844,7 +842,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
|
||||||
}
|
}
|
||||||
|
|
||||||
if sb.osSbox == nil && !sb.config.useExternalKey {
|
if sb.osSbox == nil && !sb.config.useExternalKey {
|
||||||
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
|
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
|
||||||
return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
|
return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ncfg *networkConfiguration) Skip() bool {
|
func (ncfg *networkConfiguration) Skip() bool {
|
||||||
return ncfg.DefaultBridge
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ncfg *networkConfiguration) New() datastore.KVObject {
|
func (ncfg *networkConfiguration) New() datastore.KVObject {
|
||||||
|
|
|
@ -512,7 +512,7 @@ func (n *network) initSandbox() error {
|
||||||
n.cleanupStaleSandboxes()
|
n.cleanupStaleSandboxes()
|
||||||
|
|
||||||
sbox, err := osl.NewSandbox(
|
sbox, err := osl.NewSandbox(
|
||||||
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
|
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not create network sandbox: %v", err)
|
return fmt.Errorf("could not create network sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
|
||||||
epMap["name"] = ep.name
|
epMap["name"] = ep.name
|
||||||
epMap["id"] = ep.id
|
epMap["id"] = ep.id
|
||||||
epMap["ep_iface"] = ep.iface
|
epMap["ep_iface"] = ep.iface
|
||||||
|
epMap["joinInfo"] = ep.joinInfo
|
||||||
epMap["exposed_ports"] = ep.exposedPorts
|
epMap["exposed_ports"] = ep.exposedPorts
|
||||||
if ep.generic != nil {
|
if ep.generic != nil {
|
||||||
epMap["generic"] = ep.generic
|
epMap["generic"] = ep.generic
|
||||||
|
@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
|
||||||
ib, _ := json.Marshal(epMap["ep_iface"])
|
ib, _ := json.Marshal(epMap["ep_iface"])
|
||||||
json.Unmarshal(ib, &ep.iface)
|
json.Unmarshal(ib, &ep.iface)
|
||||||
|
|
||||||
|
jb, _ := json.Marshal(epMap["joinInfo"])
|
||||||
|
json.Unmarshal(jb, &ep.joinInfo)
|
||||||
|
|
||||||
tb, _ := json.Marshal(epMap["exposed_ports"])
|
tb, _ := json.Marshal(epMap["exposed_ports"])
|
||||||
var tPorts []types.TransportPort
|
var tPorts []types.TransportPort
|
||||||
json.Unmarshal(tb, &tPorts)
|
json.Unmarshal(tb, &tPorts)
|
||||||
|
@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
|
||||||
ep.iface.CopyTo(dstEp.iface)
|
ep.iface.CopyTo(dstEp.iface)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ep.joinInfo != nil {
|
||||||
|
dstEp.joinInfo = &endpointJoinInfo{}
|
||||||
|
ep.joinInfo.CopyTo(dstEp.joinInfo)
|
||||||
|
}
|
||||||
|
|
||||||
dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
|
dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
|
||||||
copy(dstEp.exposedPorts, ep.exposedPorts)
|
copy(dstEp.exposedPorts, ep.exposedPorts)
|
||||||
|
|
||||||
|
@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controller) cleanupLocalEndpoints() {
|
func (c *controller) cleanupLocalEndpoints() {
|
||||||
|
// Get used endpoints
|
||||||
|
eps := make(map[string]interface{})
|
||||||
|
for _, sb := range c.sandboxes {
|
||||||
|
for _, ep := range sb.endpoints {
|
||||||
|
eps[ep.id] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
nl, err := c.getNetworksForScope(datastore.LocalScope)
|
nl, err := c.getNetworksForScope(datastore.LocalScope)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
|
log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
|
||||||
|
@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ep := range epl {
|
for _, ep := range epl {
|
||||||
|
if _, ok := eps[ep.id]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
|
log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
|
||||||
if err := ep.Delete(true); err != nil {
|
if err := ep.Delete(true); err != nil {
|
||||||
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
|
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
|
||||||
|
|
|
@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
|
||||||
|
|
||||||
ep.joinInfo.disableGatewayService = true
|
ep.joinInfo.disableGatewayService = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
|
||||||
|
epMap := make(map[string]interface{})
|
||||||
|
if epj.gw != nil {
|
||||||
|
epMap["gw"] = epj.gw.String()
|
||||||
|
}
|
||||||
|
if epj.gw6 != nil {
|
||||||
|
epMap["gw6"] = epj.gw6.String()
|
||||||
|
}
|
||||||
|
epMap["disableGatewayService"] = epj.disableGatewayService
|
||||||
|
epMap["StaticRoutes"] = epj.StaticRoutes
|
||||||
|
return json.Marshal(epMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
|
||||||
|
var (
|
||||||
|
err error
|
||||||
|
epMap map[string]interface{}
|
||||||
|
)
|
||||||
|
if err = json.Unmarshal(b, &epMap); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if v, ok := epMap["gw"]; ok {
|
||||||
|
epj.gw6 = net.ParseIP(v.(string))
|
||||||
|
}
|
||||||
|
if v, ok := epMap["gw6"]; ok {
|
||||||
|
epj.gw6 = net.ParseIP(v.(string))
|
||||||
|
}
|
||||||
|
epj.disableGatewayService = epMap["disableGatewayService"].(bool)
|
||||||
|
|
||||||
|
var tStaticRoute []types.StaticRoute
|
||||||
|
if v, ok := epMap["StaticRoutes"]; ok {
|
||||||
|
tb, _ := json.Marshal(v)
|
||||||
|
var tStaticRoute []types.StaticRoute
|
||||||
|
json.Unmarshal(tb, &tStaticRoute)
|
||||||
|
}
|
||||||
|
var StaticRoutes []*types.StaticRoute
|
||||||
|
for _, r := range tStaticRoute {
|
||||||
|
StaticRoutes = append(StaticRoutes, &r)
|
||||||
|
}
|
||||||
|
epj.StaticRoutes = StaticRoutes
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
|
||||||
|
dstEpj.disableGatewayService = epj.disableGatewayService
|
||||||
|
dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
|
||||||
|
copy(dstEpj.StaticRoutes, epj.StaticRoutes)
|
||||||
|
dstEpj.gw = types.GetIPCopy(epj.gw)
|
||||||
|
dstEpj.gw = types.GetIPCopy(epj.gw6)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create a new OS sandbox using the osl API before using it in SetKey
|
// Create a new OS sandbox using the osl API before using it in SetKey
|
||||||
if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil {
|
if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil {
|
||||||
t.Fatalf("Failed to create new osl sandbox")
|
t.Fatalf("Failed to create new osl sandbox")
|
||||||
} else {
|
} else {
|
||||||
defer func() {
|
defer func() {
|
||||||
|
|
|
@ -2,10 +2,13 @@ package osl
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
@ -133,6 +136,39 @@ func GC() {
|
||||||
// container id.
|
// container id.
|
||||||
func GenerateKey(containerID string) string {
|
func GenerateKey(containerID string) string {
|
||||||
maxLen := 12
|
maxLen := 12
|
||||||
|
// Read sandbox key from host for overlay
|
||||||
|
if strings.HasPrefix(containerID, "-") {
|
||||||
|
var (
|
||||||
|
index int
|
||||||
|
indexStr string
|
||||||
|
tmpkey string
|
||||||
|
)
|
||||||
|
dir, err := ioutil.ReadDir(prefix)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range dir {
|
||||||
|
id := v.Name()
|
||||||
|
if strings.HasSuffix(id, containerID[:maxLen-1]) {
|
||||||
|
indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
|
||||||
|
tmpindex, err := strconv.Atoi(indexStr)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if tmpindex > index {
|
||||||
|
index = tmpindex
|
||||||
|
tmpkey = id
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
containerID = tmpkey
|
||||||
|
if containerID == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if len(containerID) < maxLen {
|
if len(containerID) < maxLen {
|
||||||
maxLen = len(containerID)
|
maxLen = len(containerID)
|
||||||
}
|
}
|
||||||
|
@ -142,11 +178,13 @@ func GenerateKey(containerID string) string {
|
||||||
|
|
||||||
// NewSandbox provides a new sandbox instance created in an os specific way
|
// NewSandbox provides a new sandbox instance created in an os specific way
|
||||||
// provided a key which uniquely identifies the sandbox
|
// provided a key which uniquely identifies the sandbox
|
||||||
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
|
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
|
||||||
|
if !isRestore {
|
||||||
err := createNetworkNamespace(key, osCreate)
|
err := createNetworkNamespace(key, osCreate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
n := &networkNamespace{path: key, isDefault: !osCreate}
|
n := &networkNamespace{path: key, isDefault: !osCreate}
|
||||||
|
|
||||||
|
@ -347,3 +385,108 @@ func (n *networkNamespace) Destroy() error {
|
||||||
addToGarbagePaths(n.path)
|
addToGarbagePaths(n.path)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Restore restore the network namespace
|
||||||
|
func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
|
||||||
|
// restore interfaces
|
||||||
|
for name, opts := range ifsopt {
|
||||||
|
if !strings.Contains(name, "+") {
|
||||||
|
return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
|
||||||
|
}
|
||||||
|
seps := strings.Split(name, "+")
|
||||||
|
srcName := seps[0]
|
||||||
|
dstPrefix := seps[1]
|
||||||
|
i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
|
||||||
|
i.processInterfaceOptions(opts...)
|
||||||
|
if i.master != "" {
|
||||||
|
i.dstMaster = n.findDst(i.master, true)
|
||||||
|
if i.dstMaster == "" {
|
||||||
|
return fmt.Errorf("could not find an appropriate master %q for %q",
|
||||||
|
i.master, i.srcName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n.isDefault {
|
||||||
|
i.dstName = i.srcName
|
||||||
|
} else {
|
||||||
|
// due to the docker network connect/disconnect, so the dstName should
|
||||||
|
// restore from the namespace
|
||||||
|
err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error {
|
||||||
|
ifaces, err := net.Interfaces()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, iface := range ifaces {
|
||||||
|
addrs, err := iface.Addrs()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(iface.Name, "vxlan") {
|
||||||
|
if i.dstName == "vxlan" {
|
||||||
|
i.dstName = iface.Name
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// find the interface name by ip
|
||||||
|
if i.address != nil {
|
||||||
|
for _, addr := range addrs {
|
||||||
|
if addr.String() == i.address.String() {
|
||||||
|
i.dstName = iface.Name
|
||||||
|
break
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i.dstName == iface.Name {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// This is to find the interface name of the pair in overlay sandbox
|
||||||
|
if strings.HasPrefix(iface.Name, "veth") {
|
||||||
|
if i.master != "" && i.dstName == "veth" {
|
||||||
|
i.dstName = iface.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var index int
|
||||||
|
indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
|
||||||
|
if indexStr != "" {
|
||||||
|
index, err = strconv.Atoi(indexStr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index++
|
||||||
|
n.Lock()
|
||||||
|
if index > n.nextIfIndex {
|
||||||
|
n.nextIfIndex = index
|
||||||
|
}
|
||||||
|
n.iFaces = append(n.iFaces, i)
|
||||||
|
n.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// restore routes
|
||||||
|
for _, r := range routes {
|
||||||
|
n.Lock()
|
||||||
|
n.staticRoutes = append(n.staticRoutes, r)
|
||||||
|
n.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// restore gateway
|
||||||
|
if len(gw) > 0 {
|
||||||
|
n.Lock()
|
||||||
|
n.gw = gw
|
||||||
|
n.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(gw6) > 0 {
|
||||||
|
n.Lock()
|
||||||
|
n.gwv6 = gw6
|
||||||
|
n.Unlock()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
|
||||||
|
|
||||||
// NewSandbox provides a new sandbox instance created in an os specific way
|
// NewSandbox provides a new sandbox instance created in an os specific way
|
||||||
// provided a key which uniquely identifies the sandbox
|
// provided a key which uniquely identifies the sandbox
|
||||||
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
|
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -58,6 +58,9 @@ type Sandbox interface {
|
||||||
|
|
||||||
// Destroy the sandbox
|
// Destroy the sandbox
|
||||||
Destroy() error
|
Destroy() error
|
||||||
|
|
||||||
|
// restore sandbox
|
||||||
|
Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// NeighborOptionSetter interface defines the option setter methods for interface options
|
// NeighborOptionSetter interface defines the option setter methods for interface options
|
||||||
|
|
|
@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
|
||||||
|
|
||||||
// NewSandbox provides a new sandbox instance created in an os specific way
|
// NewSandbox provides a new sandbox instance created in an os specific way
|
||||||
// provided a key which uniquely identifies the sandbox
|
// provided a key which uniquely identifies the sandbox
|
||||||
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
|
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) {
|
||||||
t.Fatalf("Failed to obtain a key: %v", err)
|
t.Fatalf("Failed to obtain a key: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s, err := NewSandbox(key, true)
|
s, err := NewSandbox(key, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create a new sandbox: %v", err)
|
t.Fatalf("Failed to create a new sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) {
|
||||||
t.Fatalf("Failed to obtain a key: %v", err)
|
t.Fatalf("Failed to obtain a key: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = NewSandbox(key, true)
|
_, err = NewSandbox(key, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create a new sandbox: %v", err)
|
t.Fatalf("Failed to create a new sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) {
|
||||||
|
|
||||||
// Create another sandbox with the same key to see if we handle it
|
// Create another sandbox with the same key to see if we handle it
|
||||||
// gracefully.
|
// gracefully.
|
||||||
s, err := NewSandbox(key, true)
|
s, err := NewSandbox(key, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create a new sandbox: %v", err)
|
t.Fatalf("Failed to create a new sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) {
|
||||||
t.Fatalf("Failed to obtain a key: %v", err)
|
t.Fatalf("Failed to obtain a key: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s, err := NewSandbox(key, true)
|
s, err := NewSandbox(key, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create a new sandbox: %v", err)
|
t.Fatalf("Failed to create a new sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) {
|
||||||
t.Fatalf("Failed to obtain a key: %v", err)
|
t.Fatalf("Failed to obtain a key: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s, err := NewSandbox(key, true)
|
s, err := NewSandbox(key, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create a new sandbox: %v", err)
|
t.Fatalf("Failed to create a new sandbox: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,7 +11,7 @@ var (
|
||||||
|
|
||||||
// NewSandbox provides a new sandbox instance created in an os specific way
|
// NewSandbox provides a new sandbox instance created in an os specific way
|
||||||
// provided a key which uniquely identifies the sandbox
|
// provided a key which uniquely identifies the sandbox
|
||||||
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
|
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
|
||||||
return nil, ErrNotImplemented
|
return nil, ErrNotImplemented
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,13 @@ func init() {
|
||||||
reexec.Register("setup-resolver", reexecSetupResolver)
|
reexec.Register("setup-resolver", reexecSetupResolver)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// outputChain used for docker embed dns
|
||||||
|
outputChain = "DOCKER_OUTPUT"
|
||||||
|
//postroutingchain used for docker embed dns
|
||||||
|
postroutingchain = "DOCKER_POSTROUTING"
|
||||||
|
)
|
||||||
|
|
||||||
func reexecSetupResolver() {
|
func reexecSetupResolver() {
|
||||||
runtime.LockOSThread()
|
runtime.LockOSThread()
|
||||||
defer runtime.UnlockOSThread()
|
defer runtime.UnlockOSThread()
|
||||||
|
@ -31,10 +38,10 @@ func reexecSetupResolver() {
|
||||||
_, ipPort, _ := net.SplitHostPort(os.Args[2])
|
_, ipPort, _ := net.SplitHostPort(os.Args[2])
|
||||||
_, tcpPort, _ := net.SplitHostPort(os.Args[3])
|
_, tcpPort, _ := net.SplitHostPort(os.Args[3])
|
||||||
rules := [][]string{
|
rules := [][]string{
|
||||||
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
|
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
|
||||||
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
|
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
|
||||||
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
|
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
|
||||||
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
|
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
|
f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
|
||||||
|
@ -50,6 +57,23 @@ func reexecSetupResolver() {
|
||||||
os.Exit(3)
|
os.Exit(3)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// insert outputChain and postroutingchain
|
||||||
|
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
|
||||||
|
if err == nil {
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
|
||||||
|
} else {
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
|
||||||
|
if err == nil {
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
|
||||||
|
} else {
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
|
||||||
|
iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
|
||||||
|
}
|
||||||
|
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
if iptables.RawCombinedOutputNative(rule...) != nil {
|
if iptables.RawCombinedOutputNative(rule...) != nil {
|
||||||
log.Errorf("setting up rule failed, %v", rule)
|
log.Errorf("setting up rule failed, %v", rule)
|
||||||
|
|
|
@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
|
||||||
osSbox.Destroy()
|
osSbox.Destroy()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (sb *sandbox) restoreOslSandbox() error {
|
||||||
|
var routes []*types.StaticRoute
|
||||||
|
|
||||||
|
// restore osl sandbox
|
||||||
|
Ifaces := make(map[string][]osl.IfaceOption)
|
||||||
|
for _, ep := range sb.endpoints {
|
||||||
|
var ifaceOptions []osl.IfaceOption
|
||||||
|
ep.Lock()
|
||||||
|
joinInfo := ep.joinInfo
|
||||||
|
i := ep.iface
|
||||||
|
ep.Unlock()
|
||||||
|
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
|
||||||
|
if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
|
||||||
|
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
|
||||||
|
}
|
||||||
|
if i.mac != nil {
|
||||||
|
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
|
||||||
|
}
|
||||||
|
if len(i.llAddrs) != 0 {
|
||||||
|
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
|
||||||
|
}
|
||||||
|
Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
|
||||||
|
if joinInfo != nil {
|
||||||
|
for _, r := range joinInfo.StaticRoutes {
|
||||||
|
routes = append(routes, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ep.needResolver() {
|
||||||
|
sb.startResolver()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gwep := sb.getGatewayEndpoint()
|
||||||
|
if gwep == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// restore osl sandbox
|
||||||
|
err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
|
func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
|
||||||
sb.Lock()
|
sb.Lock()
|
||||||
if sb.osSbox == nil {
|
if sb.osSbox == nil {
|
||||||
|
|
|
@ -139,6 +139,17 @@ func (sb *sandbox) updateParentHosts() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (sb *sandbox) restorePath() {
|
||||||
|
if sb.config.resolvConfPath == "" {
|
||||||
|
sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
|
||||||
|
}
|
||||||
|
sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
|
||||||
|
if sb.config.hostsPath == "" {
|
||||||
|
sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func (sb *sandbox) setupDNS() error {
|
func (sb *sandbox) setupDNS() error {
|
||||||
var newRC *resolvconf.File
|
var newRC *resolvconf.File
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (sb *sandbox) restorePath() {
|
||||||
|
}
|
||||||
|
|
||||||
func (sb *sandbox) updateHostsFile(ifaceIP string) error {
|
func (sb *sandbox) updateHostsFile(ifaceIP string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,6 +26,7 @@ type sbState struct {
|
||||||
dbIndex uint64
|
dbIndex uint64
|
||||||
dbExists bool
|
dbExists bool
|
||||||
Eps []epState
|
Eps []epState
|
||||||
|
EpPriority map[string]int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sbs *sbState) Key() []string {
|
func (sbs *sbState) Key() []string {
|
||||||
|
@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
|
||||||
dstSbs.Cid = sbs.Cid
|
dstSbs.Cid = sbs.Cid
|
||||||
dstSbs.dbIndex = sbs.dbIndex
|
dstSbs.dbIndex = sbs.dbIndex
|
||||||
dstSbs.dbExists = sbs.dbExists
|
dstSbs.dbExists = sbs.dbExists
|
||||||
|
dstSbs.EpPriority = sbs.EpPriority
|
||||||
|
|
||||||
for _, eps := range sbs.Eps {
|
for _, eps := range sbs.Eps {
|
||||||
dstSbs.Eps = append(dstSbs.Eps, eps)
|
dstSbs.Eps = append(dstSbs.Eps, eps)
|
||||||
|
@ -123,6 +125,7 @@ func (sb *sandbox) storeUpdate() error {
|
||||||
c: sb.controller,
|
c: sb.controller,
|
||||||
ID: sb.id,
|
ID: sb.id,
|
||||||
Cid: sb.containerID,
|
Cid: sb.containerID,
|
||||||
|
EpPriority: sb.epPriority,
|
||||||
}
|
}
|
||||||
|
|
||||||
retry:
|
retry:
|
||||||
|
@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
|
||||||
return sb.controller.deleteFromStore(sbs)
|
return sb.controller.deleteFromStore(sbs)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controller) sandboxCleanup() {
|
func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
|
||||||
store := c.getStore(datastore.LocalScope)
|
store := c.getStore(datastore.LocalScope)
|
||||||
if store == nil {
|
if store == nil {
|
||||||
logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
|
logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
|
||||||
|
@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
|
||||||
controller: sbs.c,
|
controller: sbs.c,
|
||||||
containerID: sbs.Cid,
|
containerID: sbs.Cid,
|
||||||
endpoints: epHeap{},
|
endpoints: epHeap{},
|
||||||
epPriority: map[string]int{},
|
|
||||||
dbIndex: sbs.dbIndex,
|
dbIndex: sbs.dbIndex,
|
||||||
isStub: true,
|
isStub: true,
|
||||||
dbExists: true,
|
dbExists: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
|
msg := " for cleanup"
|
||||||
|
create := true
|
||||||
|
isRestore := false
|
||||||
|
if val, ok := activeSandboxes[sb.ID()]; ok {
|
||||||
|
msg = ""
|
||||||
|
sb.isStub = false
|
||||||
|
isRestore = true
|
||||||
|
opts := val.([]SandboxOption)
|
||||||
|
sb.processOptions(opts...)
|
||||||
|
sb.restorePath()
|
||||||
|
create = !sb.config.useDefaultSandBox
|
||||||
|
heap.Init(&sb.endpoints)
|
||||||
|
}
|
||||||
|
sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
|
logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
|
||||||
ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
|
ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
heap.Push(&sb.endpoints, ep)
|
heap.Push(&sb.endpoints, ep)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if _, ok := activeSandboxes[sb.ID()]; !ok {
|
||||||
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
|
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
|
||||||
if err := sb.delete(true); err != nil {
|
if err := sb.delete(true); err != nil {
|
||||||
logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
|
logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconstruct osl sandbox field
|
||||||
|
if !sb.config.useDefaultSandBox {
|
||||||
|
if err := sb.restoreOslSandbox(); err != nil {
|
||||||
|
logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
c.sboxOnce.Do(func() {
|
||||||
|
c.defOsSbox = sb.osSbox
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ep := range sb.endpoints {
|
||||||
|
// Watch for service records
|
||||||
|
if !c.isAgent() {
|
||||||
|
c.watchSvcRecord(ep)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue