From c08f5b2b8460f13f2094bae2a496bf308f7645bb Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 25 Feb 2013 14:06:22 -0800 Subject: [PATCH] Integrated the network allocator into Docker. A networking environment is assigned to each container upon Start and released whenever the container exits. --- container.go | 75 ++++++++++++++++++++++++++++++++-------------- docker.go | 22 +++++++++----- lxc_template.go | 2 +- network.go | 79 +++++++++++++++++++++++-------------------------- network_test.go | 10 +++---- 5 files changed, 110 insertions(+), 78 deletions(-) diff --git a/container.go b/container.go index 50ac5e715e..67c450282f 100644 --- a/container.go +++ b/container.go @@ -33,9 +33,12 @@ type Container struct { Config *Config Filesystem *Filesystem - Network *NetworkInterface State *State + network *NetworkInterface + networkAllocator *NetworkAllocator + NetworkConfig *NetworkConfig + SysInitPath string lxcConfigPath string cmd *exec.Cmd @@ -56,16 +59,23 @@ type Config struct { OpenStdin bool // Open stdin } -func createContainer(id string, root string, command string, args []string, layers []string, config *Config) (*Container, error) { +type NetworkConfig struct { + IpAddress string + IpPrefixLen int +} + +func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netAllocator *NetworkAllocator) (*Container, error) { container := &Container{ - Id: id, - Root: root, - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), - State: newState(), + Id: id, + Root: root, + Created: time.Now(), + Path: command, + Args: args, + Config: config, + Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), + State: newState(), + networkAllocator: netAllocator, + NetworkConfig: &NetworkConfig{}, SysInitPath: sysInitPath, lxcConfigPath: path.Join(root, "config.lxc"), @@ -88,27 +98,25 @@ func createContainer(id string, root string, command string, args []string, laye if err := container.Filesystem.createMountPoints(); err != nil { return nil, err } - var err error - if container.Network, err = allocateNetwork(); err != nil { - return nil, err - } if err := container.save(); err != nil { return nil, err } return container, nil } -func loadContainer(containerPath string) (*Container, error) { +func loadContainer(containerPath string, netAllocator *NetworkAllocator) (*Container, error) { data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) if err != nil { return nil, err } container := &Container{ - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), - lxcConfigPath: path.Join(containerPath, "config.lxc"), + stdout: newWriteBroadcaster(), + stderr: newWriteBroadcaster(), + stdoutLog: new(bytes.Buffer), + stderrLog: new(bytes.Buffer), + lxcConfigPath: path.Join(containerPath, "config.lxc"), + networkAllocator: netAllocator, + NetworkConfig: &NetworkConfig{}, } if err := json.Unmarshal(data, container); err != nil { return nil, err @@ -268,6 +276,9 @@ func (container *Container) Start() error { if err := container.Filesystem.EnsureMounted(); err != nil { return err } + if err := container.allocateNetwork(); err != nil { + return err + } if err := container.generateLXCConfig(); err != nil { return err } @@ -279,7 +290,7 @@ func (container *Container) Start() error { } // Networking - params = append(params, "-g", container.Network.Gateway.String()) + params = append(params, "-g", container.network.Gateway.String()) // User if container.Config.User != "" { @@ -356,12 +367,33 @@ func (container *Container) StderrLog() io.Reader { return strings.NewReader(container.stderrLog.String()) } +func (container *Container) allocateNetwork() error { + iface, err := container.networkAllocator.Allocate() + if err != nil { + return err + } + container.network = iface + container.NetworkConfig.IpAddress = iface.IPNet.IP.String() + container.NetworkConfig.IpPrefixLen, _ = iface.IPNet.Mask.Size() + return nil +} + +func (container *Container) releaseNetwork() error { + err := container.networkAllocator.Release(container.network) + container.network = nil + container.NetworkConfig = &NetworkConfig{} + return err +} + func (container *Container) monitor() { // Wait for the program to exit container.cmd.Wait() exitCode := container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() // Cleanup + if err := container.releaseNetwork(); err != nil { + log.Printf("%v: Failed to release network: %v", container.Id, err) + } container.stdout.Close() container.stderr.Close() if err := container.Filesystem.Umount(); err != nil { @@ -429,7 +461,6 @@ func (container *Container) Restart() error { } func (container *Container) Wait() { - for container.State.Running { container.State.wait() } diff --git a/docker.go b/docker.go index abe2d3777d..f44eb04059 100644 --- a/docker.go +++ b/docker.go @@ -11,9 +11,10 @@ import ( ) type Docker struct { - root string - repository string - containers *list.List + root string + repository string + containers *list.List + networkAllocator *NetworkAllocator } func (docker *Docker) List() []*Container { @@ -51,7 +52,7 @@ func (docker *Docker) Create(id string, command string, args []string, layers [] return nil, fmt.Errorf("Container %v already exists", id) } root := path.Join(docker.repository, id) - container, err := createContainer(id, root, command, args, layers, config) + container, err := createContainer(id, root, command, args, layers, config, docker.networkAllocator) if err != nil { return nil, err } @@ -86,7 +87,7 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(path.Join(docker.repository, v.Name())) + container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkAllocator) if err != nil { log.Printf("Failed to load container %v: %v", v.Name(), err) continue @@ -101,10 +102,15 @@ func New() (*Docker, error) { } func NewFromDirectory(root string) (*Docker, error) { + alloc, err := newNetworkAllocator(networkBridgeIface) + if err != nil { + return nil, err + } docker := &Docker{ - root: root, - repository: path.Join(root, "containers"), - containers: list.New(), + root: root, + repository: path.Join(root, "containers"), + containers: list.New(), + networkAllocator: alloc, } if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) { diff --git a/lxc_template.go b/lxc_template.go index 48e6dc732b..3e66885460 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -19,7 +19,7 @@ lxc.network.flags = up lxc.network.link = lxcbr0 lxc.network.name = eth0 lxc.network.mtu = 1500 -lxc.network.ipv4 = {{.Network.IpAddress}}/{{.Network.IpPrefixLen}} +lxc.network.ipv4 = {{.NetworkConfig.IpAddress}}/{{.NetworkConfig.IpPrefixLen}} # root filesystem {{$ROOTFS := .Filesystem.RootFS}} diff --git a/network.go b/network.go index 977db53c90..db4048feb5 100644 --- a/network.go +++ b/network.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "math/rand" "net" ) @@ -14,11 +13,12 @@ const ( ) type NetworkInterface struct { - IpAddress string - IpPrefixLen int - Gateway net.IP + IPNet net.IPNet + Gateway net.IP } +// IP utils + func networkRange(network *net.IPNet) (net.IP, net.IP) { netIP := network.IP.To4() firstIP := netIP.Mask(network.Mask) @@ -51,10 +51,11 @@ func intToIp(n int32) (net.IP, error) { } func networkSize(mask net.IPMask) (int32, error) { + m := net.IPv4Mask(0, 0, 0, 0) for i := 0; i < net.IPv4len; i++ { - mask[i] = ^mask[i] + m[i] = ^mask[i] } - buf := bytes.NewBuffer(mask) + buf := bytes.NewBuffer(m) var n int32 if err := binary.Read(buf, binary.BigEndian, &n); err != nil { return 0, err @@ -62,21 +63,7 @@ func networkSize(mask net.IPMask) (int32, error) { return n + 1, nil } -func allocateIPAddress(network *net.IPNet) (net.IP, error) { - ip, _ := networkRange(network) - netSize, err := networkSize(network.Mask) - if err != nil { - return net.IP{}, err - } - numIp, err := ipToInt(ip) - if err != nil { - return net.IP{}, err - } - numIp += rand.Int31n(netSize) - return intToIp(numIp) -} - -func getBridgeAddr(name string) (net.Addr, error) { +func getIfaceAddr(name string) (net.Addr, error) { iface, err := net.InterfaceByName(name) if err != nil { return nil, err @@ -101,31 +88,31 @@ func getBridgeAddr(name string) (net.Addr, error) { return addrs4[0], nil } -func allocateNetwork() (*NetworkInterface, error) { - bridgeAddr, err := getBridgeAddr(networkBridgeIface) +// Network allocator +func newNetworkAllocator(iface string) (*NetworkAllocator, error) { + addr, err := getIfaceAddr(iface) if err != nil { return nil, err } - bridge := bridgeAddr.(*net.IPNet) - ipPrefixLen, _ := bridge.Mask.Size() - ip, err := allocateIPAddress(bridge) - if err != nil { + network := addr.(*net.IPNet) + + alloc := &NetworkAllocator{ + iface: iface, + net: network, + } + if err := alloc.populateFromNetwork(network); err != nil { return nil, err } - iface := &NetworkInterface{ - IpAddress: ip.String(), - IpPrefixLen: ipPrefixLen, - Gateway: bridge.IP, - } - return iface, nil + return alloc, nil } type NetworkAllocator struct { iface string + net *net.IPNet queue chan (net.IP) } -func (alloc *NetworkAllocator) Acquire() (net.IP, error) { +func (alloc *NetworkAllocator) acquireIP() (net.IP, error) { select { case ip := <-alloc.queue: return ip, nil @@ -135,7 +122,7 @@ func (alloc *NetworkAllocator) Acquire() (net.IP, error) { return net.IP{}, nil } -func (alloc *NetworkAllocator) Release(ip net.IP) error { +func (alloc *NetworkAllocator) releaseIP(ip net.IP) error { select { case alloc.queue <- ip: return nil @@ -145,7 +132,7 @@ func (alloc *NetworkAllocator) Release(ip net.IP) error { return nil } -func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error { +func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { firstIP, _ := networkRange(network) size, err := networkSize(network.Mask) if err != nil { @@ -168,16 +155,24 @@ func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error { if ip.Equal(network.IP) { continue } - alloc.Release(ip) + alloc.releaseIP(ip) } return nil } -func (alloc *NetworkAllocator) PopulateFromInterface(iface string) error { - addr, err := getBridgeAddr(iface) +func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) { + // ipPrefixLen, _ := alloc.net.Mask.Size() + ip, err := alloc.acquireIP() if err != nil { - return err + return nil, err } - network := addr.(*net.IPNet) - return alloc.PopulateFromNetwork(network) + iface := &NetworkInterface{ + IPNet: net.IPNet{ip, alloc.net.Mask}, + Gateway: alloc.net.IP, + } + return iface, nil +} + +func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error { + return alloc.releaseIP(iface.IPNet.IP) } diff --git a/network_test.go b/network_test.go index c3de398681..d66e7e5393 100644 --- a/network_test.go +++ b/network_test.go @@ -103,22 +103,22 @@ func TestConversion(t *testing.T) { func TestNetworkAllocator(t *testing.T) { alloc := NetworkAllocator{} _, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc.PopulateFromNetwork(n) + alloc.populateFromNetwork(n) var lastIP net.IP for i := 0; i < 5; i++ { - ip, err := alloc.Acquire() + ip, err := alloc.acquireIP() if err != nil { t.Fatal(err) } lastIP = ip } - ip, err := alloc.Acquire() + ip, err := alloc.acquireIP() if err == nil { t.Fatal("There shouldn't be any IP addresses at this point") } // Release 1 IP - alloc.Release(lastIP) - ip, err = alloc.Acquire() + alloc.releaseIP(lastIP) + ip, err = alloc.acquireIP() if err != nil { t.Fatal(err) }