374 lines
10 KiB
Go
374 lines
10 KiB
Go
// +build windows
|
|
|
|
package lcow
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Microsoft/hcsshim"
|
|
"github.com/Microsoft/opengcs/client"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// Code for all the service VM management for the LCOW graphdriver
|
|
|
|
var errVMisTerminating = errors.New("service VM is shutting down")
|
|
var errVMUnknown = errors.New("service vm id is unknown")
|
|
var errVMStillHasReference = errors.New("Attemping to delete a VM that is still being used")
|
|
|
|
// serviceVMMap is the struct representing the id -> service VM mapping.
|
|
type serviceVMMap struct {
|
|
sync.Mutex
|
|
svms map[string]*serviceVMMapItem
|
|
}
|
|
|
|
// serviceVMMapItem is our internal structure representing an item in our
|
|
// map of service VMs we are maintaining.
|
|
type serviceVMMapItem struct {
|
|
svm *serviceVM // actual service vm object
|
|
refCount int // refcount for VM
|
|
}
|
|
|
|
type serviceVM struct {
|
|
sync.Mutex // Serialises operations being performed in this service VM.
|
|
scratchAttached bool // Has a scratch been attached?
|
|
config *client.Config // Represents the service VM item.
|
|
|
|
// Indicates that the vm is started
|
|
startStatus chan interface{}
|
|
startError error
|
|
|
|
// Indicates that the vm is stopped
|
|
stopStatus chan interface{}
|
|
stopError error
|
|
|
|
attachedVHDs map[string]int // Map ref counting all the VHDS we've hot-added/hot-removed.
|
|
unionMounts map[string]int // Map ref counting all the union filesystems we mounted.
|
|
}
|
|
|
|
// add will add an id to the service vm map. There are three cases:
|
|
// - entry doesn't exist:
|
|
// - add id to map and return a new vm that the caller can manually configure+start
|
|
// - entry does exist
|
|
// - return vm in map and increment ref count
|
|
// - entry does exist but the ref count is 0
|
|
// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
|
|
func (svmMap *serviceVMMap) add(id string) (svm *serviceVM, alreadyExists bool, err error) {
|
|
svmMap.Lock()
|
|
defer svmMap.Unlock()
|
|
if svm, ok := svmMap.svms[id]; ok {
|
|
if svm.refCount == 0 {
|
|
return svm.svm, true, errVMisTerminating
|
|
}
|
|
svm.refCount++
|
|
return svm.svm, true, nil
|
|
}
|
|
|
|
// Doesn't exist, so create an empty svm to put into map and return
|
|
newSVM := &serviceVM{
|
|
startStatus: make(chan interface{}),
|
|
stopStatus: make(chan interface{}),
|
|
attachedVHDs: make(map[string]int),
|
|
unionMounts: make(map[string]int),
|
|
config: &client.Config{},
|
|
}
|
|
svmMap.svms[id] = &serviceVMMapItem{
|
|
svm: newSVM,
|
|
refCount: 1,
|
|
}
|
|
return newSVM, false, nil
|
|
}
|
|
|
|
// get will get the service vm from the map. There are three cases:
|
|
// - entry doesn't exist:
|
|
// - return errVMUnknown
|
|
// - entry does exist
|
|
// - return vm with no error
|
|
// - entry does exist but the ref count is 0
|
|
// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
|
|
func (svmMap *serviceVMMap) get(id string) (*serviceVM, error) {
|
|
svmMap.Lock()
|
|
defer svmMap.Unlock()
|
|
svm, ok := svmMap.svms[id]
|
|
if !ok {
|
|
return nil, errVMUnknown
|
|
}
|
|
if svm.refCount == 0 {
|
|
return svm.svm, errVMisTerminating
|
|
}
|
|
return svm.svm, nil
|
|
}
|
|
|
|
// decrementRefCount decrements the ref count of the given ID from the map. There are four cases:
|
|
// - entry doesn't exist:
|
|
// - return errVMUnknown
|
|
// - entry does exist but the ref count is 0
|
|
// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
|
|
// - entry does exist but ref count is 1
|
|
// - return vm and set lastRef to true. The caller can then stop the vm, delete the id from this map
|
|
// - and execute svm.signalStopFinished to signal the threads that the svm has been terminated.
|
|
// - entry does exist and ref count > 1
|
|
// - just reduce ref count and return svm
|
|
func (svmMap *serviceVMMap) decrementRefCount(id string) (_ *serviceVM, lastRef bool, _ error) {
|
|
svmMap.Lock()
|
|
defer svmMap.Unlock()
|
|
|
|
svm, ok := svmMap.svms[id]
|
|
if !ok {
|
|
return nil, false, errVMUnknown
|
|
}
|
|
if svm.refCount == 0 {
|
|
return svm.svm, false, errVMisTerminating
|
|
}
|
|
svm.refCount--
|
|
return svm.svm, svm.refCount == 0, nil
|
|
}
|
|
|
|
// setRefCountZero works the same way as decrementRefCount, but sets ref count to 0 instead of decrementing it.
|
|
func (svmMap *serviceVMMap) setRefCountZero(id string) (*serviceVM, error) {
|
|
svmMap.Lock()
|
|
defer svmMap.Unlock()
|
|
|
|
svm, ok := svmMap.svms[id]
|
|
if !ok {
|
|
return nil, errVMUnknown
|
|
}
|
|
if svm.refCount == 0 {
|
|
return svm.svm, errVMisTerminating
|
|
}
|
|
svm.refCount = 0
|
|
return svm.svm, nil
|
|
}
|
|
|
|
// deleteID deletes the given ID from the map. If the refcount is not 0 or the
|
|
// VM does not exist, then this function returns an error.
|
|
func (svmMap *serviceVMMap) deleteID(id string) error {
|
|
svmMap.Lock()
|
|
defer svmMap.Unlock()
|
|
svm, ok := svmMap.svms[id]
|
|
if !ok {
|
|
return errVMUnknown
|
|
}
|
|
if svm.refCount != 0 {
|
|
return errVMStillHasReference
|
|
}
|
|
delete(svmMap.svms, id)
|
|
return nil
|
|
}
|
|
|
|
func (svm *serviceVM) signalStartFinished(err error) {
|
|
svm.Lock()
|
|
svm.startError = err
|
|
svm.Unlock()
|
|
close(svm.startStatus)
|
|
}
|
|
|
|
func (svm *serviceVM) getStartError() error {
|
|
<-svm.startStatus
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
return svm.startError
|
|
}
|
|
|
|
func (svm *serviceVM) signalStopFinished(err error) {
|
|
svm.Lock()
|
|
svm.stopError = err
|
|
svm.Unlock()
|
|
close(svm.stopStatus)
|
|
}
|
|
|
|
func (svm *serviceVM) getStopError() error {
|
|
<-svm.stopStatus
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
return svm.stopError
|
|
}
|
|
|
|
// hotAddVHDs waits for the service vm to start and then attaches the vhds.
|
|
func (svm *serviceVM) hotAddVHDs(mvds ...hcsshim.MappedVirtualDisk) error {
|
|
if err := svm.getStartError(); err != nil {
|
|
return err
|
|
}
|
|
return svm.hotAddVHDsAtStart(mvds...)
|
|
}
|
|
|
|
// hotAddVHDsAtStart works the same way as hotAddVHDs but does not wait for the VM to start.
|
|
func (svm *serviceVM) hotAddVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error {
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
for i, mvd := range mvds {
|
|
if _, ok := svm.attachedVHDs[mvd.HostPath]; ok {
|
|
svm.attachedVHDs[mvd.HostPath]++
|
|
continue
|
|
}
|
|
|
|
if err := svm.config.HotAddVhd(mvd.HostPath, mvd.ContainerPath, mvd.ReadOnly, !mvd.AttachOnly); err != nil {
|
|
svm.hotRemoveVHDsAtStart(mvds[:i]...)
|
|
return err
|
|
}
|
|
svm.attachedVHDs[mvd.HostPath] = 1
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// hotRemoveVHDs waits for the service vm to start and then removes the vhds.
|
|
func (svm *serviceVM) hotRemoveVHDs(mvds ...hcsshim.MappedVirtualDisk) error {
|
|
if err := svm.getStartError(); err != nil {
|
|
return err
|
|
}
|
|
return svm.hotRemoveVHDsAtStart(mvds...)
|
|
}
|
|
|
|
// hotRemoveVHDsAtStart works the same way as hotRemoveVHDs but does not wait for the VM to start.
|
|
func (svm *serviceVM) hotRemoveVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error {
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
var retErr error
|
|
for _, mvd := range mvds {
|
|
if _, ok := svm.attachedVHDs[mvd.HostPath]; !ok {
|
|
// We continue instead of returning an error if we try to hot remove a non-existent VHD.
|
|
// This is because one of the callers of the function is graphdriver.Put(). Since graphdriver.Get()
|
|
// defers the VM start to the first operation, it's possible that nothing have been hot-added
|
|
// when Put() is called. To avoid Put returning an error in that case, we simply continue if we
|
|
// don't find the vhd attached.
|
|
continue
|
|
}
|
|
|
|
if svm.attachedVHDs[mvd.HostPath] > 1 {
|
|
svm.attachedVHDs[mvd.HostPath]--
|
|
continue
|
|
}
|
|
|
|
// last VHD, so remove from VM and map
|
|
if err := svm.config.HotRemoveVhd(mvd.HostPath); err == nil {
|
|
delete(svm.attachedVHDs, mvd.HostPath)
|
|
} else {
|
|
// Take note of the error, but still continue to remove the other VHDs
|
|
logrus.Warnf("Failed to hot remove %s: %s", mvd.HostPath, err)
|
|
if retErr == nil {
|
|
retErr = err
|
|
}
|
|
}
|
|
}
|
|
return retErr
|
|
}
|
|
|
|
func (svm *serviceVM) createExt4VHDX(destFile string, sizeGB uint32, cacheFile string) error {
|
|
if err := svm.getStartError(); err != nil {
|
|
return err
|
|
}
|
|
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
return svm.config.CreateExt4Vhdx(destFile, sizeGB, cacheFile)
|
|
}
|
|
|
|
func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedVirtualDisk) (err error) {
|
|
if len(mvds) == 0 {
|
|
return fmt.Errorf("createUnionMount: error must have at least 1 layer")
|
|
}
|
|
|
|
if err = svm.getStartError(); err != nil {
|
|
return err
|
|
}
|
|
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
if _, ok := svm.unionMounts[mountName]; ok {
|
|
svm.unionMounts[mountName]++
|
|
return nil
|
|
}
|
|
|
|
var lowerLayers []string
|
|
if mvds[0].ReadOnly {
|
|
lowerLayers = append(lowerLayers, mvds[0].ContainerPath)
|
|
}
|
|
|
|
for i := 1; i < len(mvds); i++ {
|
|
lowerLayers = append(lowerLayers, mvds[i].ContainerPath)
|
|
}
|
|
|
|
logrus.Debugf("Doing the overlay mount with union directory=%s", mountName)
|
|
if err = svm.runProcess(fmt.Sprintf("mkdir -p %s", mountName), nil, nil, nil); err != nil {
|
|
return err
|
|
}
|
|
|
|
var cmd string
|
|
if mvds[0].ReadOnly {
|
|
// Readonly overlay
|
|
cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s %s",
|
|
strings.Join(lowerLayers, ","),
|
|
mountName)
|
|
} else {
|
|
upper := fmt.Sprintf("%s/upper", mvds[0].ContainerPath)
|
|
work := fmt.Sprintf("%s/work", mvds[0].ContainerPath)
|
|
|
|
if err = svm.runProcess(fmt.Sprintf("mkdir -p %s %s", upper, work), nil, nil, nil); err != nil {
|
|
return err
|
|
}
|
|
|
|
cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s,upperdir=%s,workdir=%s %s",
|
|
strings.Join(lowerLayers, ":"),
|
|
upper,
|
|
work,
|
|
mountName)
|
|
}
|
|
|
|
logrus.Debugf("createUnionMount: Executing mount=%s", cmd)
|
|
if err = svm.runProcess(cmd, nil, nil, nil); err != nil {
|
|
return err
|
|
}
|
|
|
|
svm.unionMounts[mountName] = 1
|
|
return nil
|
|
}
|
|
|
|
func (svm *serviceVM) deleteUnionMount(mountName string, disks ...hcsshim.MappedVirtualDisk) error {
|
|
if err := svm.getStartError(); err != nil {
|
|
return err
|
|
}
|
|
|
|
svm.Lock()
|
|
defer svm.Unlock()
|
|
if _, ok := svm.unionMounts[mountName]; !ok {
|
|
return nil
|
|
}
|
|
|
|
if svm.unionMounts[mountName] > 1 {
|
|
svm.unionMounts[mountName]--
|
|
return nil
|
|
}
|
|
|
|
logrus.Debugf("Removing union mount %s", mountName)
|
|
if err := svm.runProcess(fmt.Sprintf("umount %s", mountName), nil, nil, nil); err != nil {
|
|
return err
|
|
}
|
|
|
|
delete(svm.unionMounts, mountName)
|
|
return nil
|
|
}
|
|
|
|
func (svm *serviceVM) runProcess(command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
|
|
process, err := svm.config.RunProcess(command, stdin, stdout, stderr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer process.Close()
|
|
|
|
process.WaitTimeout(time.Duration(int(time.Second) * svm.config.UvmTimeoutSeconds))
|
|
exitCode, err := process.ExitCode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if exitCode != 0 {
|
|
return fmt.Errorf("svm.runProcess: command %s failed with exit code %d", command, exitCode)
|
|
}
|
|
return nil
|
|
}
|