From 07034a4420c9bd1cb964f18c0128a215ab17d5a8 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 20 Jun 2017 08:37:36 -0700 Subject: [PATCH] Vendor jhowardmsft/opengcs v0.0.3 Signed-off-by: John Howard --- vendor.conf | 1 + vendor/github.com/jhowardmsft/opengcs/LICENSE | 21 ++ .../opengcs/gogcs/client/config.go | 234 ++++++++++++++++++ .../opengcs/gogcs/client/createsandbox.go | 78 ++++++ .../opengcs/gogcs/client/hotaddvhd.go | 41 +++ .../opengcs/gogcs/client/hotremovevhd.go | 36 +++ .../opengcs/gogcs/client/layervhddetails.go | 33 +++ .../opengcs/gogcs/client/process.go | 61 +++++ .../opengcs/gogcs/client/tartovhd.go | 44 ++++ .../opengcs/gogcs/client/unsupported.go | 5 + .../opengcs/gogcs/client/utilities.go | 110 ++++++++ .../opengcs/gogcs/client/vhdtotar.go | 69 ++++++ 12 files changed, 733 insertions(+) create mode 100644 vendor/github.com/jhowardmsft/opengcs/LICENSE create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go create mode 100644 vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go diff --git a/vendor.conf b/vendor.conf index 13af6042b3..9bd91820bd 100644 --- a/vendor.conf +++ b/vendor.conf @@ -8,6 +8,7 @@ github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git github.com/gorilla/context v1.1 github.com/gorilla/mux v1.1 +github.com/jhowardmsft/opengcs v0.0.3 github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 github.com/tchap/go-patricia v2.2.6 diff --git a/vendor/github.com/jhowardmsft/opengcs/LICENSE b/vendor/github.com/jhowardmsft/opengcs/LICENSE new file mode 100644 index 0000000000..8739a025ea --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go new file mode 100644 index 0000000000..db4ccb6f6d --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go @@ -0,0 +1,234 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/Microsoft/hcsshim" + "github.com/Sirupsen/logrus" +) + +// Mode is the operational mode, both requested, and actual after verification +type Mode uint + +const ( + // Constants for the actual mode after validation + + // ModeActualError means an error has occurred during validation + ModeActualError = iota + // ModeActualVhdx means that we are going to use VHDX boot after validation + ModeActualVhdx + // ModeActualKernelInitrd means that we are going to use kernel+initrd for boot after validation + ModeActualKernelInitrd + + // Constants for the requested mode + + // ModeRequestAuto means auto-select the boot mode for a utility VM + ModeRequestAuto = iota // VHDX will be priority over kernel+initrd + // ModeRequestVhdx means request VHDX boot if possible + ModeRequestVhdx + // ModeRequestKernelInitrd means request Kernel+initrd boot if possible + ModeRequestKernelInitrd + + // defaultUvmTimeoutSeconds is the default time to wait for utility VM operations + defaultUvmTimeoutSeconds = 5 * 60 + + // DefaultSandboxSizeMB is the size of the default sandbox size in MB + DefaultSandboxSizeMB = 20 * 1024 * 1024 +) + +// Config is the structure used to configuring a utility VM to be used +// as a service VM. There are two ways of starting. Either supply a VHD, +// or a Kernel+Initrd. For the latter, both must be supplied, and both +// must be in the same directory. +// +// VHD is the priority. +// +// All paths are full host path-names. +type Config struct { + Kernel string // Kernel for Utility VM (embedded in a UEFI bootloader) + Initrd string // Initrd image for Utility VM + Vhdx string // VHD for booting the utility VM + Name string // Name of the utility VM + RequestedMode Mode // What mode is preferred when validating + ActualMode Mode // What mode was obtained during validation + UvmTimeoutSeconds int // How long to wait for the utility VM to respond in seconds + Uvm hcsshim.Container // The actual container +} + +// GenerateDefault generates a default config from a set of options +// If baseDir is not supplied, defaults to $env:ProgramFiles\lcow +func (config *Config) GenerateDefault(options []string) error { + baseDir := filepath.Join(os.Getenv("ProgramFiles"), "lcow") + + if _, err := os.Stat(baseDir); os.IsNotExist(err) { + return fmt.Errorf("opengcs: cannot create default utility VM configuration as directory '%s' was not found", baseDir) + } + + if config.UvmTimeoutSeconds < 0 { + return fmt.Errorf("opengcs: cannot generate a config when supplied a negative utility VM timeout") + } + + envTimeoutSeconds := 0 + optTimeoutSeconds := 0 + + if config.UvmTimeoutSeconds != 0 { + envTimeout := os.Getenv("OPENGCS_UVM_TIMEOUT_SECONDS") + if len(envTimeout) > 0 { + var err error + if envTimeoutSeconds, err = strconv.Atoi(envTimeout); err != nil { + return fmt.Errorf("opengcs: OPENGCS_UVM_TIMEOUT_SECONDS could not be interpreted as an integer") + } + if envTimeoutSeconds < 0 { + return fmt.Errorf("opengcs: OPENGCS_UVM_TIMEOUT_SECONDS cannot be negative") + } + } + } + + config.Vhdx = filepath.Join(baseDir, `uvm.vhdx`) + config.Kernel = filepath.Join(baseDir, `bootx64.efi`) + config.Initrd = filepath.Join(baseDir, `initrd.img`) + + for _, v := range options { + opt := strings.SplitN(v, "=", 2) + if len(opt) == 2 { + switch strings.ToLower(opt[0]) { + case "opengcskernel": + config.Kernel = opt[1] + case "opengcsinitrd": + config.Initrd = opt[1] + case "opengcsvhdx": + config.Vhdx = opt[1] + case "opengcstimeoutsecs": + var err error + if optTimeoutSeconds, err = strconv.Atoi(opt[1]); err != nil { + return fmt.Errorf("opengcs: opengcstimeoutsecs option could not be interpreted as an integer") + } + if optTimeoutSeconds < 0 { + return fmt.Errorf("opengcs: opengcstimeoutsecs option cannot be negative") + } + } + } + } + + // Which timeout are we going to take? If not through option or environment, + // then use the default constant, otherwise the maximum of the option or + // environment supplied setting. A requested on in the config supplied + // overrides all of this. + if config.UvmTimeoutSeconds == 0 { + config.UvmTimeoutSeconds = defaultUvmTimeoutSeconds + if optTimeoutSeconds != 0 || envTimeoutSeconds != 0 { + config.UvmTimeoutSeconds = optTimeoutSeconds + if envTimeoutSeconds > optTimeoutSeconds { + config.UvmTimeoutSeconds = envTimeoutSeconds + } + } + } + + return nil +} + +// validate validates a Config structure for starting a utility VM. +func (config *Config) validate() error { + config.ActualMode = ModeActualError + + if config.RequestedMode == ModeRequestVhdx && config.Vhdx == "" { + return fmt.Errorf("opengcs: config is invalid - request for VHDX mode did not supply a VHDX") + } + if config.RequestedMode == ModeRequestKernelInitrd && (config.Kernel == "" || config.Initrd == "") { + return fmt.Errorf("opengcs: config is invalid - request for Kernel+Initrd mode must supply both kernel and initrd") + } + + // Validate that if VHDX requested or auto, it exists. + if config.RequestedMode == ModeRequestAuto || config.RequestedMode == ModeRequestVhdx { + if _, err := os.Stat(config.Vhdx); os.IsNotExist(err) { + if config.RequestedMode == ModeRequestVhdx { + return fmt.Errorf("opengcs: mode requested was VHDX but '%s' could not be found", config.Vhdx) + } + } else { + config.ActualMode = ModeActualVhdx + return nil + } + } + + // So must be kernel+initrd, or auto where we fallback as the VHDX doesn't exist + if config.Initrd == "" || config.Kernel == "" { + if config.RequestedMode == ModeRequestKernelInitrd { + return fmt.Errorf("opengcs: both initrd and kernel options for utility VM boot must be supplied") + } + return fmt.Errorf("opengcs: configuration is invalid") + } + if _, err := os.Stat(config.Kernel); os.IsNotExist(err) { + return fmt.Errorf("opengcs: kernel '%s' was not found", config.Kernel) + } + if _, err := os.Stat(config.Initrd); os.IsNotExist(err) { + return fmt.Errorf("opengcs: initrd '%s' was not found", config.Initrd) + } + dk, _ := filepath.Split(config.Kernel) + di, _ := filepath.Split(config.Initrd) + if dk != di { + return fmt.Errorf("initrd '%s' and kernel '%s' must be located in the same directory", config.Initrd, config.Kernel) + } + + config.ActualMode = ModeActualKernelInitrd + return nil +} + +// Create creates a utility VM from a configuration. +func (config *Config) Create() error { + logrus.Debugf("opengcs Create: %+v", config) + + if err := config.validate(); err != nil { + return err + } + + configuration := &hcsshim.ContainerConfig{ + HvPartition: true, + Name: config.Name, + SystemType: "container", + ContainerType: "linux", + TerminateOnLastHandleClosed: true, + } + + if config.ActualMode == ModeActualVhdx { + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: config.Vhdx, + } + } else { + // TODO @jhowardmsft - with a platform change that is in-flight, remove ImagePath for + // initrd/kernel boot. Current platform requires it. + dir, _ := filepath.Split(config.Initrd) + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: dir, + LinuxInitrdPath: config.Initrd, + LinuxKernelPath: config.Kernel, + } + } + + configurationS, _ := json.Marshal(configuration) + logrus.Debugf("opengcs Create: calling HCS with '%s'", string(configurationS)) + uvm, err := hcsshim.CreateContainer(config.Name, configuration) + if err != nil { + return err + } + logrus.Debugf("opengcs Create: uvm created, starting...") + err = uvm.Start() + if err != nil { + logrus.Debugf("opengcs Create: uvm failed to start: %s", err) + // Make sure we don't leave it laying around as it's been created in HCS + uvm.Terminate() + return err + } + + config.Uvm = uvm + logrus.Debugf("opengcs Create: uvm %s is running", config.Name) + return nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go new file mode 100644 index 0000000000..66ab703735 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go @@ -0,0 +1,78 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "os" + "sync" + + "github.com/Sirupsen/logrus" +) + +var sandboxCacheLock sync.Mutex + +// CreateSandbox does what it says on the tin. This is done by copying a prebuilt-sandbox from the ServiceVM +// TODO: @jhowardmsft maxSizeInMB isn't hooked up in GCS. Needs a platform change which is in flight. +func (config *Config) CreateSandbox(destFile string, maxSizeInMB uint32, cacheFile string) error { + // Smallest we can accept is the default sandbox size as we can't size down, only expand. + if maxSizeInMB < DefaultSandboxSizeMB { + maxSizeInMB = DefaultSandboxSizeMB + } + + logrus.Debugf("opengcs: CreateSandbox: %s size:%dMB cache:%s", destFile, maxSizeInMB, cacheFile) + + // Retrieve from cache if the default size and already on disk + if maxSizeInMB == DefaultSandboxSizeMB { + sandboxCacheLock.Lock() + if _, err := os.Stat(cacheFile); err == nil { + if err := copyFile(cacheFile, destFile); err != nil { + sandboxCacheLock.Unlock() + return fmt.Errorf("opengcs: CreateSandbox: Failed to copy cached sandbox '%s' to '%s': %s", cacheFile, destFile, err) + } + sandboxCacheLock.Unlock() + logrus.Debugf("opengcs: CreateSandbox: %s fulfilled from cache", destFile) + return nil + } + sandboxCacheLock.Unlock() + } + + if config.Uvm == nil { + return fmt.Errorf("opengcs: CreateSandbox: No utility VM has been created") + } + + // TODO @jhowardmsft - needs a platform change so that can specify size. eg fmt.Sprintf("createSandbox -size %d", maxSizeInMB)) + process, err := config.createUtilsProcess("createSandbox") + if err != nil { + return fmt.Errorf("opengcs: CreateSandbox: %s: failed to create utils process: %s", destFile, err) + } + + defer func() { + process.Process.Close() + }() + + logrus.Debugf("opengcs: CreateSandbox: %s: writing from stdout", destFile) + // Get back the sandbox VHDx stream from the service VM and write it to file + resultSize, err := writeFileFromReader(destFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("createSandbox %s", destFile)) + if err != nil { + return fmt.Errorf("opengcs: CreateSandbox: %s: failed writing %d bytes to target file: %s", destFile, resultSize, err) + } + + // Populate the cache + if maxSizeInMB == DefaultSandboxSizeMB { + sandboxCacheLock.Lock() + // It may already exist due to being created on another thread, in which case no copy back needed. + if _, err := os.Stat(cacheFile); os.IsNotExist(err) { + if err := copyFile(destFile, cacheFile); err != nil { + sandboxCacheLock.Unlock() + return fmt.Errorf("opengcs: CreateSandbox: Failed to seed sandbox cache '%s' from '%s': %s", destFile, cacheFile, err) + } + } + sandboxCacheLock.Unlock() + } + + logrus.Debugf("opengcs: CreateSandbox: %s created (non-cache)", destFile) + return nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go new file mode 100644 index 0000000000..62de79016a --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go @@ -0,0 +1,41 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + + "github.com/Microsoft/hcsshim" + "github.com/Sirupsen/logrus" +) + +// HotAddVhd hot-adds a VHD to a utility VM. This is used in the global one-utility-VM- +// service-VM per host scenario. In order to do a graphdriver `Diff`, we hot-add the +// sandbox to /mnt/ so that we can run `exportSandbox` inside the utility VM to +// get a tar-stream of the sandboxes contents back to the daemon. +func (config *Config) HotAddVhd(hostPath string, containerPath string) error { + logrus.Debugf("opengcs: HotAddVhd: %s: %s", hostPath, containerPath) + + if config.Uvm == nil { + return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration") + } + + modification := &hcsshim.ResourceModificationRequestResponse{ + Resource: "MappedVirtualDisk", + Data: hcsshim.MappedVirtualDisk{ + HostPath: hostPath, + ContainerPath: containerPath, + CreateInUtilityVM: true, + //ReadOnly: true, + }, + Request: "Add", + } + logrus.Debugf("opengcs: HotAddVhd: %s to %s", hostPath, containerPath) + if err := config.Uvm.Modify(modification); err != nil { + return fmt.Errorf("opengcs: HotAddVhd: failed: %s", err) + } + logrus.Debugf("opengcs: HotAddVhd: %s added successfully", hostPath) + return nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go new file mode 100644 index 0000000000..71167d00ee --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go @@ -0,0 +1,36 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + + "github.com/Microsoft/hcsshim" + "github.com/Sirupsen/logrus" +) + +// HotRemoveVhd hot-removes a VHD from a utility VM. This is used in the global one-utility-VM- +// service-VM per host scenario. +func (config *Config) HotRemoveVhd(hostPath string) error { + logrus.Debugf("opengcs: HotRemoveVhd: %s", hostPath) + + if config.Uvm == nil { + return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration") + } + + modification := &hcsshim.ResourceModificationRequestResponse{ + Resource: "MappedVirtualDisk", + Data: hcsshim.MappedVirtualDisk{ + HostPath: hostPath, + CreateInUtilityVM: true, + }, + Request: "Remove", + } + if err := config.Uvm.Modify(modification); err != nil { + return fmt.Errorf("opengcs: HotRemoveVhd: %s failed: %s", hostPath, err) + } + logrus.Debugf("opengcs: HotRemoveVhd: %s removed successfully", hostPath) + return nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go new file mode 100644 index 0000000000..6ce29cdc56 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go @@ -0,0 +1,33 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "os" + "path/filepath" +) + +// LayerVhdDetails is a utility for getting a file name, size and indication of +// sandbox for a VHD(x) in a folder. A read-only layer will be layer.vhd. A +// read-write layer will be sandbox.vhdx. +func LayerVhdDetails(folder string) (string, int64, bool, error) { + var fileInfo os.FileInfo + isSandbox := false + filename := filepath.Join(folder, "layer.vhd") + var err error + + if fileInfo, err = os.Stat(filename); err != nil { + filename = filepath.Join(folder, "sandbox.vhdx") + if fileInfo, err = os.Stat(filename); err != nil { + if os.IsNotExist(err) { + return "", 0, isSandbox, fmt.Errorf("could not find layer or sandbox in %s", folder) + } + return "", 0, isSandbox, fmt.Errorf("error locating layer or sandbox in %s: %s", folder, err) + } + isSandbox = true + } + return filename, fileInfo.Size(), isSandbox, nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go new file mode 100644 index 0000000000..6592ab6a03 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go @@ -0,0 +1,61 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "io" + + "github.com/Microsoft/hcsshim" + "github.com/Sirupsen/logrus" +) + +// Process is the structure pertaining to a process running in a utility VM. +type process struct { + Process hcsshim.Process + Stdin io.WriteCloser + Stdout io.ReadCloser +} + +// createUtilsProcess is a convenient wrapper for hcsshim.createUtilsProcess to use when +// communicating with a utility VM. +func (config *Config) createUtilsProcess(commandLine string) (process, error) { + logrus.Debugf("opengcs: createUtilsProcess") + + if config.Uvm == nil { + return process{}, fmt.Errorf("cannot create utils process as no utility VM is in configuration") + } + + var ( + err error + proc process + ) + + env := make(map[string]string) + env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:" + processConfig := &hcsshim.ProcessConfig{ + EmulateConsole: false, + CreateStdInPipe: true, + CreateStdOutPipe: true, + CreateStdErrPipe: true, + CreateInUtilityVm: true, + WorkingDirectory: "/bin", + Environment: env, + CommandLine: commandLine, + } + proc.Process, err = config.Uvm.CreateProcess(processConfig) + if err != nil { + return process{}, fmt.Errorf("opengcs: createUtilsProcess: CreateProcess %+v failed %s", config, err) + } + + if proc.Stdin, proc.Stdout, _, err = proc.Process.Stdio(); err != nil { + proc.Process.Kill() // Should this have a timeout? + proc.Process.Close() + return process{}, fmt.Errorf("opengcs: createUtilsProcess: failed to get Stdio pipes %s", err) + } + + logrus.Debugf("opengcs: createUtilsProcess success: pid %d", proc.Process.Pid()) + return proc, nil +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go new file mode 100644 index 0000000000..8832e096e6 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go @@ -0,0 +1,44 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "io" + + "github.com/Sirupsen/logrus" +) + +// TarToVhd streams a tarstream contained in an io.Reader to a fixed vhd file +func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, error) { + logrus.Debugf("opengcs: TarToVhd: %s", targetVHDFile) + + if config.Uvm == nil { + return 0, fmt.Errorf("cannot Tar2Vhd as no utility VM is in configuration") + } + + process, err := config.createUtilsProcess("tar2vhd") + if err != nil { + return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed to create utils process tar2vhd: %s", targetVHDFile, err) + } + defer process.Process.Close() + + // Send the tarstream into the `tar2vhd`s stdin + if _, err = copyWithTimeout(process.Stdin, reader, 0, config.UvmTimeoutSeconds, fmt.Sprintf("send %s, to stdin of tar2vhd", targetVHDFile)); err != nil { + return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed to send to tar2vhd in uvm: %s", targetVHDFile, err) + } + + // Don't need stdin now we've sent everything. This signals GCS that we are finished sending data. + process.Process.CloseStdin() + + // Write stdout contents of `tar2vhd` to the VHD file + payloadSize, err := writeFileFromReader(targetVHDFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("output of tar2vhd to %s", targetVHDFile)) + if err != nil { + return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed writing VHD file: %s", targetVHDFile, err) + } + + logrus.Debugf("opengcs: TarToVhd: %s created, %d bytes", targetVHDFile, payloadSize) + return payloadSize, err +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go new file mode 100644 index 0000000000..4a7abab3b0 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go @@ -0,0 +1,5 @@ +// +build !windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go new file mode 100644 index 0000000000..427cde7795 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go @@ -0,0 +1,110 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "io" + "os" + "syscall" + "time" + "unsafe" + + "github.com/Sirupsen/logrus" +) + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procCopyFileW = modkernel32.NewProc("CopyFileW") +) + +// writeFileFromReader writes an output file from an io.Reader +func writeFileFromReader(path string, reader io.Reader, timeoutSeconds int, context string) (int64, error) { + outFile, err := os.Create(path) + if err != nil { + return 0, fmt.Errorf("opengcs: writeFileFromReader: failed to create %s: %s", path, err) + } + defer outFile.Close() + return copyWithTimeout(outFile, reader, 0, timeoutSeconds, context) +} + +// copyWithTimeout is a wrapper for io.Copy using a timeout duration +func copyWithTimeout(dst io.Writer, src io.Reader, size int64, timeoutSeconds int, context string) (int64, error) { + logrus.Debugf("opengcs: copywithtimeout: size %d: timeout %d: (%s)", size, timeoutSeconds, context) + + type resultType struct { + err error + bytes int64 + } + + done := make(chan resultType, 1) + go func() { + // TODO @jhowardmsft. Needs platform fix. Improve reliability by + // chunking the data. Ultimately can just use io.Copy instead with no loop + result := resultType{} + var copied int64 + for { + copied, result.err = io.CopyN(dst, src, 1024) + result.bytes += copied + if copied == 0 { + done <- result + break + } + // TODO @jhowardmsft - next line is debugging only. Remove + //logrus.Debugf("%s: copied so far %d\n", context, result.bytes) + } + }() + + var result resultType + timedout := time.After(time.Duration(timeoutSeconds) * time.Second) + + select { + case <-timedout: + return 0, fmt.Errorf("opengcs: copyWithTimeout: timed out (%s)", context) + case result = <-done: + if result.err != nil && result.err != io.EOF { + // See https://github.com/golang/go/blob/f3f29d1dea525f48995c1693c609f5e67c046893/src/os/exec/exec_windows.go for a clue as to why we are doing this :) + if se, ok := result.err.(syscall.Errno); ok { + const ( + errNoData = syscall.Errno(232) + errBrokenPipe = syscall.Errno(109) + ) + if se == errNoData || se == errBrokenPipe { + logrus.Debugf("opengcs: copyWithTimeout: hit NoData or BrokenPipe: %d: %s", se, context) + return result.bytes, nil + } + } + return 0, fmt.Errorf("opengcs: copyWithTimeout: error reading: '%s' after %d bytes (%s)", result.err, result.bytes, context) + } + } + logrus.Debugf("opengcs: copyWithTimeout: success - copied %d bytes (%s)", result.bytes, context) + return result.bytes, nil +} + +// copyFile is a utility for copying a file - used for the sandbox cache. +// Uses CopyFileW win32 API for performance +func copyFile(srcFile, destFile string) error { + var bFailIfExists uint32 = 1 + + lpExistingFileName, err := syscall.UTF16PtrFromString(srcFile) + if err != nil { + return err + } + lpNewFileName, err := syscall.UTF16PtrFromString(destFile) + if err != nil { + return err + } + r1, _, err := syscall.Syscall( + procCopyFileW.Addr(), + 3, + uintptr(unsafe.Pointer(lpExistingFileName)), + uintptr(unsafe.Pointer(lpNewFileName)), + uintptr(bFailIfExists)) + if r1 == 0 { + return fmt.Errorf("failed CopyFileW Win32 call from '%s' to %s: %s", srcFile, destFile, err) + } + return nil + +} diff --git a/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go new file mode 100644 index 0000000000..9504fb7c84 --- /dev/null +++ b/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go @@ -0,0 +1,69 @@ +// +build windows + +package client + +// TODO @jhowardmsft - This will move to Microsoft/opengcs soon + +import ( + "fmt" + "io" + "os" + + "github.com/Sirupsen/logrus" +) + +// VhdToTar does what is says - it exports a VHD in a specified +// folder (either a read-only layer.vhd, or a read-write sandbox.vhd) to a +// ReadCloser containing a tar-stream of the layers contents. +func (config *Config) VhdToTar(vhdFile string, uvmMountPath string, isSandbox bool, vhdSize int64) (io.ReadCloser, error) { + logrus.Debugf("opengcs: VhdToTar: %s isSandbox: %t", vhdFile, isSandbox) + + if config.Uvm == nil { + return nil, fmt.Errorf("cannot VhdToTar as no utility VM is in configuration") + } + + vhdHandle, err := os.Open(vhdFile) + if err != nil { + return nil, fmt.Errorf("opengcs: VhdToTar: failed to open %s: %s", vhdFile, err) + } + defer vhdHandle.Close() + logrus.Debugf("opengcs: VhdToTar: exporting %s, size %d, isSandbox %t", vhdHandle.Name(), vhdSize, isSandbox) + + // Different binary depending on whether a RO layer or a RW sandbox + command := "vhd2tar" + if isSandbox { + command = fmt.Sprintf("exportSandbox -path %s", uvmMountPath) + } + + // Start the binary in the utility VM + process, err := config.createUtilsProcess(command) + if err != nil { + return nil, fmt.Errorf("opengcs: VhdToTar: %s: failed to create utils process %s: %s", vhdHandle.Name(), command, err) + } + + if !isSandbox { + // Send the VHD contents to the utility VM processes stdin handle if not a sandbox + logrus.Debugf("opengcs: VhdToTar: copying the layer VHD into the utility VM") + if _, err = copyWithTimeout(process.Stdin, vhdHandle, vhdSize, config.UvmTimeoutSeconds, fmt.Sprintf("vhdtotarstream: sending %s to %s", vhdHandle.Name(), command)); err != nil { + process.Process.Close() + return nil, fmt.Errorf("opengcs: VhdToTar: %s: failed to copyWithTimeout on the stdin pipe (to utility VM): %s", vhdHandle.Name(), err) + } + } + + // Start a goroutine which copies the stdout (ie the tar stream) + reader, writer := io.Pipe() + go func() { + defer writer.Close() + defer process.Process.Close() + logrus.Debugf("opengcs: VhdToTar: copying tar stream back from the utility VM") + bytes, err := copyWithTimeout(writer, process.Stdout, vhdSize, config.UvmTimeoutSeconds, fmt.Sprintf("vhdtotarstream: copy tarstream from %s", command)) + if err != nil { + logrus.Errorf("opengcs: VhdToTar: %s: copyWithTimeout on the stdout pipe (from utility VM) failed: %s", vhdHandle.Name(), err) + } + logrus.Debugf("opengcs: VhdToTar: copied %d bytes of the tarstream of %s from the utility VM", bytes, vhdHandle.Name()) + }() + + // Return the read-side of the pipe connected to the goroutine which is reading from the stdout of the process in the utility VM + return reader, nil + +}