Revendor hcsshim

Signed-off-by: Darren Stahl <darst@microsoft.com>

hack/vendor.sh

@@ -43,7 +43,7 @@ esac
 
 # the following lines are in sorted order, FYI
 clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
-clone git github.com/Microsoft/hcsshim v0.3.0
+clone git github.com/Microsoft/hcsshim v0.3.1
 clone git github.com/Microsoft/go-winio v0.3.4
 clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps
 clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a

integration-cli/docker_cli_build_test.go

@@ -6740,3 +6740,18 @@ func (s *DockerSuite) TestBuildLabelsOverride(c *check.C) {
 	}
 
 }
+
+// Test case for #22855
+func (s *DockerSuite) TestBuildDeleteCommittedFile(c *check.C) {
+	name := "test-delete-committed-file"
+
+	_, err := buildImage(name,
+		`FROM busybox
+		RUN echo test > file
+		RUN test -e file
+		RUN rm file
+		RUN sh -c "! test -e file"`, false)
+	if err != nil {
+		c.Fatal(err)
+	}
+}

vendor/src/github.com/Microsoft/hcsshim/callback.go

@@ -40,39 +40,36 @@ type hcsNotification uint32
 type notificationChannel chan error
 
 type notifcationWatcherContext struct {
-	channel              notificationChannel
+	channels notificationChannels
-	expectedNotification hcsNotification
+	handle   hcsCallback
-	handle               hcsCallback
+}
+
+type notificationChannels map[hcsNotification]notificationChannel
+
+func newChannels() notificationChannels {
+	channels := make(notificationChannels)
+
+	channels[hcsNotificationSystemExited] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationProcessExited] = make(notificationChannel, 1)
+	channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1)
+	return channels
 }
 
 func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr {
-	var (
+	var result error
-		result       error
+	if int32(notificationStatus) < 0 {
-		completeWait = false
+		result = syscall.Errno(win32FromHresult(notificationStatus))
-	)
+	}
 
 	callbackMapLock.RLock()
-	context := callbackMap[callbackNumber]
+	channels := callbackMap[callbackNumber].channels
 	callbackMapLock.RUnlock()
 
-	if notificationType == context.expectedNotification {
+	channels[notificationType] <- result
-		if int32(notificationStatus) < 0 {
-			result = syscall.Errno(win32FromHresult(notificationStatus))
-		} else {
-			result = nil
-		}
-		completeWait = true
-	} else if notificationType == hcsNotificationSystemExited {
-		result = ErrUnexpectedContainerExit
-		completeWait = true
-	} else if notificationType == hcsNotificationServiceDisconnect {
-		result = ErrUnexpectedProcessAbort
-		completeWait = true
-	}
-
-	if completeWait {
-		context.channel <- result
-	}
 
 	return 0
 }

vendor/src/github.com/Microsoft/hcsshim/container.go

@@ -26,8 +26,9 @@ type ContainerError struct {
 }
 
 type container struct {
-	handle hcsSystem
+	handle         hcsSystem
-	id     string
+	id             string
+	callbackNumber uintptr
 }
 
 type containerProperties struct {
@@ -47,7 +48,6 @@ type containerProperties struct {
 func CreateContainer(id string, c *ContainerConfig) (Container, error) {
 	operation := "CreateContainer"
 	title := "HCSShim::" + operation
-	logrus.Debugf(title+" id=%s", id)
 
 	container := &container{
 		id: id,
@@ -59,21 +59,28 @@ func CreateContainer(id string, c *ContainerConfig) (Container, error) {
 	}
 
 	configuration := string(configurationb)
+	logrus.Debugf(title+" id=%s config=%s", id, configuration)
 
 	var (
-		handle      hcsSystem
 		resultp     *uint16
 		createError error
 	)
 	if hcsCallbacksSupported {
 		var identity syscall.Handle
-		createError = hcsCreateComputeSystem(id, configuration, identity, &handle, &resultp)
+		createError = hcsCreateComputeSystem(id, configuration, identity, &container.handle, &resultp)
-	} else {
-		createError = hcsCreateComputeSystemTP5(id, configuration, &handle, &resultp)
-	}
-	container.handle = handle
 
-	err = processAsyncHcsResult(container, createError, resultp, hcsNotificationSystemCreateCompleted, &defaultTimeout)
+		if createError == nil || createError == ErrVmcomputeOperationPending {
+			if err := container.registerCallback(); err != nil {
+				err := &ContainerError{Container: container, Operation: operation, Err: err}
+				logrus.Error(err)
+				return nil, err
+			}
+		}
+	} else {
+		createError = hcsCreateComputeSystemTP5(id, configuration, &container.handle, &resultp)
+	}
+
+	err = processAsyncHcsResult(createError, resultp, container.callbackNumber, hcsNotificationSystemCreateCompleted, &defaultTimeout)
 	if err != nil {
 		err := &ContainerError{Container: container, Operation: operation, ExtraInfo: configuration, Err: err}
 		logrus.Error(err)
@@ -122,7 +129,7 @@ func (container *container) Start() error {
 
 	var resultp *uint16
 	err := hcsStartComputeSystemTP5(container.handle, nil, &resultp)
-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemStartCompleted, &defaultTimeout)
+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemStartCompleted, &defaultTimeout)
 	if err != nil {
 		err := &ContainerError{Container: container, Operation: operation, Err: err}
 		logrus.Error(err)
@@ -186,7 +193,7 @@ func (container *container) Wait() error {
 	logrus.Debugf(title+" id=%s", container.id)
 
 	if hcsCallbacksSupported {
-		err := registerAndWaitForCallback(container, hcsNotificationSystemExited)
+		err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, nil)
 		if err != nil {
 			err := &ContainerError{Container: container, Operation: operation, Err: err}
 			logrus.Error(err)
@@ -217,7 +224,7 @@ func (container *container) WaitTimeout(timeout time.Duration) error {
 	logrus.Debugf(title+" id=%s", container.id)
 
 	if hcsCallbacksSupported {
-		err := registerAndWaitForCallbackTimeout(container, hcsNotificationSystemExited, timeout)
+		err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, &timeout)
 		if err == ErrTimeout {
 			return ErrTimeout
 		} else if err != nil {
@@ -304,7 +311,7 @@ func (container *container) Pause() error {
 
 	var resultp *uint16
 	err := hcsPauseComputeSystemTP5(container.handle, nil, &resultp)
-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemPauseCompleted, &defaultTimeout)
+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemPauseCompleted, &defaultTimeout)
 	if err != nil {
 		err := &ContainerError{Container: container, Operation: operation, Err: err}
 		logrus.Error(err)
@@ -325,7 +332,7 @@ func (container *container) Resume() error {
 	)
 
 	err := hcsResumeComputeSystemTP5(container.handle, nil, &resultp)
-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemResumeCompleted, &defaultTimeout)
+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemResumeCompleted, &defaultTimeout)
 	if err != nil {
 		err := &ContainerError{Container: container, Operation: operation, Err: err}
 		logrus.Error(err)
@@ -340,7 +347,6 @@ func (container *container) Resume() error {
 func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {
 	operation := "CreateProcess"
 	title := "HCSShim::Container::" + operation
-	logrus.Debugf(title+" id=%s", container.id)
 	var (
 		processInfo   hcsProcessInformation
 		processHandle hcsProcess
@@ -359,6 +365,7 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {
 	}
 
 	configuration := string(configurationb)
+	logrus.Debugf(title+" id=%s config=%s", container.id, configuration)
 
 	err = hcsCreateProcess(container.handle, configuration, &processInfo, &processHandle, &resultp)
 	err = processHcsResult(err, resultp)
@@ -379,6 +386,14 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {
 		},
 	}
 
+	if hcsCallbacksSupported {
+		if err := process.registerCallback(); err != nil {
+			err = &ContainerError{Container: container, Operation: operation, Err: err}
+			logrus.Error(err)
+			return nil, err
+		}
+	}
+
 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)
 	runtime.SetFinalizer(process, closeProcess)
 	return process, nil
@@ -408,6 +423,12 @@ func (container *container) OpenProcess(pid int) (Process, error) {
 		container: container,
 	}
 
+	if err := process.registerCallback(); err != nil {
+		err = &ContainerError{Container: container, Operation: operation, Err: err}
+		logrus.Error(err)
+		return nil, err
+	}
+
 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)
 	runtime.SetFinalizer(process, closeProcess)
 	return process, nil
@@ -424,6 +445,14 @@ func (container *container) Close() error {
 		return nil
 	}
 
+	if hcsCallbacksSupported {
+		if err := container.unregisterCallback(); err != nil {
+			err = &ContainerError{Container: container, Operation: operation, Err: err}
+			logrus.Error(err)
+			return err
+		}
+	}
+
 	if err := hcsCloseComputeSystem(container.handle); err != nil {
 		err = &ContainerError{Container: container, Operation: operation, Err: err}
 		logrus.Error(err)
@@ -441,7 +470,7 @@ func closeContainer(container *container) {
 	container.Close()
 }
 
-func (container *container) registerCallback(expectedNotification hcsNotification) (uintptr, error) {
+func (container *container) registerCallback() error {
 	callbackMapLock.Lock()
 	defer callbackMapLock.Unlock()
 
@@ -449,22 +478,24 @@ func (container *container) registerCallback(expectedNotification hcsNotificatio
 	nextCallback++
 
 	context := &notifcationWatcherContext{
-		expectedNotification: expectedNotification,
+		channels: newChannels(),
-		channel:              make(chan error, 1),
 	}
 	callbackMap[callbackNumber] = context
 
 	var callbackHandle hcsCallback
 	err := hcsRegisterComputeSystemCallback(container.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)
 	if err != nil {
-		return 0, err
+		return err
 	}
 	context.handle = callbackHandle
+	container.callbackNumber = callbackNumber
 
-	return callbackNumber, nil
+	return nil
 }
 
-func (container *container) unregisterCallback(callbackNumber uintptr) error {
+func (container *container) unregisterCallback() error {
+	callbackNumber := container.callbackNumber
+
 	callbackMapLock.Lock()
 	defer callbackMapLock.Unlock()
 

vendor/src/github.com/Microsoft/hcsshim/legacy.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -77,7 +78,7 @@ func readTombstones(path string) (map[string]([]string), error) {
 
 	ts := make(map[string]([]string))
 	for s.Scan() {
-		t := s.Text()[1:] // skip leading `\`
+		t := filepath.Join("Files", s.Text()[1:]) // skip leading `\`
 		dir := filepath.Dir(t)
 		ts[dir] = append(ts[dir], t)
 	}
@@ -107,6 +108,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error {
 		if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") {
 			return nil
 		}
+
 		r.result <- &fileEntry{path, info, nil}
 		if !<-r.proceed {
 			return errorIterationCanceled
@@ -120,7 +122,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error {
 			}
 			if dts, ok := ts[relPath]; ok {
 				for _, t := range dts {
-					r.result <- &fileEntry{t, nil, nil}
+					r.result <- &fileEntry{filepath.Join(r.root, t), nil, nil}
 					if !<-r.proceed {
 						return errorIterationCanceled
 					}
@@ -397,7 +399,10 @@ func (w *legacyLayerWriter) AddLink(name string, target string) error {
 }
 
 func (w *legacyLayerWriter) Remove(name string) error {
-	w.tombstones = append(w.tombstones, name)
+	if !strings.HasPrefix(name, `Files\`) {
+		return fmt.Errorf("invalid tombstone %s", name)
+	}
+	w.tombstones = append(w.tombstones, name[len(`Files\`):])
 	return nil
 }
 

vendor/src/github.com/Microsoft/hcsshim/process.go

@@ -22,11 +22,11 @@ type ProcessError struct {
 }
 
 type process struct {
-	handle             hcsProcess
+	handle         hcsProcess
-	processID          int
+	processID      int
-	container          *container
+	container      *container
-	cachedPipes        *cachedPipes
+	cachedPipes    *cachedPipes
-	killCallbackNumber uintptr
+	callbackNumber uintptr
 }
 
 type cachedPipes struct {
@@ -101,7 +101,7 @@ func (process *process) Wait() error {
 	logrus.Debugf(title+" processid=%d", process.processID)
 
 	if hcsCallbacksSupported {
-		err := registerAndWaitForCallback(process, hcsNotificationProcessExited)
+		err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil)
 		if err != nil {
 			err := &ProcessError{Operation: operation, Process: process, Err: err}
 			logrus.Error(err)
@@ -128,7 +128,7 @@ func (process *process) WaitTimeout(timeout time.Duration) error {
 	logrus.Debugf(title+" processid=%d", process.processID)
 
 	if hcsCallbacksSupported {
-		err := registerAndWaitForCallbackTimeout(process, hcsNotificationProcessExited, timeout)
+		err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout)
 		if err == ErrTimeout {
 			return ErrTimeout
 		} else if err != nil {
@@ -344,6 +344,14 @@ func (process *process) Close() error {
 		return nil
 	}
 
+	if hcsCallbacksSupported {
+		if err := process.unregisterCallback(); err != nil {
+			err = &ProcessError{Operation: operation, Process: process, Err: err}
+			logrus.Error(err)
+			return err
+		}
+	}
+
 	if err := hcsCloseProcess(process.handle); err != nil {
 		err = &ProcessError{Operation: operation, Process: process, Err: err}
 		logrus.Error(err)
@@ -361,7 +369,7 @@ func closeProcess(process *process) {
 	process.Close()
 }
 
-func (process *process) registerCallback(expectedNotification hcsNotification) (uintptr, error) {
+func (process *process) registerCallback() error {
 	callbackMapLock.Lock()
 	defer callbackMapLock.Unlock()
 
@@ -369,22 +377,24 @@ func (process *process) registerCallback(expectedNotification hcsNotification) (
 	nextCallback++
 
 	context := &notifcationWatcherContext{
-		expectedNotification: expectedNotification,
+		channels: newChannels(),
-		channel:              make(chan error, 1),
 	}
 	callbackMap[callbackNumber] = context
 
 	var callbackHandle hcsCallback
 	err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)
 	if err != nil {
-		return 0, err
+		return err
 	}
 	context.handle = callbackHandle
+	process.callbackNumber = callbackNumber
 
-	return callbackNumber, nil
+	return nil
 }
 
-func (process *process) unregisterCallback(callbackNumber uintptr) error {
+func (process *process) unregisterCallback() error {
+	callbackNumber := process.callbackNumber
+
 	callbackMapLock.Lock()
 	defer callbackMapLock.Unlock()
 	handle := callbackMap[callbackNumber].handle

vendor/src/github.com/Microsoft/hcsshim/waithelper.go

@@ -1,6 +1,7 @@
 package hcsshim
 
 import (
+	"github.com/Sirupsen/logrus"
 	"syscall"
 	"time"
 )
@@ -10,11 +11,6 @@ type waitable interface {
 	hcsWait(timeout uint32) (bool, error)
 }
 
-type callbackable interface {
-	registerCallback(expectedNotification hcsNotification) (uintptr, error)
-	unregisterCallback(callbackNumber uintptr) error
-}
-
 func waitTimeoutHelper(object waitable, timeout time.Duration) (bool, error) {
 	var (
 		millis uint32
@@ -52,62 +48,62 @@ func waitForSingleObject(handle syscall.Handle, timeout uint32) (bool, error) {
 	}
 }
 
-func processAsyncHcsResult(object callbackable, err error, resultp *uint16, expectedNotification hcsNotification, timeout *time.Duration) error {
+func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {
 	err = processHcsResult(err, resultp)
 	if err == ErrVmcomputeOperationPending {
-		if timeout != nil {
+		return waitForNotification(callbackNumber, expectedNotification, timeout)
-			err = registerAndWaitForCallbackTimeout(object, expectedNotification, *timeout)
-		} else {
-			err = registerAndWaitForCallback(object, expectedNotification)
-		}
 	}
 
 	return err
 }
 
-func registerAndWaitForCallbackTimeout(object callbackable, expectedNotification hcsNotification, timeout time.Duration) error {
+func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {
-	callbackNumber, err := object.registerCallback(expectedNotification)
-	if err != nil {
-		return err
-	}
-	defer object.unregisterCallback(callbackNumber)
-
-	return waitForNotificationTimeout(callbackNumber, timeout)
-}
-
-func registerAndWaitForCallback(object callbackable, expectedNotification hcsNotification) error {
-	callbackNumber, err := object.registerCallback(expectedNotification)
-	if err != nil {
-		return err
-	}
-	defer object.unregisterCallback(callbackNumber)
-
-	return waitForNotification(callbackNumber)
-}
-
-func waitForNotificationTimeout(callbackNumber uintptr, timeout time.Duration) error {
 	callbackMapLock.RLock()
-	channel := callbackMap[callbackNumber].channel
+	channels := callbackMap[callbackNumber].channels
 	callbackMapLock.RUnlock()
 
-	timer := time.NewTimer(timeout)
+	expectedChannel := channels[expectedNotification]
-	defer timer.Stop()
+	if expectedChannel == nil {
+		logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification)
+	}
 
+	if timeout != nil {
+		timer := time.NewTimer(*timeout)
+		defer timer.Stop()
+
+		select {
+		case err := <-expectedChannel:
+			return err
+		case err := <-channels[hcsNotificationSystemExited]:
+			// If the expected notification is hcsNotificationSystemExited which of the two selects
+			// chosen is random. Return the raw error if hcsNotificationSystemExited is expected
+			if channels[hcsNotificationSystemExited] == expectedChannel {
+				return err
+			}
+			return ErrUnexpectedContainerExit
+		case err := <-channels[hcsNotificationServiceDisconnect]:
+			// hcsNotificationServiceDisconnect should never be an expected notification
+			// it does not need the same handling as hcsNotificationSystemExited
+			logrus.Error(err)
+			return ErrUnexpectedProcessAbort
+		case <-timer.C:
+			return ErrTimeout
+		}
+	}
 	select {
-	case err := <-channel:
+	case err := <-expectedChannel:
-		return err
-	case <-timer.C:
-		return ErrTimeout
-	}
-}
-
-func waitForNotification(callbackNumber uintptr) error {
-	callbackMapLock.RLock()
-	channel := callbackMap[callbackNumber].channel
-	callbackMapLock.RUnlock()
-
-	select {
-	case err := <-channel:
 		return err
+	case err := <-channels[hcsNotificationSystemExited]:
+		// If the expected notification is hcsNotificationSystemExited which of the two selects
+		// chosen is random. Return the raw error if hcsNotificationSystemExited is expected
+		if channels[hcsNotificationSystemExited] == expectedChannel {
+			return err
+		}
+		return ErrUnexpectedContainerExit
+	case err := <-channels[hcsNotificationServiceDisconnect]:
+		// hcsNotificationServiceDisconnect should never be an expected notification
+		// it does not need the same handling as hcsNotificationSystemExited
+		logrus.Error(err)
+		return ErrUnexpectedProcessAbort
 	}
 }