diff --git a/archive/diff.go b/archive/diff.go
index 877d076c8b..58d30466e3 100644
--- a/archive/diff.go
+++ b/archive/diff.go
@@ -1,10 +1,11 @@
 package archive
 
 import (
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
+	"time"
 )
 
 // ApplyLayer parses a diff in the standard layer format from `layer`, and
@@ -17,6 +18,20 @@ func ApplyLayer(dest string, layer Archive) error {
 		return err
 	}
 
+	modifiedDirs := make(map[string]*syscall.Stat_t)
+	addDir := func(file string) {
+		d := filepath.Dir(file)
+		if _, exists := modifiedDirs[d]; !exists {
+			if s, err := os.Lstat(d); err == nil {
+				if sys := s.Sys(); sys != nil {
+					if stat, ok := sys.(*syscall.Stat_t); ok {
+						modifiedDirs[d] = stat
+					}
+				}
+			}
+		}
+	}
+
 	// Step 2: walk for whiteouts and apply them, removing them in the process
 	err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error {
 		if err != nil {
@@ -39,25 +54,42 @@ func ApplyLayer(dest string, layer Archive) error {
 		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil {
 			return err
 		} else if matched {
-			log.Printf("Removing aufs metadata %s", fullPath)
-			_ = os.RemoveAll(fullPath)
+			addDir(fullPath)
+			if err := os.RemoveAll(fullPath); err != nil {
+				return err
+			}
 		}
 
 		filename := filepath.Base(path)
 		if strings.HasPrefix(filename, ".wh.") {
 			rmTargetName := filename[len(".wh."):]
 			rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName)
+
 			// Remove the file targeted by the whiteout
-			log.Printf("Removing whiteout target %s", rmTargetPath)
-			_ = os.RemoveAll(rmTargetPath)
+			addDir(rmTargetPath)
+			if err := os.RemoveAll(rmTargetPath); err != nil {
+				return err
+			}
 			// Remove the whiteout itself
-			log.Printf("Removing whiteout %s", fullPath)
-			_ = os.RemoveAll(fullPath)
+			addDir(fullPath)
+			if err := os.RemoveAll(fullPath); err != nil {
+				return err
+			}
 		}
 		return nil
 	})
 	if err != nil {
 		return err
 	}
+
+	for k, v := range modifiedDirs {
+		aTime := time.Unix(v.Atim.Unix())
+		mTime := time.Unix(v.Mtim.Unix())
+
+		if err := os.Chtimes(k, aTime, mTime); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }