From 9b648dfac6453de5944ee4bb749115d85a253a05 Mon Sep 17 00:00:00 2001 From: "Stefan J. Wernli" Date: Mon, 24 Aug 2015 14:07:22 -0700 Subject: [PATCH] Windows: Fix long path handling for docker build Signed-off-by: Stefan J. Wernli --- builder/internals.go | 9 +- builder/internals_unix.go | 5 + builder/internals_windows.go | 14 +++ pkg/archive/archive_windows.go | 7 +- pkg/chrootarchive/archive_windows.go | 3 +- pkg/chrootarchive/diff_windows.go | 6 +- pkg/directory/directory_windows.go | 8 ++ pkg/longpath/longpath.go | 21 ++++ pkg/symlink/README.md | 3 +- pkg/symlink/fs.go | 9 ++ pkg/symlink/fs_unix.go | 11 ++ pkg/symlink/fs_windows.go | 156 +++++++++++++++++++++++++++ utils/utils.go | 8 +- utils/utils_unix.go | 11 ++ utils/utils_windows.go | 17 +++ 15 files changed, 272 insertions(+), 16 deletions(-) create mode 100644 pkg/longpath/longpath.go create mode 100644 pkg/symlink/fs_unix.go create mode 100644 pkg/symlink/fs_windows.go create mode 100644 utils/utils_unix.go create mode 100644 utils/utils_windows.go diff --git a/builder/internals.go b/builder/internals.go index bc8512802c..5d8a448a42 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -34,6 +34,7 @@ import ( "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringutils" + "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/pkg/urlutil" @@ -42,7 +43,7 @@ import ( ) func (b *builder) readContext(context io.Reader) (err error) { - tmpdirPath, err := ioutil.TempDir("", "docker-build") + tmpdirPath, err := getTempDir("", "docker-build") if err != nil { return } @@ -305,7 +306,7 @@ func calcCopyInfo(b *builder, cmdName string, cInfos *[]*copyInfo, origPath stri } // Create a tmp dir - tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote") + tmpDirName, err := getTempDir(b.contextPath, "docker-remote") if err != nil { return err } @@ -684,14 +685,14 @@ func (b *builder) run(c *daemon.Container) error { func (b *builder) checkPathForAddition(orig string) error { origPath := filepath.Join(b.contextPath, orig) - origPath, err := filepath.EvalSymlinks(origPath) + origPath, err := symlink.EvalSymlinks(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } - contextPath, err := filepath.EvalSymlinks(b.contextPath) + contextPath, err := symlink.EvalSymlinks(b.contextPath) if err != nil { return err } diff --git a/builder/internals_unix.go b/builder/internals_unix.go index 3d8ad54e90..aaa9b4205f 100644 --- a/builder/internals_unix.go +++ b/builder/internals_unix.go @@ -3,10 +3,15 @@ package builder import ( + "io/ioutil" "os" "path/filepath" ) +func getTempDir(dir, prefix string) (string, error) { + return ioutil.TempDir(dir, prefix) +} + func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { // If the destination didn't already exist, or the destination isn't a // directory, then we should Lchown the destination. Otherwise, we shouldn't diff --git a/builder/internals_windows.go b/builder/internals_windows.go index 5d9d35e3e0..d2791de490 100644 --- a/builder/internals_windows.go +++ b/builder/internals_windows.go @@ -2,6 +2,20 @@ package builder +import ( + "io/ioutil" + + "github.com/docker/docker/pkg/longpath" +) + +func getTempDir(dir, prefix string) (string, error) { + tempDir, err := ioutil.TempDir(dir, prefix) + if err != nil { + return "", err + } + return longpath.AddPrefix(tempDir), nil +} + func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { // chown is not supported on Windows return nil diff --git a/pkg/archive/archive_windows.go b/pkg/archive/archive_windows.go index ea9316031b..7d5210537b 100644 --- a/pkg/archive/archive_windows.go +++ b/pkg/archive/archive_windows.go @@ -8,15 +8,14 @@ import ( "os" "path/filepath" "strings" + + "github.com/docker/docker/pkg/longpath" ) // fixVolumePathPrefix does platform specific processing to ensure that if // the path being passed in is not in a volume path format, convert it to one. func fixVolumePathPrefix(srcPath string) string { - if !strings.HasPrefix(srcPath, `\\?\`) { - srcPath = `\\?\` + srcPath - } - return srcPath + return longpath.AddPrefix(srcPath) } // getWalkRoot calculates the root path when performing a TarWithOptions. diff --git a/pkg/chrootarchive/archive_windows.go b/pkg/chrootarchive/archive_windows.go index c44556cf56..0a500ed5c2 100644 --- a/pkg/chrootarchive/archive_windows.go +++ b/pkg/chrootarchive/archive_windows.go @@ -4,6 +4,7 @@ import ( "io" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/longpath" ) // chroot is not supported by Windows @@ -17,5 +18,5 @@ func invokeUnpack(decompressedArchive io.ReadCloser, // Windows is different to Linux here because Windows does not support // chroot. Hence there is no point sandboxing a chrooted process to // do the unpack. We call inline instead within the daemon process. - return archive.Unpack(decompressedArchive, dest, options) + return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options) } diff --git a/pkg/chrootarchive/diff_windows.go b/pkg/chrootarchive/diff_windows.go index a21bfe2ffe..dcff5ac27e 100644 --- a/pkg/chrootarchive/diff_windows.go +++ b/pkg/chrootarchive/diff_windows.go @@ -5,9 +5,9 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/longpath" ) // applyLayerHandler parses a diff in the standard layer format from `layer`, and @@ -17,9 +17,7 @@ func applyLayerHandler(dest string, layer archive.Reader, decompress bool) (size dest = filepath.Clean(dest) // Ensure it is a Windows-style volume path - if !strings.HasPrefix(dest, `\\?\`) { - dest = `\\?\` + dest - } + dest = longpath.AddPrefix(dest) if decompress { decompressed, err := archive.DecompressStream(layer) diff --git a/pkg/directory/directory_windows.go b/pkg/directory/directory_windows.go index 7a9f8cb68c..a0fc04849e 100644 --- a/pkg/directory/directory_windows.go +++ b/pkg/directory/directory_windows.go @@ -5,10 +5,18 @@ package directory import ( "os" "path/filepath" + "strings" + + "github.com/docker/docker/pkg/longpath" ) // Size walks a directory tree and returns its total size in bytes. func Size(dir string) (size int64, err error) { + fixedPath, err := filepath.Abs(dir) + if err != nil { + return + } + fixedPath = longpath.AddPrefix(fixedPath) err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, e error) error { // Ignore directory sizes if fileInfo == nil { diff --git a/pkg/longpath/longpath.go b/pkg/longpath/longpath.go new file mode 100644 index 0000000000..1dcc10a2d4 --- /dev/null +++ b/pkg/longpath/longpath.go @@ -0,0 +1,21 @@ +// longpath introduces some constants and helper functions for handling long paths +// in Windows, which are expected to be prepended with `\\?\` and followed by either +// a drive letter, a UNC server\share, or a volume identifier. + +package longpath + +import ( + "strings" +) + +// Prefix is the longpath prefix for Windows file paths. +const Prefix = `\\?\` + +// AddPrefix will add the Windows long path prefix to the path provided if +// it does not already have it. +func AddPrefix(path string) string { + if !strings.HasPrefix(path, Prefix) { + path = Prefix + path + } + return path +} diff --git a/pkg/symlink/README.md b/pkg/symlink/README.md index 0d1dbb70e6..8dba54fd08 100644 --- a/pkg/symlink/README.md +++ b/pkg/symlink/README.md @@ -1,4 +1,5 @@ -Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks +Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks, +as well as a Windows long-path aware version of filepath.EvalSymlinks from the [Go standard library](https://golang.org/pkg/path/filepath). The code from filepath.EvalSymlinks has been adapted in fs.go. diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index d305500453..dcf707f426 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -132,3 +132,12 @@ func evalSymlinksInScope(path, root string) (string, error) { // what's happening here return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil } + +// EvalSymlinks returns the path name after the evaluation of any symbolic +// links. +// If path is relative the result will be relative to the current directory, +// unless one of the components is an absolute symbolic link. +// This version has been updated to support long paths prepended with `\\?\`. +func EvalSymlinks(path string) (string, error) { + return evalSymlinks(path) +} diff --git a/pkg/symlink/fs_unix.go b/pkg/symlink/fs_unix.go new file mode 100644 index 0000000000..818004f26c --- /dev/null +++ b/pkg/symlink/fs_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package symlink + +import ( + "path/filepath" +) + +func evalSymlinks(path string) (string, error) { + return filepath.EvalSymlinks(path) +} diff --git a/pkg/symlink/fs_windows.go b/pkg/symlink/fs_windows.go new file mode 100644 index 0000000000..29bd456886 --- /dev/null +++ b/pkg/symlink/fs_windows.go @@ -0,0 +1,156 @@ +package symlink + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/docker/docker/pkg/longpath" +) + +func toShort(path string) (string, error) { + p, err := syscall.UTF16FromString(path) + if err != nil { + return "", err + } + b := p // GetShortPathName says we can reuse buffer + n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + if n > uint32(len(b)) { + b = make([]uint16, n) + n, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + } + return syscall.UTF16ToString(b), nil +} + +func toLong(path string) (string, error) { + p, err := syscall.UTF16FromString(path) + if err != nil { + return "", err + } + b := p // GetLongPathName says we can reuse buffer + n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + if n > uint32(len(b)) { + b = make([]uint16, n) + n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + } + b = b[:n] + return syscall.UTF16ToString(b), nil +} + +func evalSymlinks(path string) (string, error) { + path, err := walkSymlinks(path) + if err != nil { + return "", err + } + + p, err := toShort(path) + if err != nil { + return "", err + } + p, err = toLong(p) + if err != nil { + return "", err + } + // syscall.GetLongPathName does not change the case of the drive letter, + // but the result of EvalSymlinks must be unique, so we have + // EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`). + // Make drive letter upper case. + if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' { + p = string(p[0]+'A'-'a') + p[1:] + } else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' { + p = p[:3] + string(p[4]+'A'-'a') + p[5:] + } + return filepath.Clean(p), nil +} + +const utf8RuneSelf = 0x80 + +func walkSymlinks(path string) (string, error) { + const maxIter = 255 + originalPath := path + // consume path by taking each frontmost path element, + // expanding it if it's a symlink, and appending it to b + var b bytes.Buffer + for n := 0; path != ""; n++ { + if n > maxIter { + return "", errors.New("EvalSymlinks: too many links in " + originalPath) + } + + // A path beginnging with `\\?\` represents the root, so automatically + // skip that part and begin processing the next segment. + if strings.HasPrefix(path, longpath.Prefix) { + b.WriteString(longpath.Prefix) + path = path[4:] + continue + } + + // find next path component, p + var i = -1 + for j, c := range path { + if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) { + i = j + break + } + } + var p string + if i == -1 { + p, path = path, "" + } else { + p, path = path[:i], path[i+1:] + } + + if p == "" { + if b.Len() == 0 { + // must be absolute path + b.WriteRune(filepath.Separator) + } + continue + } + + // If this is the first segment after the long path prefix, accept the + // current segment as a volume root or UNC share and move on to the next. + if b.String() == longpath.Prefix { + b.WriteString(p) + b.WriteRune(filepath.Separator) + continue + } + + fi, err := os.Lstat(b.String() + p) + if err != nil { + return "", err + } + if fi.Mode()&os.ModeSymlink == 0 { + b.WriteString(p) + if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') { + b.WriteRune(filepath.Separator) + } + continue + } + + // it's a symlink, put it at the front of path + dest, err := os.Readlink(b.String() + p) + if err != nil { + return "", err + } + if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) { + b.Reset() + } + path = dest + string(filepath.Separator) + path + } + return filepath.Clean(b.String()), nil +} diff --git a/utils/utils.go b/utils/utils.go index 8c98d47210..a9d7b46ee2 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -199,9 +199,13 @@ func ReplaceOrAppendEnvValues(defaults, overrides []string) []string { // can be read and returns an error if some files can't be read // symlinks which point to non-existing files don't trigger an error func ValidateContextDirectory(srcPath string, excludes []string) error { - return filepath.Walk(filepath.Join(srcPath, "."), func(filePath string, f os.FileInfo, err error) error { + contextRoot, err := getContextRoot(srcPath) + if err != nil { + return err + } + return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error { // skip this directory/file if it's not in the path, it won't get added to the context - if relFilePath, err := filepath.Rel(srcPath, filePath); err != nil { + if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil { return err } else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil { return err diff --git a/utils/utils_unix.go b/utils/utils_unix.go new file mode 100644 index 0000000000..86bfb770f9 --- /dev/null +++ b/utils/utils_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package utils + +import ( + "path/filepath" +) + +func getContextRoot(srcPath string) (string, error) { + return filepath.Join(srcPath, "."), nil +} diff --git a/utils/utils_windows.go b/utils/utils_windows.go new file mode 100644 index 0000000000..80b58bd99a --- /dev/null +++ b/utils/utils_windows.go @@ -0,0 +1,17 @@ +// +build windows + +package utils + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/longpath" +) + +func getContextRoot(srcPath string) (string, error) { + cr, err := filepath.Abs(srcPath) + if err != nil { + return "", err + } + return longpath.AddPrefix(cr), nil +}