From c98e77c77c5b43bf50e8ae5296b02ce0b47ea188 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 11 Jan 2017 15:55:43 -0800 Subject: [PATCH] Windows: Use sequential file access Signed-off-by: John Howard --- cli/command/utils.go | 7 +++- daemon/graphdriver/windows/windows.go | 6 ++- pkg/system/filesys.go | 16 +++++++- pkg/system/filesys_windows.go | 55 +++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/cli/command/utils.go b/cli/command/utils.go index 1837ca41f0..f9255cf87f 100644 --- a/cli/command/utils.go +++ b/cli/command/utils.go @@ -3,16 +3,19 @@ package command import ( "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" "strings" + + "github.com/docker/docker/pkg/system" ) // CopyToFile writes the content of the reader to the specified file func CopyToFile(outfile string, r io.Reader) error { - tmpFile, err := ioutil.TempFile(filepath.Dir(outfile), ".docker_temp_") + // We use sequential file access here to avoid depleting the standby list + // on Windows. On Linux, this is a call directly to ioutil.TempFile + tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".docker_temp_") if err != nil { return err } diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go index 254cc3aeb0..24ee24003f 100644 --- a/daemon/graphdriver/windows/windows.go +++ b/daemon/graphdriver/windows/windows.go @@ -825,14 +825,16 @@ func (fg *fileGetCloserWithBackupPrivileges) Get(filename string) (io.ReadCloser var f *os.File // Open the file while holding the Windows backup privilege. This ensures that the // file can be opened even if the caller does not actually have access to it according - // to the security descriptor. + // to the security descriptor. Also use sequential file access to avoid depleting the + // standby list - Microsoft VSO Bug Tracker #9900466 err := winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error { path := longpath.AddPrefix(filepath.Join(fg.path, filename)) p, err := syscall.UTF16FromString(path) if err != nil { return err } - h, err := syscall.CreateFile(&p[0], syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN + h, err := syscall.CreateFile(&p[0], syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|fileFlagSequentialScan, 0) if err != nil { return &os.PathError{Op: "open", Path: path, Err: err} } diff --git a/pkg/system/filesys.go b/pkg/system/filesys.go index 810c794786..7aa920de1a 100644 --- a/pkg/system/filesys.go +++ b/pkg/system/filesys.go @@ -3,6 +3,7 @@ package system import ( + "io/ioutil" "os" "path/filepath" ) @@ -24,7 +25,7 @@ func IsAbs(path string) bool { return filepath.IsAbs(path) } -// The functions below here are wrappers for the equivalents in the os package. +// The functions below here are wrappers for the equivalents in the os and ioutils packages. // They are passthrough on Unix platforms, and only relevant on Windows. // CreateSequential creates the named file with mode 0666 (before umask), truncating @@ -52,3 +53,16 @@ func OpenSequential(name string) (*os.File, error) { func OpenFileSequential(name string, flag int, perm os.FileMode) (*os.File, error) { return os.OpenFile(name, flag, perm) } + +// TempFileSequential creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFileSequential(dir, prefix string) (f *os.File, err error) { + return ioutil.TempFile(dir, prefix) +} diff --git a/pkg/system/filesys_windows.go b/pkg/system/filesys_windows.go index 6094f01fd4..626d2ad886 100644 --- a/pkg/system/filesys_windows.go +++ b/pkg/system/filesys_windows.go @@ -6,8 +6,11 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" + "sync" "syscall" + "time" "unsafe" winio "github.com/Microsoft/go-winio" @@ -234,3 +237,55 @@ func syscallOpenSequential(path string, mode int, _ uint32) (fd syscall.Handle, h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) return h, e } + +// Helpers for TempFileSequential +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFileSequential is a copy of ioutil.TempFile, modified to use sequential +// file access. Below is the original comment from golang: +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFileSequential(dir, prefix string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +}