1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Windows: Docker build starting to work

Signed-off-by: John Howard <jhoward@microsoft.com>
This commit is contained in:
John Howard 2015-06-01 16:42:27 -07:00
parent ba9db62e68
commit 3c177dc877
16 changed files with 265 additions and 159 deletions

View file

@ -10,6 +10,7 @@ package builder
import (
"fmt"
"io/ioutil"
"path"
"path/filepath"
"regexp"
"runtime"
@ -39,9 +40,6 @@ func nullDispatch(b *Builder, args []string, attributes map[string]bool, origina
// in the dockerfile available from the next statement on via ${foo}.
//
func env(b *Builder, args []string, attributes map[string]bool, original string) error {
if runtime.GOOS == "windows" {
return fmt.Errorf("ENV is not supported on Windows.")
}
if len(args) == 0 {
return fmt.Errorf("ENV requires at least one argument")
}
@ -269,12 +267,39 @@ func workdir(b *Builder, args []string, attributes map[string]bool, original str
return err
}
// Note that workdir passed comes from the Dockerfile. Hence it is in
// Linux format using forward-slashes, even on Windows. However,
// b.Config.WorkingDir is in platform-specific notation (in other words
// on Windows will use `\`
workdir := args[0]
if !filepath.IsAbs(workdir) {
workdir = filepath.Join("/", b.Config.WorkingDir, workdir)
isAbs := false
if runtime.GOOS == "windows" {
// Alternate processing for Windows here is necessary as we can't call
// filepath.IsAbs(workDir) as that would verify Windows style paths,
// along with drive-letters (eg c:\pathto\file.txt). We (arguably
// correctly or not) check for both forward and back slashes as this
// is what the 1.4.2 GoLang implementation of IsAbs() does in the
// isSlash() function.
isAbs = workdir[0] == '\\' || workdir[0] == '/'
} else {
isAbs = filepath.IsAbs(workdir)
}
if !isAbs {
current := b.Config.WorkingDir
if runtime.GOOS == "windows" {
// Convert to Linux format before join
current = strings.Replace(current, "\\", "/", -1)
}
// Must use path.Join so works correctly on Windows, not filepath
workdir = path.Join("/", current, workdir)
}
// Convert to platform specific format
if runtime.GOOS == "windows" {
workdir = strings.Replace(workdir, "/", "\\", -1)
}
b.Config.WorkingDir = workdir
return b.commit("", b.Config.Cmd, fmt.Sprintf("WORKDIR %v", workdir))

View file

@ -489,7 +489,8 @@ func (b *Builder) processImageFrom(img *imagepkg.Image) error {
b.Config = img.Config
}
if len(b.Config.Env) == 0 {
// The default path will be blank on Windows (set by HCS)
if len(b.Config.Env) == 0 && daemon.DefaultPathEnv != "" {
b.Config.Env = append(b.Config.Env, "PATH="+daemon.DefaultPathEnv)
}

View file

@ -201,8 +201,11 @@ func (container *Container) LogEvent(action string) {
// symlinking to a different path) between using this method and using the
// path. See symlink.FollowSymlinkInScope for more details.
func (container *Container) GetResourcePath(path string) (string, error) {
cleanPath := filepath.Join("/", path)
return symlink.FollowSymlinkInScope(filepath.Join(container.basefs, cleanPath), container.basefs)
// IMPORTANT - These are paths on the OS where the daemon is running, hence
// any filepath operations must be done in an OS agnostic way.
cleanPath := filepath.Join(string(os.PathSeparator), path)
r, e := symlink.FollowSymlinkInScope(filepath.Join(container.basefs, cleanPath), container.basefs)
return r, e
}
// Evaluates `path` in the scope of the container's root, with proper path
@ -218,7 +221,9 @@ func (container *Container) GetResourcePath(path string) (string, error) {
// symlinking to a different path) between using this method and using the
// path. See symlink.FollowSymlinkInScope for more details.
func (container *Container) GetRootResourcePath(path string) (string, error) {
cleanPath := filepath.Join("/", path)
// IMPORTANT - These are paths on the OS where the daemon is running, hence
// any filepath operations must be done in an OS agnostic way.
cleanPath := filepath.Join(string(os.PathSeparator), path)
return symlink.FollowSymlinkInScope(filepath.Join(container.root, cleanPath), container.root)
}

View file

@ -10,8 +10,9 @@ import (
"github.com/docker/docker/pkg/archive"
)
// TODO Windows. A reasonable default at the moment.
const DefaultPathEnv = `c:\windows\system32;c:\windows\system32\WindowsPowerShell\v1.0`
// This is deliberately empty on Windows as the default path will be set by
// the container. Docker has no context of what the default path should be.
const DefaultPathEnv = ""
type Container struct {
CommonContainer
@ -48,7 +49,8 @@ func (container *Container) setupLinkedContainers() ([]string, error) {
}
func (container *Container) createDaemonEnvironment(linkedEnv []string) []string {
return nil
// On Windows, nothing to link. Just return the container environment.
return container.Config.Env
}
func (container *Container) initializeNetworking() error {

View file

@ -258,6 +258,7 @@ func (daemon *Daemon) registerMountPoints(container *Container, hostConfig *runc
return nil
}
// TODO Windows. Factor out as not relevant (as Windows daemon support not in pre-1.7)
// verifyVolumesInfo ports volumes configured for the containers pre docker 1.7.
// It reads the container configuration and creates valid mount points for the old volumes.
func (daemon *Daemon) verifyVolumesInfo(container *Container) error {

View file

@ -452,6 +452,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
}
seen[relFilePath] = true
// TODO Windows: Verify if this needs to be os.Pathseparator
// Rename the base resource
if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) {
renamedRelFilePath = relFilePath
@ -503,7 +504,8 @@ loop:
}
// Normalize name, for safety and for a simple is-root check
// This keeps "../" as-is, but normalizes "/../" to "/"
// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
// This keeps "..\" as-is, but normalizes "\..\" to "\".
hdr.Name = filepath.Clean(hdr.Name)
for _, exclude := range options.ExcludePatterns {
@ -512,7 +514,10 @@ loop:
}
}
if !strings.HasSuffix(hdr.Name, "/") {
// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
// the filepath format for the OS on which the daemon is running. Hence
// the check for a slash-suffix MUST be done in an OS-agnostic way.
if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
// Not the root directory, ensure that the parent directory exists
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
@ -529,7 +534,7 @@ loop:
if err != nil {
return err
}
if strings.HasPrefix(rel, "../") {
if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
@ -658,10 +663,13 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
if err != nil {
return err
}
if srcSt.IsDir() {
return fmt.Errorf("Can't copy a directory")
}
// Clean up the trailing slash
// Clean up the trailing slash. This must be done in an operating
// system specific manner.
if dst[len(dst)-1] == os.PathSeparator {
dst = filepath.Join(dst, filepath.Base(src))
}
@ -709,8 +717,10 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
// Destination handling is in an operating specific manner depending
// where the daemon is running. If `dst` ends with a trailing slash
// the final destination path will be `dst/base(src)` (Linux) or
// `dst\base(src)` (Windows).
func CopyFileWithTar(src, dst string) (err error) {
return defaultArchiver.CopyFileWithTar(src, dst)
}

View file

@ -84,15 +84,17 @@ func Changes(layers []string, rw string) ([]Change, error) {
if err != nil {
return err
}
path = filepath.Join("/", path)
// As this runs on the daemon side, file paths are OS specific.
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == "/" {
if path == string(os.PathSeparator) {
return nil
}
// Skip AUFS metadata
if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched {
return err
}
@ -169,12 +171,13 @@ type FileInfo struct {
}
func (root *FileInfo) LookUp(path string) *FileInfo {
// As this runs on the daemon side, file paths are OS specific.
parent := root
if path == "/" {
if path == string(os.PathSeparator) {
return root
}
pathElements := strings.Split(path, "/")
pathElements := strings.Split(path, string(os.PathSeparator))
for _, elem := range pathElements {
if elem != "" {
child := parent.children[elem]
@ -189,7 +192,8 @@ func (root *FileInfo) LookUp(path string) *FileInfo {
func (info *FileInfo) path() string {
if info.parent == nil {
return "/"
// As this runs on the daemon side, file paths are OS specific.
return string(os.PathSeparator)
}
return filepath.Join(info.parent.path(), info.name)
}
@ -257,7 +261,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
// If there were changes inside this directory, we need to add it, even if the directory
// itself wasn't changed. This is needed to properly save and restore filesystem permissions.
if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" {
// As this runs on the daemon side, file paths are OS specific.
if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) {
change := Change{
Path: info.path(),
Kind: ChangeModify,
@ -279,8 +284,9 @@ func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
}
func newRootFileInfo() *FileInfo {
// As this runs on the daemon side, file paths are OS specific.
root := &FileInfo{
name: "/",
name: string(os.PathSeparator),
children: make(map[string]*FileInfo),
}
return root

View file

@ -6,6 +6,8 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/docker/docker/pkg/system"
)
@ -48,9 +50,20 @@ func collectFileInfo(sourceDir string) (*FileInfo, error) {
if err != nil {
return err
}
relPath = filepath.Join("/", relPath)
if relPath == "/" {
// As this runs on the daemon side, file paths are OS specific.
relPath = filepath.Join(string(os.PathSeparator), relPath)
// See https://github.com/golang/go/issues/9168 - bug in filepath.Join.
// Temporary workaround. If the returned path starts with two backslashes,
// trim it down to a single backslash. Only relevant on Windows.
if runtime.GOOS == "windows" {
if strings.HasPrefix(relPath, `\\`) {
relPath = relPath[1:]
}
}
if relPath == string(os.PathSeparator) {
return nil
}

View file

@ -7,9 +7,11 @@ import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/pools"
"github.com/docker/docker/pkg/system"
)
@ -40,12 +42,35 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
// Normalize name, for safety and for a simple is-root check
hdr.Name = filepath.Clean(hdr.Name)
if !strings.HasSuffix(hdr.Name, "/") {
// Windows does not support filenames with colons in them. Ignore
// these files. This is not a problem though (although it might
// appear that it is). Let's suppose a client is running docker pull.
// The daemon it points to is Windows. Would it make sense for the
// client to be doing a docker pull Ubuntu for example (which has files
// with colons in the name under /usr/share/man/man3)? No, absolutely
// not as it would really only make sense that they were pulling a
// Windows image. However, for development, it is necessary to be able
// to pull Linux images which are in the repository.
//
// TODO Windows. Once the registry is aware of what images are Windows-
// specific or Linux-specific, this warning should be changed to an error
// to cater for the situation where someone does manage to upload a Linux
// image but have it tagged as Windows inadvertantly.
if runtime.GOOS == "windows" {
if strings.Contains(hdr.Name, ":") {
logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
continue
}
}
// Note as these operations are platform specific, so must the slash be.
if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
// Not the root directory, ensure that the parent directory exists.
// This happened in some tests where an image had a tarfile without any
// parent directories.
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
err = system.MkdirAll(parentPath, 0600)
if err != nil {
@ -74,13 +99,14 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
}
continue
}
path := filepath.Join(dest, hdr.Name)
rel, err := filepath.Rel(dest, path)
if err != nil {
return 0, err
}
if strings.HasPrefix(rel, "../") {
// Note as these operations are platform specific, so must the slash be.
if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
base := filepath.Base(path)

View file

@ -1,68 +1,23 @@
package chrootarchive
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/docker/docker/pkg/system"
)
var chrootArchiver = &archive.Archiver{Untar: Untar}
func untar() {
runtime.LockOSThread()
flag.Parse()
var options *archive.TarOptions
if runtime.GOOS != "windows" {
//read the options from the pipe "ExtraFiles"
if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
fatal(err)
}
} else {
if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
fatal(err)
}
}
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
// Explanation of Windows difference. Windows does not support chroot.
// untar() is a helper function for the command line in the format
// "docker docker-untar directory input". In Windows, directory will be
// something like <pathto>\docker-buildnnnnnnnnn. So, just use that directory
// directly instead.
//
// One example of where this is used is in the docker build command where the
// dockerfile will be unpacked to the machine on which the daemon runs.
rootPath := "/"
if runtime.GOOS == "windows" {
rootPath = flag.Arg(0)
}
if err := archive.Unpack(os.Stdin, rootPath, options); err != nil {
fatal(err)
}
// fully consume stdin in case it is zero padded
flush(os.Stdin)
os.Exit(0)
}
// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive may be compressed with one of the following algorithms:
// identity (uncompressed), gzip, bzip2, xz.
func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
if tarArchive == nil {
return fmt.Errorf("Empty archive")
}
@ -84,67 +39,9 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
if err != nil {
return err
}
var data []byte
var r, w *os.File
defer decompressedArchive.Close()
if runtime.GOOS != "windows" {
// We can't pass a potentially large exclude list directly via cmd line
// because we easily overrun the kernel's max argument/environment size
// when the full image list is passed (e.g. when this is used by
// `docker load`). We will marshall the options via a pipe to the
// child
// This solution won't work on Windows as it will fail in golang
// exec_windows.go as at the lowest layer because attr.Files > 3
r, w, err = os.Pipe()
if err != nil {
return fmt.Errorf("Untar pipe failure: %v", err)
}
} else {
// We can't pass the exclude list directly via cmd line
// because we easily overrun the shell max argument list length
// when the full image list is passed (e.g. when this is used
// by `docker load`). Instead we will add the JSON marshalled
// and placed in the env, which has significantly larger
// max size
data, err = json.Marshal(options)
if err != nil {
return fmt.Errorf("Untar json encode: %v", err)
}
}
cmd := reexec.Command("docker-untar", dest)
cmd.Stdin = decompressedArchive
if runtime.GOOS != "windows" {
cmd.ExtraFiles = append(cmd.ExtraFiles, r)
output := bytes.NewBuffer(nil)
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Start(); err != nil {
return fmt.Errorf("Untar error on re-exec cmd: %v", err)
}
//write the options to the pipe for the untar exec to read
if err := json.NewEncoder(w).Encode(options); err != nil {
return fmt.Errorf("Untar json encode to pipe failed: %v", err)
}
w.Close()
if err := cmd.Wait(); err != nil {
return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
}
return nil
}
cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("Untar %s %s", err, out)
}
return nil
return invokeUnpack(decompressedArchive, dest, options)
}
// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
@ -165,8 +62,8 @@ func CopyWithTar(src, dst string) error {
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
// If `dst` ends with a trailing slash '/' ('\' on Windows), the final
// destination path will be `dst/base(src)` or `dst\base(src)`
func CopyFileWithTar(src, dst string) (err error) {
return chrootArchiver.CopyFileWithTar(src, dst)
}

View file

@ -3,7 +3,17 @@
package chrootarchive
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"runtime"
"syscall"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
)
func chroot(path string) error {
@ -12,3 +22,64 @@ func chroot(path string) error {
}
return syscall.Chdir("/")
}
// untar is the entry-point for docker-untar on re-exec. This is not used on
// Windows as it does not support chroot, hence no point sandboxing through
// chroot and rexec.
func untar() {
runtime.LockOSThread()
flag.Parse()
var options *archive.TarOptions
//read the options from the pipe "ExtraFiles"
if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
fatal(err)
}
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
if err := archive.Unpack(os.Stdin, "/", options); err != nil {
fatal(err)
}
// fully consume stdin in case it is zero padded
flush(os.Stdin)
os.Exit(0)
}
func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
// We can't pass a potentially large exclude list directly via cmd line
// because we easily overrun the kernel's max argument/environment size
// when the full image list is passed (e.g. when this is used by
// `docker load`). We will marshall the options via a pipe to the
// child
r, w, err := os.Pipe()
if err != nil {
return fmt.Errorf("Untar pipe failure: %v", err)
}
cmd := reexec.Command("docker-untar", dest)
cmd.Stdin = decompressedArchive
cmd.ExtraFiles = append(cmd.ExtraFiles, r)
output := bytes.NewBuffer(nil)
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Start(); err != nil {
return fmt.Errorf("Untar error on re-exec cmd: %v", err)
}
//write the options to the pipe for the untar exec to read
if err := json.NewEncoder(w).Encode(options); err != nil {
return fmt.Errorf("Untar json encode to pipe failed: %v", err)
}
w.Close()
if err := cmd.Wait(); err != nil {
return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
}
return nil
}

View file

@ -1,6 +1,21 @@
package chrootarchive
import (
"io"
"github.com/docker/docker/pkg/archive"
)
// chroot is not supported by Windows
func chroot(path string) error {
return nil
}
func invokeUnpack(decompressedArchive io.ReadCloser,
dest string,
options *archive.TarOptions) error {
// Windows is different to Linux here because Windows does not support
// chroot. Hence there is no point sandboxing a chrooted process to
// do the unpack. We call inline instead within the daemon process.
return archive.Unpack(decompressedArchive, dest, options)
}

View file

@ -1,3 +1,5 @@
//+build !windows
package chrootarchive
import (
@ -19,41 +21,35 @@ type applyLayerResponse struct {
LayerSize int64 `json:"layerSize"`
}
// applyLayer is the entry-point for docker-applylayer on re-exec. This is not
// used on Windows as it does not support chroot, hence no point sandboxing
// through chroot and rexec.
func applyLayer() {
var (
root = "/"
tmpDir = ""
err error
)
runtime.LockOSThread()
flag.Parse()
if runtime.GOOS != "windows" {
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
// We need to be able to set any perms
oldmask, err := system.Umask(0)
defer system.Umask(oldmask)
if err != nil {
fatal(err)
}
} else {
// As Windows does not support chroot or umask, we use the directory
// passed in which will be <pathto>\docker-buildnnnnnnnn instead of
// the 'chroot-root', "/"
root = flag.Arg(0)
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
if tmpDir, err = ioutil.TempDir(root, "temp-docker-extract"); err != nil {
// We need to be able to set any perms
oldmask, err := system.Umask(0)
defer system.Umask(oldmask)
if err != nil {
fatal(err)
}
if tmpDir, err = ioutil.TempDir("/", "temp-docker-extract"); err != nil {
fatal(err)
}
os.Setenv("TMPDIR", tmpDir)
size, err := archive.UnpackLayer(root, os.Stdin)
size, err := archive.UnpackLayer("/", os.Stdin)
os.RemoveAll(tmpDir)
if err != nil {
fatal(err)

View file

@ -0,0 +1,32 @@
package chrootarchive
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/docker/docker/pkg/archive"
)
func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
dest = filepath.Clean(dest)
decompressed, err := archive.DecompressStream(layer)
if err != nil {
return 0, err
}
defer decompressed.Close()
tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract")
if err != nil {
return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err)
}
s, err := archive.UnpackLayer(dest, decompressed)
os.RemoveAll(tmpDir)
if err != nil {
return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest)
}
return s, nil
}

View file

@ -1,3 +1,5 @@
// +build !windows
package chrootarchive
import (
@ -10,8 +12,8 @@ import (
)
func init() {
reexec.Register("docker-untar", untar)
reexec.Register("docker-applyLayer", applyLayer)
reexec.Register("docker-untar", untar)
}
func fatal(err error) {

View file

@ -0,0 +1,4 @@
package chrootarchive
func init() {
}