Support platform file paths through escape

Signed-off-by: John Howard <jhoward@microsoft.com>

builder/dockerfile/parser/line_parsers.go

@@ -94,12 +94,12 @@ func parseWords(rest string) []string {
 				blankOK = true
 				phase = inQuote
 			}
-			if ch == '\\' {
+			if ch == tokenEscape {
 				if pos+1 == len(rest) {
-					continue // just skip \ at end
+					continue // just skip an escape token at end of line
 				}
-				// If we're not quoted and we see a \, then always just
-				// add \ plus the char to the word, even if the char
+				// If we're not quoted and we see an escape token, then always just
+				// add the escape token plus the char to the word, even if the char
 				// is a quote.
 				word += string(ch)
 				pos++
@@ -112,11 +112,11 @@ func parseWords(rest string) []string {
 			if ch == quote {
 				phase = inWord
 			}
-			// \ is special except for ' quotes - can't escape anything for '
-			if ch == '\\' && quote != '\'' {
+			// The escape token is special except for ' quotes - can't escape anything for '
+			if ch == tokenEscape && quote != '\'' {
 				if pos+1 == len(rest) {
 					phase = inWord
-					continue // just skip \ at end
+					continue // just skip the escape token at end
 				}
 				pos++
 				nextCh := rune(rest[pos])

builder/dockerfile/parser/parser.go

@@ -3,6 +3,7 @@ package parser
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"regexp"
 	"strings"
@@ -37,10 +38,26 @@ type Node struct {
 var (
 	dispatch              map[string]func(string) (*Node, map[string]bool, error)
 	tokenWhitespace       = regexp.MustCompile(`[\t\v\f\r ]+`)
-	tokenLineContinuation = regexp.MustCompile(`\\[ \t]*$`)
+	tokenLineContinuation *regexp.Regexp
+	tokenEscape           rune
+	tokenEscapeCommand    = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
 	tokenComment          = regexp.MustCompile(`^#.*$`)
+	lookingForDirectives  bool
+	directiveEscapeSeen   bool
 )
 
+const defaultTokenEscape = "\\"
+
+// setTokenEscape sets the default token for escaping characters in a Dockerfile.
+func setTokenEscape(s string) error {
+	if s != "`" && s != "\\" {
+		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
+	}
+	tokenEscape = rune(s[0])
+	tokenLineContinuation = regexp.MustCompile(`\` + s + `[ \t]*$`)
+	return nil
+}
+
 func init() {
 	// Dispatch Table. see line_parsers.go for the parse functions.
 	// The command is parsed and mapped to the line parser. The line parser
@@ -70,6 +87,29 @@ func init() {
 
 // ParseLine parse a line and return the remainder.
 func ParseLine(line string) (string, *Node, error) {
+
+	// Handle the parser directive '# escape=<char>. Parser directives must preceed
+	// any builder instruction or other comments, and cannot be repeated.
+	if lookingForDirectives {
+		tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
+		if len(tecMatch) > 0 {
+			if directiveEscapeSeen == true {
+				return "", nil, fmt.Errorf("only one escape parser directive can be used")
+			}
+			for i, n := range tokenEscapeCommand.SubexpNames() {
+				if n == "escapechar" {
+					if err := setTokenEscape(tecMatch[i]); err != nil {
+						return "", nil, err
+					}
+					directiveEscapeSeen = true
+					return "", nil, nil
+				}
+			}
+		}
+	}
+
+	lookingForDirectives = false
+
 	if line = stripComments(line); line == "" {
 		return "", nil, nil
 	}
@@ -103,6 +143,9 @@ func ParseLine(line string) (string, *Node, error) {
 // Parse is the main parse routine.
 // It handles an io.ReadWriteCloser and returns the root of the AST.
 func Parse(rwc io.Reader) (*Node, error) {
+	directiveEscapeSeen = false
+	lookingForDirectives = true
+	setTokenEscape(defaultTokenEscape) // Assume the default token for escape
 	currentLine := 0
 	root := &Node{}
 	root.StartLine = -1

builder/dockerfile/parser/parser_test.go

@@ -131,22 +131,22 @@ func TestLineInformation(t *testing.T) {
 		t.Fatalf("Error parsing dockerfile %s: %v", testFileLineInfo, err)
 	}
 
-	if ast.StartLine != 4 || ast.EndLine != 30 {
-		fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 4, 30, ast.StartLine, ast.EndLine)
+	if ast.StartLine != 5 || ast.EndLine != 31 {
+		fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 5, 31, ast.StartLine, ast.EndLine)
 		t.Fatalf("Root line information doesn't match result.")
 	}
 	if len(ast.Children) != 3 {
 		fmt.Fprintf(os.Stderr, "Wrong number of child: expected(%d), actual(%d)\n", 3, len(ast.Children))
-		t.Fatalf("Root line information doesn't match result.")
+		t.Fatalf("Root line information doesn't match result for %s", testFileLineInfo)
 	}
 	expected := [][]int{
-		{4, 4},
-		{10, 11},
-		{16, 30},
+		{5, 5},
+		{11, 12},
+		{17, 31},
 	}
 	for i, child := range ast.Children {
 		if child.StartLine != expected[i][0] || child.EndLine != expected[i][1] {
-			fmt.Fprintf(os.Stderr, "Wrong line information for child %d: expected(%d-%d), actual(%d-%d)\n",
+			t.Logf("Wrong line information for child %d: expected(%d-%d), actual(%d-%d)\n",
 				i, expected[i][0], expected[i][1], child.StartLine, child.EndLine)
 			t.Fatalf("Root line information doesn't match result.")
 		}

builder/dockerfile/parser/testfile-line/Dockerfile

@@ -1,3 +1,4 @@
+# ESCAPE=\
 
 
 

builder/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile

@@ -1,3 +1,4 @@
+#escape=\
 FROM brimstone/ubuntu:14.04
 
 MAINTAINER brimstone@the.narro.ws

builder/dockerfile/parser/testfiles/escape-after-comment/Dockerfile

@@ -0,0 +1,9 @@
+# Comment here. Should not be looking for the following parser directive.
+# Hence the following line will be ignored, and the subsequent backslash
+# continuation will be the default.
+# escape = `
+
+FROM image
+MAINTAINER foo@bar.com
+ENV GOPATH \
+\go

builder/dockerfile/parser/testfiles/escape-after-comment/result

@@ -0,0 +1,3 @@
+(from "image")
+(maintainer "foo@bar.com")
+(env "GOPATH" "\\go")

builder/dockerfile/parser/testfiles/escape-nonewline/Dockerfile

@@ -0,0 +1,7 @@
+# escape = ``
+# There is no white space line after the directives. This still succeeds, but goes
+# against best practices.
+FROM image
+MAINTAINER foo@bar.com
+ENV GOPATH `
+\go

builder/dockerfile/parser/testfiles/escape-nonewline/result

@@ -0,0 +1,3 @@
+(from "image")
+(maintainer "foo@bar.com")
+(env "GOPATH" "\\go")

builder/dockerfile/parser/testfiles/escape/Dockerfile

@@ -0,0 +1,6 @@
+#escape = `
+
+FROM image
+MAINTAINER foo@bar.com
+ENV GOPATH `
+\go

builder/dockerfile/parser/testfiles/escape/result

@@ -0,0 +1,3 @@
+(from "image")
+(maintainer "foo@bar.com")
+(env "GOPATH" "\\go")

docs/reference/builder.md

@@ -106,27 +106,197 @@ repository to its registry*](../userguide/containers/dockerrepos.md#contributing
 
 Here is the format of the `Dockerfile`:
 
-    # Comment
-    INSTRUCTION arguments
+```Dockerfile
+# Comment
+INSTRUCTION arguments
+```
 
-The instruction is not case-sensitive, however convention is for them to
-be UPPERCASE in order to distinguish them from arguments more easily.
+The instruction is not case-sensitive. However, convention is for them to
+be UPPERCASE to distinguish them from arguments more easily.
 
-Docker runs the instructions in a `Dockerfile` in order. **The
-first instruction must be \`FROM\`** in order to specify the [*Base
-Image*](glossary.md#base-image) from which you are building.
 
-Docker will treat lines that *begin* with `#` as a
-comment. A `#` marker anywhere else in the line will
-be treated as an argument. This allows statements like:
+Docker runs instructions in a `Dockerfile` in order. **The first 
+instruction must be \`FROM\`** in order to specify the [*Base
+Image*](glossary.md#base-image) from which you are building. 
 
-    # Comment
-    RUN echo 'we are running some # of cool things'
+Docker treats lines that *begin* with `#` as a comment, unless the line is 
+a valid [parser directive](builder.md#parser directives). A `#` marker anywhere
+else in a line is treated as an argument. This allows statements like:
 
-Here is the set of instructions you can use in a `Dockerfile` for building
-images.
+```Dockerfile
+# Comment
+RUN echo 'we are running some # of cool things'
+```
 
-### Environment replacement
+Line continuation characters are not supported in comments.
+
+## Parser directives
+
+Parser directives are optional, and affect the way in which subsequent lines 
+in a `Dockerfile` are handled. Parser directives do not add layers to the build,
+and will not be shown as a build step. Parser directives are written as a
+special type of comment in the form `# directive=value`. A single directive
+may only be used once.
+
+Once a comment, empty line or builder instruction has been processed, Docker 
+no longer looks for parser directives. Instead it treats anything formatted
+as a parser directive as a comment and does not attempt to validate if it might
+be a parser directive. Therefore, all parser directives must be at the very
+top of a `Dockerfile`. 
+
+Parser directives are not case-sensitive. However, convention is for them to
+be lowercase. Convention is also to include a blank line following any 
+parser directives. Line continuation characters are not supported in parser
+directives.
+
+Due to these rules, the following examples are all invalid:
+
+Invalid due to line continuation:
+
+```Dockerfile
+# direc \
+tive=value
+```
+
+Invalid due to appearing twice:
+
+```Dockerfile
+# directive=value1
+# directive=value2
+
+FROM ImageName
+```
+    
+Treated as a comment due to appearing after a builder instruction:
+
+```Dockerfile
+FROM ImageName
+# directive=value
+```
+
+Treated as a comment due to appearing after a comment which is not a parser
+directive:
+
+```Dockerfile
+# About my dockerfile
+FROM ImageName
+# directive=value
+```
+
+The unknown directive is treated as a comment due to not being recognized. In
+addition, the known directive is treated as a comment due to appearing after
+a comment which is not a parser directive.
+
+```Dockerfile
+# unknowndirective=value
+# knowndirective=value
+```    
+    
+Non line-breaking whitespace is permitted in a parser directive. Hence, the
+following lines are all treated identically: 
+
+```Dockerfile
+#directive=value
+# directive =value
+#	directive= value
+# directive = value
+#	  dIrEcTiVe=value
+```
+
+The following parser directive is supported:
+
+* `escape`
+
+## escape
+
+    # escape=\ (backslash)
+
+Or
+
+    # escape=` (backtick)
+
+The `escape` directive sets the character used to escape characters in a 
+`Dockerfile`. If not specified, the default escape character is `\`. 
+
+The escape character is used both to escape characters in a line, and to
+escape a newline. This allows a `Dockerfile` instruction to
+span multiple lines. Note that regardless of whether the `escape` parser
+directive is included in a `Dockerfile`, *escaping is not performed in 
+a `RUN` command, except at the end of a line.* 
+
+Setting the escape character to `` ` `` is especially useful on 
+`Windows`, where `\` is the directory path separator. `` ` `` is consistent 
+with [Windows PowerShell](https://technet.microsoft.com/en-us/library/hh847755.aspx).
+
+Consider the following example which would fail in a non-obvious way on 
+`Windows`. The second `\` at the end of the second line would be interpreted as an
+escape for the newline, instead of a target of the escape from the first `\`. 
+Similarly, the `\` at the end of the third line would, assuming it was actually
+handled as an instruction, cause it be treated as a line continuation. The result
+of this dockerfile is that second and third lines are considered a single
+instruction: 
+
+```Dockerfile
+FROM windowsservercore
+COPY testfile.txt c:\\
+RUN dir c:\
+```
+
+Results in:
+
+    PS C:\John> docker build -t cmd .
+    Sending build context to Docker daemon 3.072 kB
+    Step 1 : FROM windowsservercore
+     ---> dbfee88ee9fd
+    Step 2 : COPY testfile.txt c:RUN dir c:
+    GetFileAttributesEx c:RUN: The system cannot find the file specified.
+    PS C:\John> 
+
+One solution to the above would be to use `/` as the target of both the `COPY`
+instruction, and `dir`. However, this syntax is, at best, confusing as it is not
+natural for paths on `Windows`, and at worst, error prone as not all commands on
+`Windows` support `/` as the path separator.
+
+By adding the `escape` parser directive, the following `Dockerfile` succeeds as 
+expected with the use of natural platform semantics for file paths on `Windows`:
+
+    # escape=`
+    
+    FROM windowsservercore
+    COPY testfile.txt c:\
+    RUN dir c:\
+
+Results in:
+
+    PS C:\John> docker build -t succeeds --no-cache=true .
+    Sending build context to Docker daemon 3.072 kB
+    Step 1 : FROM windowsservercore
+     ---> dbfee88ee9fd
+    Step 2 : COPY testfile.txt c:\
+     ---> 99ceb62e90df
+    Removing intermediate container 62afbe726221
+    Step 3 : RUN dir c:\
+     ---> Running in a5ff53ad6323
+     Volume in drive C has no label.
+     Volume Serial Number is 1440-27FA
+    
+     Directory of c:\
+    
+    03/25/2016  05:28 AM    <DIR>          inetpub
+    03/25/2016  04:22 AM    <DIR>          PerfLogs
+    04/22/2016  10:59 PM    <DIR>          Program Files
+    03/25/2016  04:22 AM    <DIR>          Program Files (x86)
+    04/18/2016  09:26 AM                 4 testfile.txt
+    04/22/2016  10:59 PM    <DIR>          Users
+    04/22/2016  10:59 PM    <DIR>          Windows
+                   1 File(s)              4 bytes
+                   6 Dir(s)  21,252,689,920 bytes free
+     ---> 2569aa19abef
+    Removing intermediate container a5ff53ad6323
+    Successfully built 2569aa19abef
+    PS C:\John>
+
+## Environment replacement
 
 Environment variables (declared with [the `ENV` statement](#env)) can also be
 used in certain instructions as variables to be interpreted by the
@@ -192,7 +362,7 @@ will result in `def` having a value of `hello`, not `bye`. However,
 `ghi` will have a value of `bye` because it is not part of the same command
 that set `abc` to `bye`.
 
-### .dockerignore file
+## .dockerignore file
 
 Before the docker CLI sends the context to the docker daemon, it looks
 for a file named `.dockerignore` in the root directory of the context.

integration-cli/docker_cli_build_test.go

@@ -3353,9 +3353,10 @@ func (s *DockerSuite) TestBuildAddToSymlinkDest(c *check.C) {
 }
 
 func (s *DockerSuite) TestBuildEscapeWhitespace(c *check.C) {
-	name := "testbuildescaping"
+	name := "testbuildescapewhitespace"
 
 	_, err := buildImage(name, `
+  # ESCAPE=\
   FROM busybox
   MAINTAINER "Docker \
 IO <io@\