diff --git a/builder/dockerfile/parser/line_parsers.go b/builder/dockerfile/parser/line_parsers.go index 1d7ece43dc..adf15ed5a5 100644 --- a/builder/dockerfile/parser/line_parsers.go +++ b/builder/dockerfile/parser/line_parsers.go @@ -94,12 +94,12 @@ func parseWords(rest string) []string { blankOK = true phase = inQuote } - if ch == '\\' { + if ch == tokenEscape { if pos+1 == len(rest) { - continue // just skip \ at end + continue // just skip an escape token at end of line } - // If we're not quoted and we see a \, then always just - // add \ plus the char to the word, even if the char + // If we're not quoted and we see an escape token, then always just + // add the escape token plus the char to the word, even if the char // is a quote. word += string(ch) pos++ @@ -112,11 +112,11 @@ func parseWords(rest string) []string { if ch == quote { phase = inWord } - // \ is special except for ' quotes - can't escape anything for ' - if ch == '\\' && quote != '\'' { + // The escape token is special except for ' quotes - can't escape anything for ' + if ch == tokenEscape && quote != '\'' { if pos+1 == len(rest) { phase = inWord - continue // just skip \ at end + continue // just skip the escape token at end } pos++ nextCh := rune(rest[pos]) diff --git a/builder/dockerfile/parser/parser.go b/builder/dockerfile/parser/parser.go index ece601a957..e42904fef8 100644 --- a/builder/dockerfile/parser/parser.go +++ b/builder/dockerfile/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( "bufio" + "fmt" "io" "regexp" "strings" @@ -37,10 +38,26 @@ type Node struct { var ( dispatch map[string]func(string) (*Node, map[string]bool, error) tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`) - tokenLineContinuation = regexp.MustCompile(`\\[ \t]*$`) + tokenLineContinuation *regexp.Regexp + tokenEscape rune + tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P.).*$`) tokenComment = regexp.MustCompile(`^#.*$`) + lookingForDirectives bool + directiveEscapeSeen bool ) +const defaultTokenEscape = "\\" + +// setTokenEscape sets the default token for escaping characters in a Dockerfile. +func setTokenEscape(s string) error { + if s != "`" && s != "\\" { + return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s) + } + tokenEscape = rune(s[0]) + tokenLineContinuation = regexp.MustCompile(`\` + s + `[ \t]*$`) + return nil +} + func init() { // Dispatch Table. see line_parsers.go for the parse functions. // The command is parsed and mapped to the line parser. The line parser @@ -70,6 +87,29 @@ func init() { // ParseLine parse a line and return the remainder. func ParseLine(line string) (string, *Node, error) { + + // Handle the parser directive '# escape=. Parser directives must preceed + // any builder instruction or other comments, and cannot be repeated. + if lookingForDirectives { + tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line)) + if len(tecMatch) > 0 { + if directiveEscapeSeen == true { + return "", nil, fmt.Errorf("only one escape parser directive can be used") + } + for i, n := range tokenEscapeCommand.SubexpNames() { + if n == "escapechar" { + if err := setTokenEscape(tecMatch[i]); err != nil { + return "", nil, err + } + directiveEscapeSeen = true + return "", nil, nil + } + } + } + } + + lookingForDirectives = false + if line = stripComments(line); line == "" { return "", nil, nil } @@ -103,6 +143,9 @@ func ParseLine(line string) (string, *Node, error) { // Parse is the main parse routine. // It handles an io.ReadWriteCloser and returns the root of the AST. func Parse(rwc io.Reader) (*Node, error) { + directiveEscapeSeen = false + lookingForDirectives = true + setTokenEscape(defaultTokenEscape) // Assume the default token for escape currentLine := 0 root := &Node{} root.StartLine = -1 diff --git a/builder/dockerfile/parser/parser_test.go b/builder/dockerfile/parser/parser_test.go index 983a590a62..4025186ba6 100644 --- a/builder/dockerfile/parser/parser_test.go +++ b/builder/dockerfile/parser/parser_test.go @@ -131,22 +131,22 @@ func TestLineInformation(t *testing.T) { t.Fatalf("Error parsing dockerfile %s: %v", testFileLineInfo, err) } - if ast.StartLine != 4 || ast.EndLine != 30 { - fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 4, 30, ast.StartLine, ast.EndLine) + if ast.StartLine != 5 || ast.EndLine != 31 { + fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 5, 31, ast.StartLine, ast.EndLine) t.Fatalf("Root line information doesn't match result.") } if len(ast.Children) != 3 { fmt.Fprintf(os.Stderr, "Wrong number of child: expected(%d), actual(%d)\n", 3, len(ast.Children)) - t.Fatalf("Root line information doesn't match result.") + t.Fatalf("Root line information doesn't match result for %s", testFileLineInfo) } expected := [][]int{ - {4, 4}, - {10, 11}, - {16, 30}, + {5, 5}, + {11, 12}, + {17, 31}, } for i, child := range ast.Children { if child.StartLine != expected[i][0] || child.EndLine != expected[i][1] { - fmt.Fprintf(os.Stderr, "Wrong line information for child %d: expected(%d-%d), actual(%d-%d)\n", + t.Logf("Wrong line information for child %d: expected(%d-%d), actual(%d-%d)\n", i, expected[i][0], expected[i][1], child.StartLine, child.EndLine) t.Fatalf("Root line information doesn't match result.") } diff --git a/builder/dockerfile/parser/testfile-line/Dockerfile b/builder/dockerfile/parser/testfile-line/Dockerfile index 0e77e85e4f..c7601c9f69 100644 --- a/builder/dockerfile/parser/testfile-line/Dockerfile +++ b/builder/dockerfile/parser/testfile-line/Dockerfile @@ -1,3 +1,4 @@ +# ESCAPE=\ diff --git a/builder/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile b/builder/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile index 5c75a2e0ca..0364ef9d96 100644 --- a/builder/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile +++ b/builder/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile @@ -1,3 +1,4 @@ +#escape=\ FROM brimstone/ubuntu:14.04 MAINTAINER brimstone@the.narro.ws diff --git a/builder/dockerfile/parser/testfiles/escape-after-comment/Dockerfile b/builder/dockerfile/parser/testfiles/escape-after-comment/Dockerfile new file mode 100644 index 0000000000..6def7efdcd --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape-after-comment/Dockerfile @@ -0,0 +1,9 @@ +# Comment here. Should not be looking for the following parser directive. +# Hence the following line will be ignored, and the subsequent backslash +# continuation will be the default. +# escape = ` + +FROM image +MAINTAINER foo@bar.com +ENV GOPATH \ +\go \ No newline at end of file diff --git a/builder/dockerfile/parser/testfiles/escape-after-comment/result b/builder/dockerfile/parser/testfiles/escape-after-comment/result new file mode 100644 index 0000000000..21522a880b --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape-after-comment/result @@ -0,0 +1,3 @@ +(from "image") +(maintainer "foo@bar.com") +(env "GOPATH" "\\go") diff --git a/builder/dockerfile/parser/testfiles/escape-nonewline/Dockerfile b/builder/dockerfile/parser/testfiles/escape-nonewline/Dockerfile new file mode 100644 index 0000000000..08a8cc4326 --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape-nonewline/Dockerfile @@ -0,0 +1,7 @@ +# escape = `` +# There is no white space line after the directives. This still succeeds, but goes +# against best practices. +FROM image +MAINTAINER foo@bar.com +ENV GOPATH ` +\go \ No newline at end of file diff --git a/builder/dockerfile/parser/testfiles/escape-nonewline/result b/builder/dockerfile/parser/testfiles/escape-nonewline/result new file mode 100644 index 0000000000..21522a880b --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape-nonewline/result @@ -0,0 +1,3 @@ +(from "image") +(maintainer "foo@bar.com") +(env "GOPATH" "\\go") diff --git a/builder/dockerfile/parser/testfiles/escape/Dockerfile b/builder/dockerfile/parser/testfiles/escape/Dockerfile new file mode 100644 index 0000000000..ef30414a5e --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape/Dockerfile @@ -0,0 +1,6 @@ +#escape = ` + +FROM image +MAINTAINER foo@bar.com +ENV GOPATH ` +\go \ No newline at end of file diff --git a/builder/dockerfile/parser/testfiles/escape/result b/builder/dockerfile/parser/testfiles/escape/result new file mode 100644 index 0000000000..21522a880b --- /dev/null +++ b/builder/dockerfile/parser/testfiles/escape/result @@ -0,0 +1,3 @@ +(from "image") +(maintainer "foo@bar.com") +(env "GOPATH" "\\go") diff --git a/docs/reference/builder.md b/docs/reference/builder.md index f1838ee0ea..0f987bb778 100644 --- a/docs/reference/builder.md +++ b/docs/reference/builder.md @@ -106,27 +106,197 @@ repository to its registry*](../userguide/containers/dockerrepos.md#contributing Here is the format of the `Dockerfile`: - # Comment - INSTRUCTION arguments +```Dockerfile +# Comment +INSTRUCTION arguments +``` -The instruction is not case-sensitive, however convention is for them to -be UPPERCASE in order to distinguish them from arguments more easily. +The instruction is not case-sensitive. However, convention is for them to +be UPPERCASE to distinguish them from arguments more easily. -Docker runs the instructions in a `Dockerfile` in order. **The -first instruction must be \`FROM\`** in order to specify the [*Base -Image*](glossary.md#base-image) from which you are building. -Docker will treat lines that *begin* with `#` as a -comment. A `#` marker anywhere else in the line will -be treated as an argument. This allows statements like: +Docker runs instructions in a `Dockerfile` in order. **The first +instruction must be \`FROM\`** in order to specify the [*Base +Image*](glossary.md#base-image) from which you are building. - # Comment - RUN echo 'we are running some # of cool things' +Docker treats lines that *begin* with `#` as a comment, unless the line is +a valid [parser directive](builder.md#parser directives). A `#` marker anywhere +else in a line is treated as an argument. This allows statements like: -Here is the set of instructions you can use in a `Dockerfile` for building -images. +```Dockerfile +# Comment +RUN echo 'we are running some # of cool things' +``` -### Environment replacement +Line continuation characters are not supported in comments. + +## Parser directives + +Parser directives are optional, and affect the way in which subsequent lines +in a `Dockerfile` are handled. Parser directives do not add layers to the build, +and will not be shown as a build step. Parser directives are written as a +special type of comment in the form `# directive=value`. A single directive +may only be used once. + +Once a comment, empty line or builder instruction has been processed, Docker +no longer looks for parser directives. Instead it treats anything formatted +as a parser directive as a comment and does not attempt to validate if it might +be a parser directive. Therefore, all parser directives must be at the very +top of a `Dockerfile`. + +Parser directives are not case-sensitive. However, convention is for them to +be lowercase. Convention is also to include a blank line following any +parser directives. Line continuation characters are not supported in parser +directives. + +Due to these rules, the following examples are all invalid: + +Invalid due to line continuation: + +```Dockerfile +# direc \ +tive=value +``` + +Invalid due to appearing twice: + +```Dockerfile +# directive=value1 +# directive=value2 + +FROM ImageName +``` + +Treated as a comment due to appearing after a builder instruction: + +```Dockerfile +FROM ImageName +# directive=value +``` + +Treated as a comment due to appearing after a comment which is not a parser +directive: + +```Dockerfile +# About my dockerfile +FROM ImageName +# directive=value +``` + +The unknown directive is treated as a comment due to not being recognized. In +addition, the known directive is treated as a comment due to appearing after +a comment which is not a parser directive. + +```Dockerfile +# unknowndirective=value +# knowndirective=value +``` + +Non line-breaking whitespace is permitted in a parser directive. Hence, the +following lines are all treated identically: + +```Dockerfile +#directive=value +# directive =value +# directive= value +# directive = value +# dIrEcTiVe=value +``` + +The following parser directive is supported: + +* `escape` + +## escape + + # escape=\ (backslash) + +Or + + # escape=` (backtick) + +The `escape` directive sets the character used to escape characters in a +`Dockerfile`. If not specified, the default escape character is `\`. + +The escape character is used both to escape characters in a line, and to +escape a newline. This allows a `Dockerfile` instruction to +span multiple lines. Note that regardless of whether the `escape` parser +directive is included in a `Dockerfile`, *escaping is not performed in +a `RUN` command, except at the end of a line.* + +Setting the escape character to `` ` `` is especially useful on +`Windows`, where `\` is the directory path separator. `` ` `` is consistent +with [Windows PowerShell](https://technet.microsoft.com/en-us/library/hh847755.aspx). + +Consider the following example which would fail in a non-obvious way on +`Windows`. The second `\` at the end of the second line would be interpreted as an +escape for the newline, instead of a target of the escape from the first `\`. +Similarly, the `\` at the end of the third line would, assuming it was actually +handled as an instruction, cause it be treated as a line continuation. The result +of this dockerfile is that second and third lines are considered a single +instruction: + +```Dockerfile +FROM windowsservercore +COPY testfile.txt c:\\ +RUN dir c:\ +``` + +Results in: + + PS C:\John> docker build -t cmd . + Sending build context to Docker daemon 3.072 kB + Step 1 : FROM windowsservercore + ---> dbfee88ee9fd + Step 2 : COPY testfile.txt c:RUN dir c: + GetFileAttributesEx c:RUN: The system cannot find the file specified. + PS C:\John> + +One solution to the above would be to use `/` as the target of both the `COPY` +instruction, and `dir`. However, this syntax is, at best, confusing as it is not +natural for paths on `Windows`, and at worst, error prone as not all commands on +`Windows` support `/` as the path separator. + +By adding the `escape` parser directive, the following `Dockerfile` succeeds as +expected with the use of natural platform semantics for file paths on `Windows`: + + # escape=` + + FROM windowsservercore + COPY testfile.txt c:\ + RUN dir c:\ + +Results in: + + PS C:\John> docker build -t succeeds --no-cache=true . + Sending build context to Docker daemon 3.072 kB + Step 1 : FROM windowsservercore + ---> dbfee88ee9fd + Step 2 : COPY testfile.txt c:\ + ---> 99ceb62e90df + Removing intermediate container 62afbe726221 + Step 3 : RUN dir c:\ + ---> Running in a5ff53ad6323 + Volume in drive C has no label. + Volume Serial Number is 1440-27FA + + Directory of c:\ + + 03/25/2016 05:28 AM inetpub + 03/25/2016 04:22 AM PerfLogs + 04/22/2016 10:59 PM Program Files + 03/25/2016 04:22 AM Program Files (x86) + 04/18/2016 09:26 AM 4 testfile.txt + 04/22/2016 10:59 PM Users + 04/22/2016 10:59 PM Windows + 1 File(s) 4 bytes + 6 Dir(s) 21,252,689,920 bytes free + ---> 2569aa19abef + Removing intermediate container a5ff53ad6323 + Successfully built 2569aa19abef + PS C:\John> + +## Environment replacement Environment variables (declared with [the `ENV` statement](#env)) can also be used in certain instructions as variables to be interpreted by the @@ -192,7 +362,7 @@ will result in `def` having a value of `hello`, not `bye`. However, `ghi` will have a value of `bye` because it is not part of the same command that set `abc` to `bye`. -### .dockerignore file +## .dockerignore file Before the docker CLI sends the context to the docker daemon, it looks for a file named `.dockerignore` in the root directory of the context. diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 392a0efccc..49e4faf0d0 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -3353,9 +3353,10 @@ func (s *DockerSuite) TestBuildAddToSymlinkDest(c *check.C) { } func (s *DockerSuite) TestBuildEscapeWhitespace(c *check.C) { - name := "testbuildescaping" + name := "testbuildescapewhitespace" _, err := buildImage(name, ` + # ESCAPE=\ FROM busybox MAINTAINER "Docker \ IO